wader/fq
1
1
Fork 0
mirror of https://github.com/wader/fq.git synced 2024-12-27 07:24:48 +03:00
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'master' into postgres_merge_master_0

Browse Source
This commit is contained in:
Pavel Safonov 2023-05-03 09:44:02 +03:00
commit bb2659d442
1456 changed files with 264509 additions and 21580 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/ci.yml vendored
View File

@ -7,7 +7,7 @@ on:
pull_request:
env:
GOLANGCILINT_VERSION: 1.50.1
GOLANGCILINT_VERSION: "1.52.2"
jobs:
lint:
@ -15,7 +15,7 @@ jobs:
steps:
- uses: actions/setup-go@v3
with:
go-version: 1.19.3
go-version: "1.20.3"
- uses: actions/checkout@v3
- uses: golangci/golangci-lint-action@v3
with:
@ -47,7 +47,7 @@ jobs:
- uses: actions/checkout@v3
- uses: actions/setup-go@v3
with:
go-version: 1.19.3
go-version: "1.20.3"
- name: Test
env:
GOARCH: ${{ matrix.goarch }}

2
.github/workflows/release.yml vendored
View File

@ -15,7 +15,7 @@ jobs:
- uses: actions/checkout@v3
- uses: actions/setup-go@v3
with:
go-version: 1.19.3
go-version: "1.20.3"
- uses: goreleaser/goreleaser-action@v2
with:
distribution: goreleaser

8
.golangci.yml
View File

@ -23,7 +23,12 @@ linters:
- unconvert
- unparam
- wastedassign
linters-settings:
revive:
rules:
- name: unused-parameter
disabled: true
exhaustive:
default-signifies-exhaustive: true
gosec:
@ -36,9 +41,12 @@ linters-settings:
- rela
- equalisation
- synchronisation
run:
timeout: 5m
skip-dirs:
# allow md5
- dev
- doc
# ignore warnings in code from crypto/tls and zmap/zcrypto
- format/tls/tlsdecrypt

17
.goreleaser.yml
View File

@ -1,3 +1,5 @@
# to test
# docker run -ti -v "$PWD:$PWD" -w "$PWD" goreleaser/goreleaser:latest release --snapshot --rm-dist
project_name: fq
before:
@ -30,17 +32,22 @@ checksum:
archives:
- files:
# skip all other files
- none*
rlcp: true
format_overrides:
- goos: windows
format: zip
- goos: darwin
format: zip
replacements:
darwin: macos
snapshot:
name_template: "{{.Tag}}-next"
# fq_0.2.0_linux_amd64.tar.gz
# fq_0.2.0_macos_arm64.zip
name_template: >-
{{ .ProjectName }}_
{{- .Version }}_
{{- if eq .Os "darwin" }}macos_
{{- else }}{{ .Os }}_{{ end }}
{{- .Arch }}
changelog:
sort: asc

16
.vscode/launch.tmpl.json vendored Normal file
View File

@ -0,0 +1,16 @@
{
"version": "0.2.0",
"configurations": [
{
"name": "Debug fq",
"type": "go",
"request": "launch",
"mode": "auto",
"showLog": true,
"program": ".",
"cwd": "${workspaceFolder}",
"env": {},
"args": ["-d", "mp4", ".", "file"]
}
]
}

5
.vscode/settings.json vendored
View File

@ -61,6 +61,7 @@
"gojqex",
"golangci",
"gopacket",
"gopanic",
"GOPATH",
"gosec",
"gosimple",
@ -154,6 +155,8 @@
"tfhd",
"tfra",
"tmpl",
"to_xml",
"to_xmlentities",
"toactual",
"toarray",
"toboolean",
@ -167,8 +170,6 @@
"torepr",
"tosym",
"tovalue",
"toxml",
"toxmlentities",
"traf",
"trak",
"trex",

4
Bumpfile
View File

@ -2,10 +2,10 @@ Dockerfile
Makefile
go.mod
github-golangci-lint /GOLANGCILINT_VERSION: ([\d.]+)/ git:https://github.com/golangci/golangci-lint.git|^1
github-golangci-lint /GOLANGCILINT_VERSION: "([\d.]+)"/ git:https://github.com/golangci/golangci-lint.git|^1
github-golangci-lint link "Release notes" https://github.com/golangci/golangci-lint/releases/tag/v$LATEST
github-go-version /go-version: ([\d.]+)/ git:https://github.com/actions/go-versions.git|/(.*)-.*/$1/|^1
github-go-version /go-version: "([\d.]+)"/ git:https://github.com/actions/go-versions.git|/(.*)-.*/$1/|^1
.github/workflows/ci.yml
.github/workflows/release.yml

555
CHANGES.md
View File

@ -1,3 +1,558 @@
# 0.5.0
Mostly a bug fix release but adds `-V` for easy JSON output.
## Changes
- Add `-V` argument to default output JSON instead of decode tree in case of decode value. #385 Thanks @peterwaller-arm for reminding me to merge this.
```sh
# default in case of decode value is to show a hexdump tree
$ fq '.headers | grep_by(.id=="TSSE").text' file.mp3
│00 01 02 03 04 05 06 07 08 09 0a 0b│0123456789ab│
0x0c│ 4c 61 76│ Lav│.headers[0].frames[0].text: "Lavf58.76.100"
0x18│66 35 38 2e 37 36 2e 31 30 30 00 │f58.76.100. │
# with -V an implicit "tovalue" is done
$ fq -V '.headers | grep_by(.id=="TSSE").text' file.mp3
"Lavf58.76.100"
# and in combination with -r will for strings output a "raw string" without quotes
# for other types like number, object, array etc -r makes not difference (same as jq)
$ fq -Vr '.headers | grep_by(.id=="TSSE").text' file.mp3
Lavf58.76.100
```
As a side note `-V` can be used with binary type also. Then the binary data will be interpreted as UTF-8 and turned into a string.
```sh
# trailing null terminator ends up as codepoint zero `\u0000`
$ fq -V '.headers | grep_by(.id=="TSSE").text | tobytes' file.mp3
"Lavf58.76.100\u0000"
# with -r null terminator and a new line is outputted
$ fq -Vr '.headers | grep_by(.id=="TSSE").text | tobytes' file.mp3 | hexdump -C
00000000 4c 61 76 66 35 38 2e 37 36 2e 31 30 30 00 0a |Lavf58.76.100..|
0000000f
# in contrast raw binary output has no new line separator
$ fq '.headers | grep_by(.id=="TSSE").text | tobytes' doc/file.mp3 | hexdump -C
00000000 4c 61 76 66 35 38 2e 37 36 2e 31 30 30 00 |Lavf58.76.100.|
0000000e
```
- Fix issue using decode value in object passed as argument to internal function. #638
```sh
# this used to fail but now works
fq '.tracks[0].samples[10] | avc_au({length_size: <decode value>})' file.mp4
```
- Some typo fixes. Thanks @retokromer and @peterwaller-arm
## Decoder changes
- `aiff` Basic AIFF decoder added. #614
- `matroska` Update to latest specification. #640
- `msgpack` Fix bug decoding some fixstr lengths. #636 Thanks @schmee for reporting.
## Changelog
* 4ad1cced Update docker-golang to 1.20.3 from 1.20.2
* f7dca477 Update github-go-version to 1.20.3 from 1.20.2
* c9608939 Update github-golangci-lint to 1.52.0 from 1.51.2
* 0a6b46c8 Update github-golangci-lint to 1.52.1 from 1.52.0
* c4eb67d9 Update github-golangci-lint to 1.52.2 from 1.52.1
* 19140a6f Update gomod-creasty-defaults to 1.7.0 from 1.6.0
* 6e5df724 Update gomod-golang-x-crypto to 0.8.0 from 0.7.0
* 6c4aebfe Update gomod-golang-x-net to 0.9.0 from 0.8.0
* f13cc979 Update gomod-golang/text to 0.9.0 from 0.8.0
* e2af57ee Update gomod-gopacket to 1.1.0 from 1.0.0
* a63fd684 Update make-golangci-lint to 1.52.0 from 1.51.2
* d3d1f0e8 Update make-golangci-lint to 1.52.1 from 1.52.0
* f0b08457 Update make-golangci-lint to 1.52.2 from 1.52.1
* dc4a82ee aiff: Add basic decoder
* c5f6809b decode,fuzz,dev: Move recoverable error check to recoverfn.Run
* 980ecdba decode: Add float 80 reader
* a6c4db75 decode: Cleanup old unused help system code
* 87e5bb14 fix typo
* 0b6ef2a9 golangci-lint: Disable revive unused-parameter and update for new default config
* 427ce78d interp: Add --value-output/-V option to do tovalue before output
* 9a1ef84c interp: Allow and convert JQValues:s (ex decode value) in function arg objects
* 3dd2c61d interp: Fix input completion regression in sub-REPLs
* 5415bfca interp: Make completion work again
* 2a2b64dd matroska: Update ebml specification
* 82da99c9 msgpack: Add str, array and object type tests
* 97360d6f msgpack: fixstr length field is 5 bits
* ffc66db0 readline: remove direct access to (*Instance).Config
* e1b02312 wav: Cleanup avi leftovers
# 0.4.0
TLS decode and decryption, better streaming matroska/webm support, support raw IP in PCAP and bug fixes.
## Changes
- Fix panic when interrupting big JSON output. #573
- Support passing options (`-o name=value`) to nested decoders. #589
- Allows for example to pass keylog to a TLS decoder inside a PCAP file or to tell a container decoders to not decode samples inside a ZIP file etc.
- Exit with error if `-o name=@path` fails to read file at `path`. #597
## Decoder changes
- `id3v2` Properly decode CTOC subframes. #606
- `matroska`
- Now supports streaming matroska and webm better (master elements with unknown size). #576 #581
- Add `decode_samples` option. #574
- Spec update and clean up of symbols and descriptions. #580
- `pcap,pcapng` Support raw IPv4 and IPv6 link frames. #599 #590
- `tls` Add Transport layer security decoder and decryption. #603
- Supports TLS 1.0, 1.1, 1.2 and some SSL 3.0.
- Decodes records and most messages and extensions.
- Can decrypt most common cipher suites if a keylog is provided. See documentation for list of supported ciphers suites.
```sh
# show first 50 bytes of decrypted client/server TLS application data stream
# -o keylog=@file.pcap.keylog is used to read keylog from a file
# first .stream is TCP stream, second .stream the application data stream
$ fq -o keylog=@file.pcap.keylog '.tcp_connections[0].["client", "server"].stream.stream | tobytes[0:50] | dd' file.pcap
│00 01 02 03 04 05 06 07 08 09 0a 0b│0123456789ab│
0x00│47 45 54 20 2f 64 75 6d 70 2f 6c 6f│GET /dump/lo│.: raw bits 0x0-0x31.7 (50)
0x0c│67 20 48 54 54 50 2f 31 2e 31 0d 0a│g HTTP/1.1..│
0x18│48 6f 73 74 3a 20 69 6e 77 61 64 65│Host: inwade│
0x24│72 2e 63 6f 6d 0d 0a 55 73 65 72 2d│r.com..User-│
0x30│41 67 │Ag │
│00 01 02 03 04 05 06 07 08 09 0a 0b│0123456789ab│
0x00│48 54 54 50 2f 31 2e 31 20 32 30 30│HTTP/1.1 200│.: raw bits 0x0-0x31.7 (50)
0x0c│20 4f 4b 0d 0a 41 63 63 65 70 74 2d│ OK..Accept-│
0x18│52 61 6e 67 65 73 3a 20 62 79 74 65│Ranges: byte│
0x24│73 0d 0a 43 6f 6e 74 65 6e 74 2d 4c│s..Content-L│
0x30│65 6e │en │
# show first TLS record from server
$ fq '.tcp_connections[0].server.stream.records[0] | d' file.pcap
│00 01 02 03 04 05 06 07 08 09 0a 0b│0123456789ab│.tcp_connections[1].server.stream.records[0]{}: record
0x00│16 │. │ type: "handshake" (22) (valid)
0x00│ 03 03 │ .. │ version: "tls1.2" (0x303) (valid)
0x00│ 00 40 │ .@ │ length: 64
│ │ │ message{}:
0x00│ 02 │ . │ type: "server_hello" (2)
0x00│ 00 00 3c │ ..< │ length: 60
0x00│ 03 03 │ .. │ version: "tls1.2" (0x303)
│ │ │ random{}:
0x00│ 86│ .│ gmt_unix_time: 2249760024 (2041-04-16T21:20:24Z)
0x0c│18 9d 18 │... │
0x0c│ 19 92 33 c2 21 ce 4f 97 30│ ..3.!.O.0│ random_bytes: raw bits
0x18│28 98 b3 fd 1e 15 f4 36 bb e9 14 f4│(......6....│
0x24│67 61 66 79 d5 3f 06 │gafy.?. │
0x24│ 00 │ . │ session_id_length: 0
│ │ │ session_id: raw bits
0x24│ c0 2f │ ./ │ cipher_suit: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" (0xc02f)
0x24│ 00 │ . │ compression_method: "null" (0x0)
0x24│ 00│ .│ extensions_length: 20
0x30│14 │. │
│ │ │ extensions[0:2]:
│ │ │ [0]{}: extension
0x30│ ff 01 │ .. │ type: "renegotiation_info" (65281)
0x30│ 00 01 │ .. │ length: 1
0x30│ 00 │ . │ data: raw bits
│ │ │ [1]{}: extension
0x30│ 00 10 │ .. │ type: "application_layer_protocol_negotiation" (16)
0x30│ 00 0b │ .. │ length: 11
0x30│ 00 09│ ..│ serer_names_length: 9
│ │ │ protocols[0:1]:
│ │ │ [0]{}: protocol
0x3c│08 │. │ length: 8
0x3c│ 68 74 74 70 2f 31 2e 31 │ http/1.1 │ name: "http/1.1"
# use ja3.jq to calculate ja3 TLS fingerprint
# https://github.com/wader/fq/blob/master/format/tls/testdata/ja3.jq
$ fq -L path/to/ja3 'include "ja3"; pcap_ja3' file.pcap
[
{
"client_ip": "192.168.1.193",
"client_port": 64126,
"ja3": "771,4866-4867-4865-49196-49200-159-52393-52392-52394-49195-49199-158-49188-49192-107-49187-49191-103-49162-49172-57-49161-49171-51-157-156-61-60-53-47-255,0-11-10-16-22-23-49-13-43-45-51-21,29-23-30-25-24,0-1-2",
"ja3_digest": "bc29aa426fc99c0be1b9be941869f88a",
"server_ip": "46.101.135.150",
"server_port": 443
}
]
```
- `toml` Fail faster to speed up probe. Could in some cases read the whole file before failing. Thanks @0-wiz-0 for report. #594
- `zip` Properly decode EOCD record in zip64 files. Thanks @0-wiz-0 for report and spec interpretation. #586 #596
- `xml` Fail faster to speed up probe. Could in some cases read the whole file before failing. Thanks @0-wiz-0 for report. #594
## Changelog
* 0581ecea Update docker-golang to 1.20.1 from 1.20.0
* 72870a5a Update docker-golang to 1.20.2 from 1.20.1
* 02e573a9 Update github-go-version to 1.20.1 from 1.20.0, 1.20.0, 1.20.0
* c5130887 Update github-go-version to 1.20.2 from 1.20.1
* ce263726 Update github-golangci-lint to 1.51.1 from 1.51.0
* 75bfdda3 Update github-golangci-lint to 1.51.2 from 1.51.1
* b1d9306b Update gomod-golang-x-crypto to 0.6.0 from 0.5.0
* c03d3ccd Update gomod-golang-x-crypto to 0.7.0 from 0.6.0
* 2430fba7 Update gomod-golang-x-net to 0.6.0 from 0.5.0
* dd8ab799 Update gomod-golang-x-net to 0.7.0 from 0.6.0
* 80a07446 Update gomod-golang-x-net to 0.8.0 from 0.7.0
* 97643b98 Update gomod-golang/text to 0.7.0 from 0.6.0
* e7168b99 Update gomod-golang/text to 0.8.0 from 0.7.0
* 36df57eb Update make-golangci-lint to 1.51.1 from 1.51.0
* 70e08faa Update make-golangci-lint to 1.51.2 from 1.51.1
* 50d26ec7 colorjson: Handle encoding error value
* 5c8e1151 colorjson: Refactor to option struct
* 8e0dde03 decode: Support multiple format args and some rename and refactor
* a1bb630a doc,fq: Improve cli help and some cleanup
* 156aeeca doc: Add FOSDEM 2023 talk
* 3e0ebafa doc: Run make doc
* 3cc83837 gojq: Update fq fork
* dec433fc help,markdown: Fix double line breaks when converting to text
* c75a83c8 help: Show default option value as JSON
* cc52a441 id3v2: Decode subframes for CTOC and add struct for headers
* dc79a73b interp,json: Move error handling to colorjson
* 73db6587 interp: Exit with error if -o name=@path fails to be read, also document
* c8666eeb ipv4_packet,ipv6_packet,sll_packet,sll2_packet: Support ipv4/ipv6 link frames and pass correct in arg
* b60aceca matroska: Add decode_samples option
* 9aaf2ddf matroska: Add unknown size test and add description to ebml header
* a8d0bf4d matroska: Assume master with unknown size has ended if a valid parent is found
* 0d14d7b4 matroska: Handle unknown size for non-master types a bit better
* c890a289 matroska: Update spec and make refs in descriptions look nicer
* 6c032455 pcap,pcapng,ipv4,ipv6: Support raw link type (ipv4 or ipv6)
* d4ea6632 pcap: Add ipv4 fragments tcp test
* f50bd6ee readline: Update fq fork
* 9852f56b tls: Add TLS 1.0, 1.1, 1.2 decode and decryption
* 56edb59e toml,xml: Fail fast on invalid content
* 5228fdd6 zip: Correctly look for and decode both zip32/64 EOCD record
* bdd6718d zip: Correctly peek for zip64 EOCD
# 0.3.0
Bug fix release, no new features mostly due to holidays and busy with other things (some jq related!).
Also been preparing for a [talk about fq](https://fosdem.org/2023/schedule/event/bintools_fq/) this weekend at [FOSDEM 2023](https://fosdem.org/2023/).
## Changes
* TCP reassembly is now less strict about invalid TCP options. Turns out some options might end up wrong in packet captures due to hardware acceleration etc. For example it seems to be common that TCP segments end up larger than configured connection MSS. Now PCAP:s with those kinds of TCP segments should be reassembled correctly.
* REPL now handles the del key properly. Before it could in some cases cause the output to be ignored.
## Decoder changes
- `mp3` Add option for max unknown bits to handle more mis-probing. Default to 50%
- `mp4`
- `ftyp` set minor description to date for "qt" files
- `tkhd` decode enabled, preview, etc flags
- `udta` Handle case wit box type is key and value rest of box
- `sgpd`,`sbgp` Change grouping type to a string as it seems to be what it is in practice.
- `tcp_segment` Decode all standard options, MSS, Window scale, timestamp etc. Rename "maxseg" to "mss".
## Changelog
* 8702e1d1 Update docker-golang to 1.19.5 from 1.19.4
* a7f37d73 Update docker-golang to 1.20.0 from 1.19.5
* 826d9a52 Update github-go-version to 1.19.5 from 1.19.4, 1.19.4, 1.19.4
* d338c8b7 Update github-go-version to 1.20.0 from 1.19.5, 1.19.5, 1.19.5
* ad4919a8 Update github-golangci-lint to 1.51.0 from 1.50.1
* e8ecbf95 Update gomod-golang/text to 0.6.0 from 0.5.0
* f1057b9b Update make-golangci-lint to 1.51.0 from 1.50.1
* ca27e426 doc: Add _parent for decode values and clenaup doc a bit
* b04a650b flac_picture,mpeg: Fix trailing ")" typo in map sym and description
* 57144b2f github-action: Use quotes becase yaml (1.20 -> 1.2)
* 0aa6e3e2 gojq: Update rebased fq fork
* 7855b359 gomod: Update non-bump tracked mods and add bump config
* 6e17de36 goreleaser: Use name_template instead of deprecated archive replacements
* 8b49b42f interp: Wrap Binary in decodeValue to fix prompt issue with bits/bytes format
* 2d82c05f mp3: Add max_unknown option to fail decode if too much unknown bits
* f386a515 mp4: Decode qt minor verison as YYYY.MM description
* 3555dc67 mp4: Decode tkhd flags
* c3e3b3e9 mp4: Decode udta metadata boxes without meta box
* c49012db mp4: sgpd,sbgp: Change grouping_type to a string
* 63403658 mp4: udta: Handle box with value rest of box
* 55ef7a4b readline: Update fq fork to fix draw issue when using del key
* 1eb5e502 tcp: Ignore TCP option check for now as it seems unreliable in dumps
* 62e2cef5 tcp_segment: Decode standard options and rename maxseg to mss
# 0.2.0
This ended up being a release to cleanup old sins in the decoder internals and change some defaults how binary values work with JSON and string functions.
It also adds a new Time Zone Information Format decoder `tzif` (Thanks Takashi Oguma @bitbears-dev) and a new Apple BookmarkData decoder `apple_bookmark` decoder (Thanks David McDonald @dgmcdona). Also a new function `from_ns_keyed_archiver` was added to convert NSKeyedArchiver encoded objects into JSON.
A possible breaking change is that now all `from`/`to` prefix functions now has a `from_`/`to_` prefix, ex: `from_mp3` instead of `frommp3`. There are some few exceptions to this. Note that the functions named just be the format name, ex `mp3` are still around.
In other fq related news [jq-lsp](https://github.com/wader/jq-lsp) got some fixed and additions and seems to work fine with neovim. It's also possible to use jq-lsp with vscode using [vscode-jq](https://github.com/wader/vscode-jq).
## Changes
- All functions that had a `from`/`to` prefix now has the prefix `from_`/`to_`. This is to be easier to read and more consistent, there are still some exceptions like `tovalue`, `torepr`, `tobytes` etc but in general anything that does not deal with primitive types is now `snake_case`. #535
- Change default `bit_formats` option value (how raw bits values are represented in JSON) from `snippet` to `string`. `snippet` meant truncated bits as base64. Now all bits are included as a UTF-8 string. The string will be binary safe (not lose any data) when used internally in fq but will lose data when represented in JSON as some bytes can't be encoded as UTF-8. #499
- Don't auto convert to binary for string/regexp functions, turned out this is very confusing. Now you have to manually use `tobytes` etc to convert to binary value. #540
```sh
# This used to not work as test/1 would convert decode values to the source bytes
# (0x00 0x00 0x00 0x01) in this case. Now the jq value (symbolic in this case) will
# be used instead. You can do ".test | tobytes" to get old behavior.
#
# find all types with a "mdta." prefix
$ fq -o line_bytes=10 'grep_by(.type | test(`^mdta\.`))' file.mp4
│00 01 02 03 04 05 06 07 08 09│0123456789│.boxes[3].boxes[2].boxes[0].boxes[2].boxes[0]{}: box
0x528│ 00 00 00 1c │ .... │ size: 28
0x528│ 00 00 00 01│ ....│ type: "mdta.title" ("\x00\x00\x00\x01")
0x532│00 00 00 14 64 61 74 61 00 00│....data..│ boxes[0:1]:
0x53c│00 01 00 00 00 00 74 65 73 74│......test│
│00 01 02 03 04 05 06 07 08 09│0123456789│.boxes[3].boxes[2].boxes[0].boxes[2].boxes[1]{}: box
0x546│00 00 00 25 │...% │ size: 37
0x546│ 00 00 00 02 │ .... │ type: "mdta.encoder" ("\x00\x00\x00\x02")
0x546│ 00 00│ ..│ boxes[0:1]:
0x550│00 1d 64 61 74 61 00 00 00 01│..data....│
0x55a│00 00 00 00 4c 61 76 66 35 39│....Lavf59│
0x564│2e 32 37 2e 31 30 30│ │.27.100│ │
```
- Fix panic when cancel (ctrl-c etc) before interpreter is executing. Thanks @pldin601 for reporting. #495
- Fix error using JQValue:s in assign/update paths, ex `.[<JQValue here>] = 123` #509
- Rename fields added for bit-ranges not used by a decoder from `unknown#` to `gap#`. "unknown" is probably a useful field name in some formats and "gap" describe better what it is. #500
- Big decode API internals refactor to split scalars types into their own go types so they can store per type specific values. This also opens up for more ways to make fq both faster and more memory efficient. It also makes the decode API more type safe and makes it possible to experiment with decode DLS that uses chained methods etc. #523
## Decoder changes
- `apple_bookmark` New Apple BookmarkData decoder. Thanks David McDonald @dgmcdona. #493
- `bplist`
- Fix decoding of UID types
- Adds a `lost_and_found` array with unused values
- Fix an endian issue for unicode strings
- Add NSKeyedArchiver to JSON helper function `from_ns_keyed_archiver`, see `bplist` docs for details on how to use it. Thanks David McDonald @dgmcdona. #493
```
# decode bplist, from_ns_keyed_archiver converts to JSON plist and then into object data as JSON, find app bookmarks keys and expand them as bookmark data and convert to represented JSON, and finally build path to applications
$ fq -r 'from_ns_keyed_archiver | (.. | .Bookmark? // empty) |= (apple_bookmark | torepr) | .. | .target_path? // empty | join("/")' recentapps.sfl2
System/Applications/Utilities/Terminal.app
Applications/Spotify.app
System/Applications/Calculator.app
System/Applications/Preview.app
Applications/Alacritty.app
Applications/DB Browser for SQLite.app
System/Applications/System Preferences.app
System/Library/CoreServices/Applications/Directory Utility.app
System/Applications/Utilities/Activity Monitor.app
Applications/Safari.app
```
- `tzif` new Time Zone Information Format decoder. Thanks Takashi Oguma @bitbears-dev. #498
- `mp4`
- Map `mdta` metadata namespace and key names for `ilst` child boxes. #521
```sh
$ fq 'grep_by(.type=="ilst").boxes | map({key: .type, value: .boxes[0].data}) | from_entries' file.mp4
# create object with all ilst key/value pairs
{
"mdta.encoder": "Lavf59.27.100",
"mdta.title": "test"
}
# query specific value
$ fq -r 'grep_by(.type=="mdta.encoder").boxes[0].data | tovalue' file.mp4
Lavf59.27.100
```
- Support `sidx` version 1. #506
- Add description and symbolic values for traf sample flags, makes it easier to see and query for I-frames etc. #514
```
# which boxes has depends_on flags
$ fq 'grep_by(.sample_depends_on) | parent.type' fragmented.mp4
```
- Support PNG codec mapping. #492
- Decode `pdin` boxes. #524
- Decode `hnti` boxes. #513
- `mp3_tags` Add VBRI support and split into into `mp3_frame_xing` and `mp3_frame_vbri` decoders. #525
## Changelog
* 7fa8b635 Add related file format projects to README
* 4fdb7362 Update docker-golang to 1.19.4 from 1.19.3
* 519eff6c Update github-go-version to 1.19.4 from 1.19.3, 1.19.3, 1.19.3
* 2a91d293 Update gomod-golang/text to 0.5.0 from 0.4.0
* cb15b371 added checks to prevent infinite looping and recursion
* c2445335 added some sfl2 test files to bplist package
* 7d13cf73 adds flag parsing to applebookmark
* 71b17d03 apple bookmarkdata decoder initial commit
* 8f39ef63 bplist: Harmonize ns_keyed_archive jq style a bit
* cba72dbd bplist: added overload for from_ns_keyed_archiver jq func
* 129b4b70 bplist: doc: update docs to reflect changes to ns_keyed_archiver
* 9dab3c60 bplist: minor fix to from_ns_keyed_archiver
* 448c3efb bplist: update docs with from_ns_keyed_archiver reference, add error case to function
* a9047c02 bplist: updates from_ns_keyed_archiver to do automatic torepr based on format detection
* 4a28e44f changes decoder package name from bookmark to apple_bookmark
* d0b044c2 converts to snake_case and refactors decode helper
* d199793a created stack type
* e77f7769 decode,interp: Rename unknown gap fields from "unknown#" to "gap#"
* a85da295 decode: Make FieldFormat usage more consistent
* 9b81d4d3 decode: More type safe API and split scalar into multiple types
* 3ec0ba3f decode: add ns_keyed_archiver, restructure apple decoder into apple package
* 330d5f7f decode: apple_bookmark: simplifies flag decoding
* 93f2aa5d decode: change PosLoopDetector to use generics
* 7e98b538 decode: fix type on defer function call, test: add loop.fqtest
* a873819e decode: fixes endian of unicode strings
* f747873d decode: implements lost and found for unreferenced objects
* b45f9fa6 decode: improve stack push/pop
* a162e07b decode: minor change to method receiver name
* 3232f9cc decode: moves PosLoopDetector into its own package
* 7c9504c7 decode: moves macho decoder to apple package
* 70834678 decode: remove dead code from ns_keyed_archiver
* 7ab44662 decode: remove unused field from decoder, unused parens from torepr
* bdb81662 decode: removed unnecessary struct
* 98eab8cb decode: rename parameter for consistency
* 04379df8 decode: revert decode.D back, place posLoopDetector in apple_bookmark
* 7fb674b5 decode: unexport methods
* fa368bb7 decode: updates all.go with correct macho path
* 0287ffa4 decoding well but torepr needs work
* 42debe58 dev,doc,make: Cleanup makefile and have proper targets for *.md and *.svg
* 423bab9e dev,test: Use jqtest code from jqjq for jq tests
* 6fc84a88 doc,dev: Add more usage and dev tips
* 2fc16ae2 doc: Add some padding margin to formats table to make it less likely to cause git conflicts
* 62f377c2 doc: fixes snippet for recursive bookmark searching
* 22064f50 doc: remake
* 4aad2fde doc: remake
* b872b1a3 doc: remake
* 1e1fc551 fixed one more snake_case letter
* d0b76cae fixes broken test and removes long link from markdown body
* 5146f28d fixes broken test for all.fqtest
* 253033cc fixes broken uid parsing in plist decoder
* f535ad3d fixes spacing in jq files
* 64351e8b fixes tests and adds torepr test
* c7d00b87 fixes unknown bit ranges
* 8f930aac forgot to add bookmark.jq in last commit
* 164e527b gojq: Update rebased fq fork
* 6c869451 gojq: Update rebased fq fork
* 578b84d4 interp,display: Add workaround for go 1.18 when escaping 0x7f
* 42d9f2c2 interp,help: Properly count line length when breaking on whole words
* 8d69f1fb interp: Change default bits_format=string
* 6c229d73 interp: Don't auto convert to binary for string functions, is just confusing
* 568afff3 interp: Fix panic when trigger before any context has been pushed
* e3ae1440 interp: Rename to/from<format> functions to to_/from_<format>
* ba88a684 interp: mimic jq: if expr arg is given read stdin even if tty
* 9bd65f93 migrates tests to per-sample files
* f7d7a49f missed a letter on last commit - converting to snake_case
* 2f37cb55 mod: Update modules not tracked with bump
* 55f4f1aa moved a flag bit fields into correct positions
* 9e5a072e mp3_frame_tags: Covert to decode group and split to mp3_frame_{xing,vbri} decoders
* 48522e3c mp3_tags,mp3: Add VBRI header support and rename tags to tag as there is only one
* 83ccedc5 mp4,decode: Properly decode ilst items (both mdta and mdir)
* 1dea40e6 mp4,doc: Add JSON box tree example and reorder a bit
* b1b3b63d mp4: Add namespace to mdta ilst boxes
* 7b60b24a mp4: Add pdin box support
* ef2d5232 mp4: Add png mapping
* 5fb81a14 mp4: Add sym and description for traf sample flags
* 1d6ce2c0 mp4: Decode hint and hnti child boxes
* 9ac453a1 mp4: Fix typo in sample flags sample_is_depended_on description
* a23fe618 mp4: sidx version 1 segment_duration is s64
* 3942db79 pkg/decode/D: Adds PushAndPop, Push, Pop methods. doc: adds help_applebookmark.fqtestdecode: converts applebookmark to use new d.PushAndPop method
* 0c216dff refactors some decoder logic in apple_bookmark for better querying
* 34db9d7f regenerated docs, added tests, fixed torepr
* 0a72635a remade documentation
* 1352598a removed commented out line
* 81269430 removed unnecessary conversions
* 5b1455e7 removed unused function
* 63a3ca20 removes underscore from apple_bookmark package name
* a351c346 removes unused function
* 2ee6360b support tzif (time zone information format)
* 8d5dcff8 test: applebookmark: adds problematic test case
* 63a4e80c test: fixed doc test
* 47a568e0 text,test: Unbeak base64 tests
* 44c91d82 tweaks apple_bookmark markdown documentation
* fd22426b tzif: add help_tzif.fqtest
* c4e7fc79 tzif: moved document to tzif.md
* abde823a tzif: use PeekFindByte() to find end of the string
* 4481a77a tzif: use scalar.Fn() to define a mapper ad hoc
* dbc6fccd updated doc with apple reference
* f5e25fca updated docs
* 6f4d1cb1 updated documentation
* b2aeac6a updates bplist fq tests
* a23ac8f5 updates fqtest for torepr in apple_bookmarkdata
# 0.1.0
Adds `avi` decoder and replace `raw` with more convenient `bits` and `bytes` format. Otherwise mostly small updates and bug fixes.
Increase minor version. fq does not use sementic versioning (yet) but it's probably a good idea to start increase minor version when adding features to be able to do patch releases.
In other fq related news:
- I gave a [talk about fq](https://www.youtube.com/watch?v=-Pwt5KL-xRs&t=1450s) at [No Time To Wait 6](https://mediaarea.net/NoTimeToWait6) a conference about open media, standardization, and audiovisual preservation.
- While prototyping writing decoders directly in jq for fq I ended up [implementing jq in jq](https://github.com/wader/jqjq). Still thinking and working on how to do decoders in jq.
## Changes
- Replace `raw` format with `bits` and `bytes` format that decodes directly to a binary with different unit size.
```sh
$ echo -n 'hello' | fq -d bytes '.[-3:]' > last_3_bytes
$ echo 'hello' | fq -d bytes '.[1]'
101
$ echo 'hello' | fq -c -d bits '[.[range(8)]]'
[0,1,1,0,1,0,0,0]
```
## Decoder changes
- `avc_au` Support annexb format (used in AVI). #476
- `avi` Add AVI (Audio Video Interleaved) decoder. #476
```sh
# extract samples for stream 1
$ fq '.streams[1].samples[] | tobytes' file.avi > stream01.mp3
```
- `bits` Replaces `raw` but is a binary using bit units. #485
- `bytes` Replaces `raw` but is a binary using byte units. #485
- `bplist`
- Fix signed interger decoding. #451 @dgmcdona
- Use correct size for references and check for infinite loops. #454 @dgmcdona
- `flac_frame` Correctly decode zero escape sample size. #461
- `id3v2` Fix decoding of COMM and TXXX with missing null terminator. #468
- `matroska` Updated to latest specification. #455
- `mp3_frame` Use frame size calculation from spec instead of own as it seems to not work in some cases. #480
- `mp3_frame_tags` Replaces `xing` and also decodes "lame extensions" for both Xing and Info. #481
- `raw` Removed. #485
- `wav` More codec symbol names and now shares RIFF code with AVI decoder. #476
- `yaml` Fix type panic for large intergers. #462
## Changelog
* 7b6492ee Improve README.md a bit, one more demo and move up usage
* 4e069625 Update docker-golang to 1.19.2 from 1.19.1
* e0334497 Update docker-golang to 1.19.3 from 1.19.2
* f3f2648b Update github-go-version to 1.19.2 from 1.19.1, 1.19.1, 1.19.1
* 003197eb Update github-go-version to 1.19.3 from 1.19.2, 1.19.2, 1.19.2
* 453963dd Update github-golangci-lint to 1.50.1 from 1.50.0
* 56dcb3a0 Update gomod-BurntSushi/toml to 1.2.1 from 1.2.0
* 101b2806 Update gomod-golang/text to 0.3.8 from 0.3.7
* d80f12c7 Update gomod-golang/text to 0.4.0 from 0.3.8
* 753927ba Update make-golangci-lint to 1.50.1 from 1.50.0
* 4d8dd5c5 adds check for recursion in decodeReference, adds test to verify fix
* b7c4576c adds necessary cast
* 46b7ab32 adds test to verify fix
* 4ee7dd8a changes Errorf to Fatalf on infinite loops
* 41b2d1ad cli: Better decode error help
* 7254b0f9 decode,elf,fuzz: TryBytesRange error on negative size
* bafd1f56 decode,fuzz: Signed integer (S) read require at least one bit
* 2a86d323 doc,rtmp,pcap,markdown: Add more examples
* 0b9b0173 doc: Add gomarkdown to license/dependencies
* 4bfd9d81 doc: Add link to nttw6 presentation video and slides
* fb1a91ac drop indented else block per lint
* 4dd594c1 fixes bad path in test output
* f9a1d3f4 fixes calculation of floating point lengths
* 236fbc17 fixes reference calculation to use reference size from trailer
* ac86f931 fixes signed integer parsing
* fb2a2b94 flac,fuzz: Fatal error on negative partition sample count
* 7859be1e flac_frame: Properly decode zero escape sample size
* 7cb2a6c9 fuzz: gotip not needed anymore
* cef4245b fuzz: make fuzz GROUP=mp4 to fuzz one group
* 413d4250 gofmt
* 349d9497 gojq: Update rebased fq fork
* 450f5844 gojq: Update rebased fq fork
* d8641ab1 gomod: Update modules that lack bump config
* f66e2244 id3v2: In the wild COMM and TXXX frame might not have a null terminator
* b09d6116 makes dictionary key type checking more sensible
* d07b2eec markdown,fuzz: Update gomarkdown
* 646f32d5 matroska: Fix path tests and make _tree_path more robust
* e748079e matroska: Update spec and regenerate
* 1c7d3252 mod: Update ones without bump config
* 2de87539 mp3_frame: Fix issue calc frame size for some configs
* c3a0686c mp3_frame_tags: Refactor and rename xing format to mp3_frame_tags
* d75748d8 mp4: Decode more sample flags
* c93301fc raw,bits,bytes: Replace raw format with bits and bytes format that decode to a binary
* b08e25ce removes unnecessary cast
* 2b3adbe8 renames test data file
* 0cf46e11 wav,avi,avc_au: Add avi decoder and refactor wav decoder
* 26069167 yaml,fuzz: gojq.Normalize value to fix type panic
# 0.0.10
## Changes

2
Dockerfile
View File

@ -1,5 +1,5 @@
# bump: docker-golang /FROM golang:([\d.]+)/ docker:golang|^1
FROM golang:1.19.3-bullseye AS base
FROM golang:1.20.3-bullseye AS base
# expect is used to test cli
RUN \

85
Makefile
View File

@ -3,57 +3,51 @@ GO_BUILD_LDFLAGS ?= -s -w
all: test fq
.PHONY: fq
fq:
# used to force make to always redo
.PHONY: always
fq: always
CGO_ENABLED=0 go build -o fq -ldflags "${GO_BUILD_LDFLAGS}" ${GO_BUILD_FLAGS} .
.PHONY: test
test: testgo testjq testcli
test: always testgo testjq testcli
test-race: always testgo-race testjq testcli
.PHONY: test-race
test-race: testgo-race testjq testcli
.PHONY: testgo
# figure out all go pakges with test files
# figure out all go packages with test files
testgo: PKGS=$(shell find . -name "*_test.go" | xargs -n 1 dirname | sort | uniq)
testgo:
testgo: always
go test -timeout 20m ${RACE} ${VERBOSE} ${COVER} ${PKGS}
.PHONY: testgo-race
testgo-race: RACE=-race
testgo-race: testgo
.PHONY: testjq
testjq: fq
@pkg/interp/testjq.sh ./fq pkg/interp/*_test.jq
testjq: $(shell find . -name "*.jq.test")
%.jq.test: fq
@echo $@
@./fq -rRs -L pkg/interp 'include "jqtest"; run_tests' $@
.PHONY: testcli
testcli: fq
@pkg/cli/test_exp.sh ./fq pkg/cli/test_repl.exp
@pkg/cli/test_exp.sh ./fq pkg/cli/test_cli_ctrlc.exp
@pkg/cli/test_exp.sh ./fq pkg/cli/test_cli_ctrld.exp
.PHONY: cover
cover: COVER=-cover -coverpkg=./... -coverprofile=cover.out
cover: test
go tool cover -html=cover.out -o cover.out.html
cat cover.out.html | grep '<option value="file' | sed -E 's/.*>(.*) \((.*)%\)<.*/\2 \1/' | sort -rn
.PHONY: doc
doc: doc/formats.svg doc/demo.svg
doc: doc/display_json.svg
doc: doc/display_decode_value.svg
doc: doc/display_decode_value_d.svg
doc: doc/display_decode_value_dv.svg
@doc/mdsh.sh ./fq *.md doc/*.md
doc: always
doc: $(wildcard doc/*.svg)
doc: $(wildcard *.md doc/*.md)
doc/%.svg: doc/%.sh fq
(cd doc ; ../$< ../fq) | go run github.com/wader/ansisvg@master > $@
%.md: fq
@doc/mdsh.sh ./fq $@
doc/%.svg: fq
(cd doc ; ../$@.sh ../fq) | go run github.com/wader/ansisvg@master > $@
.PHONY: doc/formats.svg
doc/formats.svg: fq
# ignore graphviz version as it causes diff when nothing has changed
./fq -rnf doc/formats_diagram.jq | dot -Tsvg | sed 's/Generated by graphviz.*//' > doc/formats.svg
@# ignore graphviz version as it causes diff when nothing has changed
./fq -rnf doc/formats_diagram.jq | dot -Tsvg | sed 's/Generated by graphviz.*//' >doc/formats.svg
doc/file.mp3: Makefile
ffmpeg -y -f lavfi -i sine -f lavfi -i testsrc -map 0:0 -map 1:0 -t 20ms "$@"
@ -61,36 +55,28 @@ doc/file.mp3: Makefile
doc/file.mp4: Makefile
ffmpeg -y -f lavfi -i sine -f lavfi -i testsrc -c:a aac -c:v h264 -f mp4 -t 20ms "$@"
.PHONY: gogenerate
gogenerate:
gogenerate: always
go generate -x ./...
.PHONY: lint
lint:
lint: always
# bump: make-golangci-lint /golangci-lint@v([\d.]+)/ git:https://github.com/golangci/golangci-lint.git|^1
# bump: make-golangci-lint link "Release notes" https://github.com/golangci/golangci-lint/releases/tag/v$LATEST
go run github.com/golangci/golangci-lint/cmd/golangci-lint@v1.50.1 run
go run github.com/golangci/golangci-lint/cmd/golangci-lint@v1.52.2 run
.PHONY: depgraph.svg
depgraph.svg:
depgraph.svg: always
go run github.com/kisielk/godepgraph@latest github.com/wader/fq | dot -Tsvg -o godepgraph.svg
# make memprof ARGS=". test.mp3"
# make cpuprof ARGS=". test.mp3"
.PHONY: prof
prof:
go build -tags profile -o fq.prof fq.go
prof: always
go build -tags profile -o fq.prof .
CPUPROFILE=fq.cpu.prof MEMPROFILE=fq.mem.prof ./fq.prof ${ARGS}
.PHONY: memprof
memprof: prof
go tool pprof -http :5555 fq.prof fq.mem.prof
.PHONY: cpuprof
cpuprof: prof
go tool pprof -http :5555 fq.prof fq.cpu.prof
.PHONY: update-gomod
update-gomod:
update-gomod: always
GOPROXY=direct go get -d github.com/wader/readline@fq
GOPROXY=direct go get -d github.com/wader/gojq@fq
go mod tidy
@ -99,19 +85,18 @@ update-gomod:
# Usage: make fuzz GROUP=mp4 # fuzz a group (each format is a group also)
# TODO: as decode recovers panic and "repanics" unrecoverable errors this is a bit hacky at the moment
# Retrigger:
# try to decode crsash with all formats:
# try to decode crash with all formats in order to see which one paniced:
# cat format/testdata/fuzz/FuzzFormats/... | go run dev/fuzzbytes.go | go run . -d bytes '. as $b | formats | keys[] as $f | $b | decode($f)'
# convert crash into raw bytes:
# cat format/testdata/fuzz/FuzzFormats/... | go run dev/fuzzbytes.go | fq -d bytes tobase64
# fq -n '"..." | frombase64 | ...'
.PHONY: fuzz
fuzz:
# cat format/testdata/fuzz/FuzzFormats/... | go run dev/fuzzbytes.go | fq -d bytes to_base64
# fq -n '"..." | from_base64 | ...'
fuzz: always
# in other terminal: tail -f /tmp/repanic
FUZZTEST=1 REPANIC_LOG=/tmp/repanic go test -v -run Fuzz -fuzz=Fuzz ./format/
FUZZTEST=1 go test -v -run Fuzz -fuzz=Fuzz ./format/
# usage: make release VERSION=0.0.1
# tag forked dependeces for history and to make then stay around
.PHONY: release
release: always
release: WADER_GOJQ_COMMIT=$(shell go list -m -f '{{.Version}}' github.com/wader/gojq | sed 's/.*-\(.*\)/\1/')
release: WADER_READLINE_COMMIT=$(shell go list -m -f '{{.Version}}' github.com/wader/readline | sed 's/.*-\(.*\)/\1/')
release:

28
README.md
View File

@ -37,8 +37,10 @@ In summary it aims to be jq, hexdump, dd and gdb for files combined into one.
[aac_frame](doc/formats.md#aac_frame),
adts,
adts_frame,
aiff,
amf0,
apev2,
[apple_bookmark](doc/formats.md#apple_bookmark),
ar,
[asn1_ber](doc/formats.md#asn1_ber),
av1_ccr,
@ -105,7 +107,8 @@ macho_fat,
[matroska](doc/formats.md#matroska),
[mp3](doc/formats.md#mp3),
mp3_frame,
mp3_frame_tags,
mp3_frame_vbri,
mp3_frame_xing,
[mp4](doc/formats.md#mp4),
mpeg_asc,
mpeg_es,
@ -134,7 +137,9 @@ sll_packet,
tar,
tcp_segment,
tiff,
[tls](doc/formats.md#tls),
toml,
[tzif](doc/formats.md#tzif),
udp_datagram,
vorbis_comment,
vorbis_packet,
@ -157,6 +162,7 @@ For details see [formats.md](doc/formats.md) and [usage.md](doc/usage.md).
## Presentations
- "fq - jq for binary formats" at [FOSDEM 2023](https://fosdem.org/2023/) - [video & slides](https://fosdem.org/2023/schedule/event/bintools_fq/)
- "fq - jq for binary formats" at [No time to wait 6](https://mediaarea.net/NoTimeToWait6) - [video](https://www.youtube.com/watch?v=-Pwt5KL-xRs&t=1450s) - [slides](doc/presentations/nttw6/fq-nttw6-slides.pdf)
- "fq - jq for binary formats" at [Binary Tools Summit 2022](https://binary-tools.net/summit.html) - [video](https://www.youtube.com/watch?v=GJOq_b0eb-s&list=PLTj8twuHdQz-JcX7k6eOwyVPDB8CyfZc8&index=1) - [slides](doc/presentations/bts2022/fq-bts2022-v1.pdf)
@ -164,7 +170,7 @@ For details see [formats.md](doc/formats.md) and [usage.md](doc/usage.md).
Use one of the methods listed below or download a pre-built [release](https://github.com/wader/fq/releases) for macOS, Linux or Windows. Unarchive it and move the executable to `PATH` etc.
On macOS if you don't install using a method below you might have to manually allow the binary to run. This can be done by trying to run the binary, ignore the warning and then go into security preference and allow it. Same can be done with this command:
On macOS if you don't install using one of the method below you might have to manually allow the binary to run. This can be done by trying to run the binary, ignore the warning and then go into security preference and allow it. Same can be done with this command:
```sh
xattr -d com.apple.quarantine fq && spctl --add fq
@ -242,10 +248,10 @@ cp "$(go env GOPATH)/bin/fq" /usr/local/bin
To build, run and test from source:
```sh
# build an run
go run fq.go
# or
# build and run
go run .
# build and run with arguments
go run . -d mp3 . file.mp3
# just build
go build -o fq .
# run all tests and build binary
@ -267,7 +273,9 @@ jq implementation [gojq](https://github.com/itchyny/gojq). I also want to thank
[HexFiend](https://github.com/HexFiend/HexFiend) for inspiration and ideas and [stedolan](https://github.com/stedolan)
for inventing the [jq](https://github.com/stedolan/jq) language.
### Similar or related projects
### Similar or related works
#### Tools
- [HexFiend](https://github.com/HexFiend/HexFiend) Hex editor for macOS with format template support.
- [binspector](https://github.com/binspector/binspector) Binary format analysis tool with query langauge and REPL.
@ -281,6 +289,13 @@ for inventing the [jq](https://github.com/stedolan/jq) language.
- [hachoir](https://github.com/vstinner/hachoir) General python library for working binary data.
- [scapy](https://scapy.net) Decode/Encode formats, focus on network protocols.
#### Projects and Standards
- [Let's Solve the File Format Problem](http://fileformats.archiveteam.org)
- [PRONOM](https://www.nationalarchives.gov.uk/PRONOM/) file format registry
- [Sustainability of Digital Formats](https://www.loc.gov/preservation/digital/formats/) at Library of Congress
- [Data Format Description Language](https://en.wikipedia.org/wiki/Data_Format_Description_Language)
## License
`fq` is distributed under the terms of the MIT License.
@ -301,3 +316,4 @@ Licenses of direct dependencies:
- golang/snappy https://github.com/golang/snappy/blob/master/LICENSE (BSD)
- golang/x/* https://github.com/golang/text/blob/master/LICENSE (BSD)
- gopkg.in/yaml.v3 https://github.com/go-yaml/yaml/blob/v3/LICENSE (MIT)
- Parts of go crypto/tls and github.com/zmap/zcrypto https://github.com/zmap/zcrypto/blob/master/LICENSE (Apache)

2
dev/fuzzbytes.go
View File

@ -1,5 +1,5 @@
// tool to convert go fuzz input files to bytes
// Usage: cat format/testdata/fuzz/FuzzFormats/144bde49b40c90fd05d302ec90b6ddb2b6d6aea553bad520a8b954797e40fe72 | go run dev/fuzzbytes.go | go run fq.go
// Usage: cat format/testdata/fuzz/FuzzFormats/144bde49b40c90fd05d302ec90b6ddb2b6d6aea553bad520a8b954797e40fe72 | go run dev/fuzzbytes.go | go run .
package main
import (

9
dev/snippets.jq
View File

@ -1,11 +1,6 @@
def protobuf_to_value:
.fields | map({(.name | tostring): (.enum // .value)}) | add;
# hack to parse just a box
# <binary> | mp4_box
def mp4_box:
[0, 0, 0, 16, "ftyp", "isom", 0, 0 , 2 , 0, .] | mp4.boxes;
# converted from https://github.com/FFmpeg/FFmpeg/blob/870bfe16a12bf09dca3a4ae27ef6f81a2de80c40/libavutil/display.c av_display_rotation_get
def mp4_matrix_structure_rotation:
( .a as $s0
@ -41,8 +36,8 @@ def urldecode:
# ex: .frames | changes(.header.sample_rate)
def changes(f): streaks_by(f)[].[0];
def toradix62sp: toradix(62; "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ");
def fromradix62sp: fromradix(62; {
def to_radix62sp: to_radix(62; "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ");
def from_radix62sp: from_radix(62; {
"0": 0, "1": 1, "2": 2, "3": 3,"4": 4, "5": 5, "6": 6, "7": 7, "8": 8, "9": 9,
"a": 10, "b": 11, "c": 12, "d": 13, "e": 14, "f": 15, "g": 16,
"h": 17, "i": 18, "j": 19, "k": 20, "l": 21, "m": 22, "n": 23,

4
doc/TODO.md
View File

@ -4,10 +4,10 @@
- Buffers/string duality is confusing, most string functions should be wrapped to understand binary.
- REPL cancel seems to sometimes exit a sub-REPl without properly cleanup options.
- Value errors, can only be accessed with `._error`.
- Framed (add unknown in gaps) decode should be on struct level not format?
- Framed (add unknown gaps) decode should be on struct level not format?
- `tovalue({bits_format: "base64"})` only affect root value.
- Auto complete of non-global variables is broken. `scope` is broken for variables.
- `echo '{} {} {}' | jq` vs `echo '{} {} {}' | fq` works differently. fq currently decodes one root format and might add unknown fields etc. Maybe should work differently for `json` format?
- `echo '{} {} {}' | jq` vs `echo '{} {} {}' | fq` works differently. fq currently decodes one root format and might add unknown gap fields etc. Maybe should work differently for `json` format?
- `format/0` overlap with jq builtin `format/1`. What to rename it to? `decode_format`?
- repl expression returning a value that produced lots of output can't be interrupted. This is becaus ctrl-c currently only interrupts the eval interpreter, outputted value is printed (`display`) by parent interpreter.
- Rework cli/repl user interrupt (context cancel via ctrl-c), see comment in Interp.Main

2571
doc/demo.svg
View File

File diff suppressed because it is too large Load Diff
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 82 KiB

After

Width:  |  Height:  |  Size: 74 KiB

12
doc/demo.sh → doc/demo.svg.sh
View File

@ -14,11 +14,11 @@ s() {
c "Overview of mp3 file"
s "fq . file.mp3"
echo
c "Show ID3v2 tag inside mp3 file"
s "fq '.headers[0]' file.mp3"
c "Show header of first ID3v2 tag inside mp3 file"
s "fq '.headers[0].header' file.mp3"
echo
c "Show encoder software used"
s "fq -r '.frames[0].tags.encoder | tovalue' file.mp3"
s "fq -r '.frames[0].tag.encoder | tovalue' file.mp3"
echo
c "Decode at two offsets as mp3_frame and show bitrate"
s "fq -d bytes '.[0xb79,0xc49:] | mp3_frame.header.bitrate' file.mp3"
@ -29,10 +29,10 @@ s "file file.png"
rm -f file.png
echo
c "Resolution of embedded PNG cover art as YAML"
s "fq -r '.headers[0].frames[] | grep_by(.id == \"APIC\") | grep_by(.type == \"IHDR\") | {res: {width, height}} | toyaml' file.mp3"
s "fq -r '.headers[0].frames[] | grep_by(.id == \"APIC\") | grep_by(.type == \"IHDR\") | {res: {width, height}} | to_yaml' file.mp3"
#echo
c "Add query parameter to URL"
s "echo 'http://host?a=b' | fq -Rr 'fromurl | .query.b = \"a b c\" | tourl'"
s "echo 'http://host?a=b' | fq -Rr 'from_url | .query.b = \"a b c\" | to_url'"
echo
c "Extract JSON and base64 encoded query parameter p"
s "echo 'https://host?p=eyJhIjoiaGVsbG8ifQ%3D%3D' | fq -R 'fromurl.query.p | frombase64 | fromjson'"
s "echo 'https://host?p=eyJhIjoiaGVsbG8ifQ%3D%3D' | fq -R 'from_url.query.p | from_base64 | fromjson'"

58
doc/dev.md
View File

@ -11,26 +11,46 @@ to `format/format.go` and don't forget to change the string constant.
### Some general tips
- Main goal is to produce a tree structure that is user-friendly and easy to work with.
Prefer a nice and easy query tree structure over nice decoder implementation.
Prefer a nice and easy tree structure over nice decoder implementation.
- Use same names, symbols, constant number bases etc as in specification.
But maybe in lowercase to be jq/JSON-ish.
- Decode only ranges you know what they are. If possible let "parent" decide what to do with unknown
bits by using `*Decode*Len/Range/Limit` functions. fq will also automatically add "unknown" fields if
- Decode only ranges you know what they are. If possible let "parent" decide what to do with unknown gaps
bits by using `*Decode*Len/Range/Limit` functions. fq will also automatically add "gap" fields if
it finds gaps.
- If you have decode helpers functions that decode a bunch of fields etc it is usually nice to make it only decode fields, not seek or add it's own "containing" struct. That way the function will be easier to reuse and only do one thing. Ex the helper `func decodeHeader(d *decode.D)` can then be use as `d.FieldStruct("header", decodeHeader)`, `d.SeekRel(1234, decodeHeader)` or `d.SeekRel(1234, func(d *decode.D) { d.FieldStruct("header, decodeHeader") }`
- Try to not decode too much as one value.
A length encoded int could be two fields, but maybe a length prefixed string should be one.
Flags can be struct with bit-fields.
- Map as many value as possible to more symbolic values.
- Map as many value as possible to symbolic values.
- Endian is inherited inside one format decoder, defaults to big endian for new format decoder
- Make sure zero length or no frames found etc fails decoding
- Make sure zero length or no frames/packets etc fails decoding
- If format is in the probe group make sure to validate input to make it non-ambiguous with other decoders
- Try keep decoder code as declarative as possible
- Try keep decoder code "declarative" if possible
- Split into multiple sub formats if possible. Makes it possible to use them separately.
- Validate/Assert
- Error/Fatal/panic
- Is format probeable or not
- Can new formats be added to other formats
- Does the new format include existing formats
- Can new formats be added to other formats?
- Does the new format include existing formats?
### Checklist
- Commits:
- Use commit messages with a context prefix to make it easier to find and understand, ex:<br>
`mp3: Validate sync correctly`
- Tests:
- If possible use a pair of `testdata/file` and `testdata/file.fqtest` where `file.fqtest` is `$ fq dv file` or `$ fq 'dv,torepr' file` if there is `torepr` support.
- If `dv` produces a lof of output maybe use `dv({array_truncate: 50})` etc
- Run `go test ./format -run TestFormats/<name>` to test expected output.
- Run `WRITE_ACTUAL=1 go test ./format -run TestFormats/<name>` to write current output as expected output.
- If you have format specific documentation:
- Put it in `format/*/<name>.md` and use `//go:embed <name>.md`/`interp.RegisterFS(..)` to embed/register it.
- Use simple markdown, just sections (depth starts at 3, `### Section`), paragraphs, lists and links.
- No heading section is needs with format name, will be added by `make doc` and fq cli help system.
- Add a `testdata/<name>_help.fqtest` with just `$ fq -h <name>` to test CLI help.
- If in doubt look at `mp4.md`/`mp4.go` etc.
- Run `make README.md doc/formats.md` to update md files.
- Run linter `make lint`
- Run fuzzer `make fuzz GROUP=<name>`, see usage in Makefile
### Decoder API
@ -77,7 +97,7 @@ d.FieldUTF8("magic", 4)
// create a new struct and add it as "headers", returns a *decode.D
d.FieldStruct("headers", func(d *decode.D) {
// read 8 bit unsigned integer, map it and add it as "type", returns a uint64
d.FieldU8("type", scalar.UToSymStr{
d.FieldU8("type", scalar.UintMapSymStr{
1: "start",
// ...
})
@ -94,7 +114,7 @@ will produce something like this:
Children: []*decode.Value{
*decode.Value{
Name: "magic",
V: scalar.S{
V: scalar.Str{
Actual: "abcd", // read and set by UTF8 reader
},
Range: ranges.Range{Start: 0, Len: 32},
@ -107,9 +127,9 @@ will produce something like this:
Children: []*decode.Value{
*decode.Value{
Name: "type",
V: scalar.S{
V: scalar.Uint{
Actual: uint64(1), // read and set by U8 reader
Sym: "start", // set by UToSymStr scalar.Mapper
Sym: "start", // set by UintMapSymStr scalar.Mapper
},
Range: ranges.Range{Start: 32, Len: 8},
},
@ -165,7 +185,7 @@ Decoder authors will probably not have to create them.
Keeps track of
- Actual value. Decoded value represented using a go type like `uint64`, `string` etc. For example a value reader by a utf8 or utf16 reader both will ends up as a `string`.
- Symbolic value. Optional symbolic representation of the actual value. For example a `scalar.UToSymStr` would map an actual `uint64` to a symbolic `string`.
- Symbolic value. Optional symbolic representation of the actual value. For example a `scalar.UintMapSymStr` would map an actual `uint64` to a symbolic `string`.
- String description of the value.
- Number representation
@ -185,7 +205,7 @@ I ususally use `-d <format>` and `dv` while developing, that way you will get a
even if it fails. `dv` gives verbose output and also includes stacktrace.
```sh
go run fq.go -d <format> dv file
go run . -d <format> dv file
```
If the format is inside some other format it can be handy to first extract the bits and run
@ -205,7 +225,7 @@ make things more comfortable. Also using vscode/delve for debugging should work
launch `args` are setup etc.
```
watchexec "go run fq.go -d aac_frame dv aac_frame"
watchexec "go run . -d aac_frame dv aac_frame"
```
Some different ways to run tests:
@ -215,7 +235,7 @@ make test
# run all go tests
go test ./...
# run all tests for one format
go test -run TestFQTests/mp4 ./format/
go test -run TestFormats/mp4 ./format/
# write all actual outputs
WRITE_ACTUAL=1 go test ./...
# write actual output for specific tests
@ -242,12 +262,12 @@ Split debug and normal output even when using repl:
Write `log` package output and stderr to a file that can be `tail -f`:ed in another terminal:
```sh
LOGFILE=/tmp/log go run fq.go ... 2>>/tmp/log
LOGFILE=/tmp/log go run . ... 2>>/tmp/log
```
gojq execution debug:
```sh
GOJQ_DEBUG=1 go run -tags debug fq.go ...
GOJQ_DEBUG=1 go run -tags debug . ...
```
Memory and CPU profile (will open a browser):

0
doc/display_decode_value.sh → doc/display_decode_value.svg.sh
View File

0
doc/display_decode_value_d.sh → doc/display_decode_value_d.svg.sh
View File

0
doc/display_decode_value_dv.sh → doc/display_decode_value_dv.svg.sh
View File

0
doc/display_json.sh → doc/display_json.svg.sh
View File

4
doc/formats.jq
View File

@ -60,8 +60,8 @@ def formats_table:
, .uses
];
[ ""
, (.[0] | . as $rc | $rc.string | rpad(" "; $rc.maxwidth))
, (.[1] | . as $rc | $rc.string | rpad(" "; $rc.maxwidth))
, (.[0] | . as $rc | $rc.string | rpad(" "; $rc.maxwidth | [., .+20] | max))
, (.[1] | . as $rc | $rc.string | rpad(" "; $rc.maxwidth | [., .+20] | max))
, .[2].string
, ""
] | join("|")

563
doc/formats.md
View File

@ -2,125 +2,131 @@
[fq -rn -L . 'include "formats"; formats_table']: sh-start
|Name |Description |Dependencies|
|- |- |-|
|[`aac_frame`](#aac_frame) |Advanced&nbsp;Audio&nbsp;Coding&nbsp;frame |<sub></sub>|
|`adts` |Audio&nbsp;Data&nbsp;Transport&nbsp;Stream |<sub>`adts_frame`</sub>|
|`adts_frame` |Audio&nbsp;Data&nbsp;Transport&nbsp;Stream&nbsp;frame |<sub>`aac_frame`</sub>|
|`amf0` |Action&nbsp;Message&nbsp;Format&nbsp;0 |<sub></sub>|
|`apev2` |APEv2&nbsp;metadata&nbsp;tag |<sub>`image`</sub>|
|`ar` |Unix&nbsp;archive |<sub>`probe`</sub>|
|[`asn1_ber`](#asn1_ber) |ASN1&nbsp;BER&nbsp;(basic&nbsp;encoding&nbsp;rules,&nbsp;also&nbsp;CER&nbsp;and&nbsp;DER)|<sub></sub>|
|`av1_ccr` |AV1&nbsp;Codec&nbsp;Configuration&nbsp;Record |<sub></sub>|
|`av1_frame` |AV1&nbsp;frame |<sub>`av1_obu`</sub>|
|`av1_obu` |AV1&nbsp;Open&nbsp;Bitstream&nbsp;Unit |<sub></sub>|
|`avc_annexb` |H.264/AVC&nbsp;Annex&nbsp;B |<sub>`avc_nalu`</sub>|
|[`avc_au`](#avc_au) |H.264/AVC&nbsp;Access&nbsp;Unit |<sub>`avc_nalu`</sub>|
|`avc_dcr` |H.264/AVC&nbsp;Decoder&nbsp;Configuration&nbsp;Record |<sub>`avc_nalu`</sub>|
|`avc_nalu` |H.264/AVC&nbsp;Network&nbsp;Access&nbsp;Layer&nbsp;Unit |<sub>`avc_sps` `avc_pps` `avc_sei`</sub>|
|`avc_pps` |H.264/AVC&nbsp;Picture&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`avc_sei` |H.264/AVC&nbsp;Supplemental&nbsp;Enhancement&nbsp;Information |<sub></sub>|
|`avc_sps` |H.264/AVC&nbsp;Sequence&nbsp;Parameter&nbsp;Set |<sub></sub>|
|[`avi`](#avi) |Audio&nbsp;Video&nbsp;Interleaved |<sub>`avc_au` `hevc_au` `mp3_frame` `flac_frame`</sub>|
|[`avro_ocf`](#avro_ocf) |Avro&nbsp;object&nbsp;container&nbsp;file |<sub></sub>|
|[`bencode`](#bencode) |BitTorrent&nbsp;bencoding |<sub></sub>|
|`bitcoin_blkdat` |Bitcoin&nbsp;blk.dat |<sub>`bitcoin_block`</sub>|
|[`bitcoin_block`](#bitcoin_block) |Bitcoin&nbsp;block |<sub>`bitcoin_transaction`</sub>|
|`bitcoin_script` |Bitcoin&nbsp;script |<sub></sub>|
|`bitcoin_transaction` |Bitcoin&nbsp;transaction |<sub>`bitcoin_script`</sub>|
|[`bits`](#bits) |Raw&nbsp;bits |<sub></sub>|
|[`bplist`](#bplist) |Apple&nbsp;Binary&nbsp;Property&nbsp;List |<sub></sub>|
|`bsd_loopback_frame` |BSD&nbsp;loopback&nbsp;frame |<sub>`inet_packet`</sub>|
|[`bson`](#bson) |Binary&nbsp;JSON |<sub></sub>|
|[`bytes`](#bytes) |Raw&nbsp;bytes |<sub></sub>|
|`bzip2` |bzip2&nbsp;compression |<sub>`probe`</sub>|
|[`cbor`](#cbor) |Concise&nbsp;Binary&nbsp;Object&nbsp;Representation |<sub></sub>|
|[`csv`](#csv) |Comma&nbsp;separated&nbsp;values |<sub></sub>|
|`dns` |DNS&nbsp;packet |<sub></sub>|
|`dns_tcp` |DNS&nbsp;packet&nbsp;(TCP) |<sub></sub>|
|`elf` |Executable&nbsp;and&nbsp;Linkable&nbsp;Format |<sub></sub>|
|`ether8023_frame` |Ethernet&nbsp;802.3&nbsp;frame |<sub>`inet_packet`</sub>|
|`exif` |Exchangeable&nbsp;Image&nbsp;File&nbsp;Format |<sub></sub>|
|`fairplay_spc` |FairPlay&nbsp;Server&nbsp;Playback&nbsp;Context |<sub></sub>|
|`flac` |Free&nbsp;Lossless&nbsp;Audio&nbsp;Codec&nbsp;file |<sub>`flac_metadatablocks` `flac_frame`</sub>|
|[`flac_frame`](#flac_frame) |FLAC&nbsp;frame |<sub></sub>|
|`flac_metadatablock` |FLAC&nbsp;metadatablock |<sub>`flac_streaminfo` `flac_picture` `vorbis_comment`</sub>|
|`flac_metadatablocks` |FLAC&nbsp;metadatablocks |<sub>`flac_metadatablock`</sub>|
|`flac_picture` |FLAC&nbsp;metadatablock&nbsp;picture |<sub>`image`</sub>|
|`flac_streaminfo` |FLAC&nbsp;streaminfo |<sub></sub>|
|`gif` |Graphics&nbsp;Interchange&nbsp;Format |<sub></sub>|
|`gzip` |gzip&nbsp;compression |<sub>`probe`</sub>|
|`hevc_annexb` |H.265/HEVC&nbsp;Annex&nbsp;B |<sub>`hevc_nalu`</sub>|
|[`hevc_au`](#hevc_au) |H.265/HEVC&nbsp;Access&nbsp;Unit |<sub>`hevc_nalu`</sub>|
|`hevc_dcr` |H.265/HEVC&nbsp;Decoder&nbsp;Configuration&nbsp;Record |<sub>`hevc_nalu`</sub>|
|`hevc_nalu` |H.265/HEVC&nbsp;Network&nbsp;Access&nbsp;Layer&nbsp;Unit |<sub>`hevc_vps` `hevc_pps` `hevc_sps`</sub>|
|`hevc_pps` |H.265/HEVC&nbsp;Picture&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`hevc_sps` |H.265/HEVC&nbsp;Sequence&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`hevc_vps` |H.265/HEVC&nbsp;Video&nbsp;Parameter&nbsp;Set |<sub></sub>|
|[`html`](#html) |HyperText&nbsp;Markup&nbsp;Language |<sub></sub>|
|`icc_profile` |International&nbsp;Color&nbsp;Consortium&nbsp;profile |<sub></sub>|
|`icmp` |Internet&nbsp;Control&nbsp;Message&nbsp;Protocol |<sub></sub>|
|`icmpv6` |Internet&nbsp;Control&nbsp;Message&nbsp;Protocol&nbsp;v6 |<sub></sub>|
|`id3v1` |ID3v1&nbsp;metadata |<sub></sub>|
|`id3v11` |ID3v1.1&nbsp;metadata |<sub></sub>|
|`id3v2` |ID3v2&nbsp;metadata |<sub>`image`</sub>|
|`ipv4_packet` |Internet&nbsp;protocol&nbsp;v4&nbsp;packet |<sub>`ip_packet`</sub>|
|`ipv6_packet` |Internet&nbsp;protocol&nbsp;v6&nbsp;packet |<sub>`ip_packet`</sub>|
|`jpeg` |Joint&nbsp;Photographic&nbsp;Experts&nbsp;Group&nbsp;file |<sub>`exif` `icc_profile`</sub>|
|`json` |JavaScript&nbsp;Object&nbsp;Notation |<sub></sub>|
|`jsonl` |JavaScript&nbsp;Object&nbsp;Notation&nbsp;Lines |<sub></sub>|
|[`macho`](#macho) |Mach-O&nbsp;macOS&nbsp;executable |<sub></sub>|
|`macho_fat` |Fat&nbsp;Mach-O&nbsp;macOS&nbsp;executable&nbsp;(multi-architecture) |<sub>`macho`</sub>|
|[`markdown`](#markdown) |Markdown |<sub></sub>|
|[`matroska`](#matroska) |Matroska&nbsp;file |<sub>`aac_frame` `av1_ccr` `av1_frame` `avc_au` `avc_dcr` `flac_frame` `flac_metadatablocks` `hevc_au` `hevc_dcr` `image` `mp3_frame` `mpeg_asc` `mpeg_pes_packet` `mpeg_spu` `opus_packet` `vorbis_packet` `vp8_frame` `vp9_cfm` `vp9_frame`</sub>|
|[`mp3`](#mp3) |MP3&nbsp;file |<sub>`id3v2` `id3v1` `id3v11` `apev2` `mp3_frame`</sub>|
|`mp3_frame` |MPEG&nbsp;audio&nbsp;layer&nbsp;3&nbsp;frame |<sub>`mp3_frame_tags`</sub>|
|`mp3_frame_tags` |MP3&nbsp;frame&nbsp;info/xing&nbsp;tags |<sub></sub>|
|[`mp4`](#mp4) |ISOBMFF,&nbsp;QuickTime&nbsp;and&nbsp;similar |<sub>`aac_frame` `av1_ccr` `av1_frame` `avc_au` `avc_dcr` `flac_frame` `flac_metadatablocks` `hevc_au` `hevc_dcr` `icc_profile` `id3v2` `image` `jpeg` `mp3_frame` `mpeg_es` `mpeg_pes_packet` `opus_packet` `prores_frame` `protobuf_widevine` `pssh_playready` `vorbis_packet` `vp9_frame` `vpx_ccr`</sub>|
|`mpeg_asc` |MPEG-4&nbsp;Audio&nbsp;Specific&nbsp;Config |<sub></sub>|
|`mpeg_es` |MPEG&nbsp;Elementary&nbsp;Stream |<sub>`mpeg_asc` `vorbis_packet`</sub>|
|`mpeg_pes` |MPEG&nbsp;Packetized&nbsp;elementary&nbsp;stream |<sub>`mpeg_pes_packet` `mpeg_spu`</sub>|
|`mpeg_pes_packet` |MPEG&nbsp;Packetized&nbsp;elementary&nbsp;stream&nbsp;packet |<sub></sub>|
|`mpeg_spu` |Sub&nbsp;Picture&nbsp;Unit&nbsp;(DVD&nbsp;subtitle) |<sub></sub>|
|`mpeg_ts` |MPEG&nbsp;Transport&nbsp;Stream |<sub></sub>|
|[`msgpack`](#msgpack) |MessagePack |<sub></sub>|
|`ogg` |OGG&nbsp;file |<sub>`ogg_page` `vorbis_packet` `opus_packet` `flac_metadatablock` `flac_frame`</sub>|
|`ogg_page` |OGG&nbsp;page |<sub></sub>|
|`opus_packet` |Opus&nbsp;packet |<sub>`vorbis_comment`</sub>|
|[`pcap`](#pcap) |PCAP&nbsp;packet&nbsp;capture |<sub>`link_frame` `tcp_stream` `ipv4_packet`</sub>|
|`pcapng` |PCAPNG&nbsp;packet&nbsp;capture |<sub>`link_frame` `tcp_stream` `ipv4_packet`</sub>|
|`png` |Portable&nbsp;Network&nbsp;Graphics&nbsp;file |<sub>`icc_profile` `exif`</sub>|
|`prores_frame` |Apple&nbsp;ProRes&nbsp;frame |<sub></sub>|
|[`protobuf`](#protobuf) |Protobuf |<sub></sub>|
|`protobuf_widevine` |Widevine&nbsp;protobuf |<sub>`protobuf`</sub>|
|`pssh_playready` |PlayReady&nbsp;PSSH |<sub></sub>|
|[`rtmp`](#rtmp) |Real-Time&nbsp;Messaging&nbsp;Protocol |<sub>`amf0` `mpeg_asc`</sub>|
|`sll2_packet` |Linux&nbsp;cooked&nbsp;capture&nbsp;encapsulation&nbsp;v2 |<sub>`inet_packet`</sub>|
|`sll_packet` |Linux&nbsp;cooked&nbsp;capture&nbsp;encapsulation |<sub>`inet_packet`</sub>|
|`tar` |Tar&nbsp;archive |<sub>`probe`</sub>|
|`tcp_segment` |Transmission&nbsp;control&nbsp;protocol&nbsp;segment |<sub></sub>|
|`tiff` |Tag&nbsp;Image&nbsp;File&nbsp;Format |<sub>`icc_profile`</sub>|
|`toml` |Tom's&nbsp;Obvious,&nbsp;Minimal&nbsp;Language |<sub></sub>|
|`udp_datagram` |User&nbsp;datagram&nbsp;protocol |<sub>`udp_payload`</sub>|
|`vorbis_comment` |Vorbis&nbsp;comment |<sub>`flac_picture`</sub>|
|`vorbis_packet` |Vorbis&nbsp;packet |<sub>`vorbis_comment`</sub>|
|`vp8_frame` |VP8&nbsp;frame |<sub></sub>|
|`vp9_cfm` |VP9&nbsp;Codec&nbsp;Feature&nbsp;Metadata |<sub></sub>|
|`vp9_frame` |VP9&nbsp;frame |<sub></sub>|
|`vpx_ccr` |VPX&nbsp;Codec&nbsp;Configuration&nbsp;Record |<sub></sub>|
|[`wasm`](#wasm) |WebAssembly&nbsp;Binary&nbsp;Format |<sub></sub>|
|`wav` |WAV&nbsp;file |<sub>`id3v2` `id3v1` `id3v11`</sub>|
|`webp` |WebP&nbsp;image |<sub>`vp8_frame`</sub>|
|[`xml`](#xml) |Extensible&nbsp;Markup&nbsp;Language |<sub></sub>|
|`yaml` |YAML&nbsp;Ain't&nbsp;Markup&nbsp;Language |<sub></sub>|
|[`zip`](#zip) |ZIP&nbsp;archive |<sub>`probe`</sub>|
|`image` |Group |<sub>`gif` `jpeg` `mp4` `png` `tiff` `webp`</sub>|
|`inet_packet` |Group |<sub>`ipv4_packet` `ipv6_packet`</sub>|
|`ip_packet` |Group |<sub>`icmp` `icmpv6` `tcp_segment` `udp_datagram`</sub>|
|`link_frame` |Group |<sub>`bsd_loopback_frame` `ether8023_frame` `sll2_packet` `sll_packet`</sub>|
|`probe` |Group |<sub>`adts` `ar` `avi` `avro_ocf` `bitcoin_blkdat` `bplist` `bzip2` `elf` `flac` `gif` `gzip` `jpeg` `json` `jsonl` `macho` `macho_fat` `matroska` `mp3` `mp4` `mpeg_ts` `ogg` `pcap` `pcapng` `png` `tar` `tiff` `toml` `wasm` `wav` `webp` `xml` `yaml` `zip`</sub>|
|`tcp_stream` |Group |<sub>`dns_tcp` `rtmp`</sub>|
|`udp_payload` |Group |<sub>`dns`</sub>|
|Name |Description |Dependencies|
|- |- |-|
|[`aac_frame`](#aac_frame) |Advanced&nbsp;Audio&nbsp;Coding&nbsp;frame |<sub></sub>|
|`adts` |Audio&nbsp;Data&nbsp;Transport&nbsp;Stream |<sub>`adts_frame`</sub>|
|`adts_frame` |Audio&nbsp;Data&nbsp;Transport&nbsp;Stream&nbsp;frame |<sub>`aac_frame`</sub>|
|`aiff` |Audio&nbsp;Interchange&nbsp;File&nbsp;Format |<sub></sub>|
|`amf0` |Action&nbsp;Message&nbsp;Format&nbsp;0 |<sub></sub>|
|`apev2` |APEv2&nbsp;metadata&nbsp;tag |<sub>`image`</sub>|
|[`apple_bookmark`](#apple_bookmark) |Apple&nbsp;BookmarkData |<sub></sub>|
|`ar` |Unix&nbsp;archive |<sub>`probe`</sub>|
|[`asn1_ber`](#asn1_ber) |ASN1&nbsp;BER&nbsp;(basic&nbsp;encoding&nbsp;rules,&nbsp;also&nbsp;CER&nbsp;and&nbsp;DER) |<sub></sub>|
|`av1_ccr` |AV1&nbsp;Codec&nbsp;Configuration&nbsp;Record |<sub></sub>|
|`av1_frame` |AV1&nbsp;frame |<sub>`av1_obu`</sub>|
|`av1_obu` |AV1&nbsp;Open&nbsp;Bitstream&nbsp;Unit |<sub></sub>|
|`avc_annexb` |H.264/AVC&nbsp;Annex&nbsp;B |<sub>`avc_nalu`</sub>|
|[`avc_au`](#avc_au) |H.264/AVC&nbsp;Access&nbsp;Unit |<sub>`avc_nalu`</sub>|
|`avc_dcr` |H.264/AVC&nbsp;Decoder&nbsp;Configuration&nbsp;Record |<sub>`avc_nalu`</sub>|
|`avc_nalu` |H.264/AVC&nbsp;Network&nbsp;Access&nbsp;Layer&nbsp;Unit |<sub>`avc_sps` `avc_pps` `avc_sei`</sub>|
|`avc_pps` |H.264/AVC&nbsp;Picture&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`avc_sei` |H.264/AVC&nbsp;Supplemental&nbsp;Enhancement&nbsp;Information |<sub></sub>|
|`avc_sps` |H.264/AVC&nbsp;Sequence&nbsp;Parameter&nbsp;Set |<sub></sub>|
|[`avi`](#avi) |Audio&nbsp;Video&nbsp;Interleaved |<sub>`avc_au` `hevc_au` `mp3_frame` `flac_frame`</sub>|
|[`avro_ocf`](#avro_ocf) |Avro&nbsp;object&nbsp;container&nbsp;file |<sub></sub>|
|[`bencode`](#bencode) |BitTorrent&nbsp;bencoding |<sub></sub>|
|`bitcoin_blkdat` |Bitcoin&nbsp;blk.dat |<sub>`bitcoin_block`</sub>|
|[`bitcoin_block`](#bitcoin_block) |Bitcoin&nbsp;block |<sub>`bitcoin_transaction`</sub>|
|`bitcoin_script` |Bitcoin&nbsp;script |<sub></sub>|
|`bitcoin_transaction` |Bitcoin&nbsp;transaction |<sub>`bitcoin_script`</sub>|
|[`bits`](#bits) |Raw&nbsp;bits |<sub></sub>|
|[`bplist`](#bplist) |Apple&nbsp;Binary&nbsp;Property&nbsp;List |<sub></sub>|
|`bsd_loopback_frame` |BSD&nbsp;loopback&nbsp;frame |<sub>`inet_packet`</sub>|
|[`bson`](#bson) |Binary&nbsp;JSON |<sub></sub>|
|[`bytes`](#bytes) |Raw&nbsp;bytes |<sub></sub>|
|`bzip2` |bzip2&nbsp;compression |<sub>`probe`</sub>|
|[`cbor`](#cbor) |Concise&nbsp;Binary&nbsp;Object&nbsp;Representation |<sub></sub>|
|[`csv`](#csv) |Comma&nbsp;separated&nbsp;values |<sub></sub>|
|`dns` |DNS&nbsp;packet |<sub></sub>|
|`dns_tcp` |DNS&nbsp;packet&nbsp;(TCP) |<sub></sub>|
|`elf` |Executable&nbsp;and&nbsp;Linkable&nbsp;Format |<sub></sub>|
|`ether8023_frame` |Ethernet&nbsp;802.3&nbsp;frame |<sub>`inet_packet`</sub>|
|`exif` |Exchangeable&nbsp;Image&nbsp;File&nbsp;Format |<sub></sub>|
|`fairplay_spc` |FairPlay&nbsp;Server&nbsp;Playback&nbsp;Context |<sub></sub>|
|`flac` |Free&nbsp;Lossless&nbsp;Audio&nbsp;Codec&nbsp;file |<sub>`flac_metadatablocks` `flac_frame`</sub>|
|[`flac_frame`](#flac_frame) |FLAC&nbsp;frame |<sub></sub>|
|`flac_metadatablock` |FLAC&nbsp;metadatablock |<sub>`flac_streaminfo` `flac_picture` `vorbis_comment`</sub>|
|`flac_metadatablocks` |FLAC&nbsp;metadatablocks |<sub>`flac_metadatablock`</sub>|
|`flac_picture` |FLAC&nbsp;metadatablock&nbsp;picture |<sub>`image`</sub>|
|`flac_streaminfo` |FLAC&nbsp;streaminfo |<sub></sub>|
|`gif` |Graphics&nbsp;Interchange&nbsp;Format |<sub></sub>|
|`gzip` |gzip&nbsp;compression |<sub>`probe`</sub>|
|`hevc_annexb` |H.265/HEVC&nbsp;Annex&nbsp;B |<sub>`hevc_nalu`</sub>|
|[`hevc_au`](#hevc_au) |H.265/HEVC&nbsp;Access&nbsp;Unit |<sub>`hevc_nalu`</sub>|
|`hevc_dcr` |H.265/HEVC&nbsp;Decoder&nbsp;Configuration&nbsp;Record |<sub>`hevc_nalu`</sub>|
|`hevc_nalu` |H.265/HEVC&nbsp;Network&nbsp;Access&nbsp;Layer&nbsp;Unit |<sub>`hevc_vps` `hevc_pps` `hevc_sps`</sub>|
|`hevc_pps` |H.265/HEVC&nbsp;Picture&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`hevc_sps` |H.265/HEVC&nbsp;Sequence&nbsp;Parameter&nbsp;Set |<sub></sub>|
|`hevc_vps` |H.265/HEVC&nbsp;Video&nbsp;Parameter&nbsp;Set |<sub></sub>|
|[`html`](#html) |HyperText&nbsp;Markup&nbsp;Language |<sub></sub>|
|`icc_profile` |International&nbsp;Color&nbsp;Consortium&nbsp;profile |<sub></sub>|
|`icmp` |Internet&nbsp;Control&nbsp;Message&nbsp;Protocol |<sub></sub>|
|`icmpv6` |Internet&nbsp;Control&nbsp;Message&nbsp;Protocol&nbsp;v6 |<sub></sub>|
|`id3v1` |ID3v1&nbsp;metadata |<sub></sub>|
|`id3v11` |ID3v1.1&nbsp;metadata |<sub></sub>|
|`id3v2` |ID3v2&nbsp;metadata |<sub>`image`</sub>|
|`ipv4_packet` |Internet&nbsp;protocol&nbsp;v4&nbsp;packet |<sub>`ip_packet`</sub>|
|`ipv6_packet` |Internet&nbsp;protocol&nbsp;v6&nbsp;packet |<sub>`ip_packet`</sub>|
|`jpeg` |Joint&nbsp;Photographic&nbsp;Experts&nbsp;Group&nbsp;file |<sub>`exif` `icc_profile`</sub>|
|`json` |JavaScript&nbsp;Object&nbsp;Notation |<sub></sub>|
|`jsonl` |JavaScript&nbsp;Object&nbsp;Notation&nbsp;Lines |<sub></sub>|
|[`macho`](#macho) |Mach-O&nbsp;macOS&nbsp;executable |<sub></sub>|
|`macho_fat` |Fat&nbsp;Mach-O&nbsp;macOS&nbsp;executable&nbsp;(multi-architecture) |<sub>`macho`</sub>|
|[`markdown`](#markdown) |Markdown |<sub></sub>|
|[`matroska`](#matroska) |Matroska&nbsp;file |<sub>`aac_frame` `av1_ccr` `av1_frame` `avc_au` `avc_dcr` `flac_frame` `flac_metadatablocks` `hevc_au` `hevc_dcr` `image` `mp3_frame` `mpeg_asc` `mpeg_pes_packet` `mpeg_spu` `opus_packet` `vorbis_packet` `vp8_frame` `vp9_cfm` `vp9_frame`</sub>|
|[`mp3`](#mp3) |MP3&nbsp;file |<sub>`id3v2` `id3v1` `id3v11` `apev2` `mp3_frame`</sub>|
|`mp3_frame` |MPEG&nbsp;audio&nbsp;layer&nbsp;3&nbsp;frame |<sub>`mp3_frame_tags`</sub>|
|`mp3_frame_vbri` |MP3&nbsp;frame&nbsp;Fraunhofer&nbsp;encoder&nbsp;variable&nbsp;bitrate&nbsp;tag |<sub></sub>|
|`mp3_frame_xing` |MP3&nbsp;frame&nbsp;Xing/Info&nbsp;tag |<sub></sub>|
|[`mp4`](#mp4) |ISOBMFF,&nbsp;QuickTime&nbsp;and&nbsp;similar |<sub>`aac_frame` `av1_ccr` `av1_frame` `avc_au` `avc_dcr` `flac_frame` `flac_metadatablocks` `hevc_au` `hevc_dcr` `icc_profile` `id3v2` `image` `jpeg` `mp3_frame` `mpeg_es` `mpeg_pes_packet` `opus_packet` `png` `prores_frame` `protobuf_widevine` `pssh_playready` `vorbis_packet` `vp9_frame` `vpx_ccr`</sub>|
|`mpeg_asc` |MPEG-4&nbsp;Audio&nbsp;Specific&nbsp;Config |<sub></sub>|
|`mpeg_es` |MPEG&nbsp;Elementary&nbsp;Stream |<sub>`mpeg_asc` `vorbis_packet`</sub>|
|`mpeg_pes` |MPEG&nbsp;Packetized&nbsp;elementary&nbsp;stream |<sub>`mpeg_pes_packet` `mpeg_spu`</sub>|
|`mpeg_pes_packet` |MPEG&nbsp;Packetized&nbsp;elementary&nbsp;stream&nbsp;packet |<sub></sub>|
|`mpeg_spu` |Sub&nbsp;Picture&nbsp;Unit&nbsp;(DVD&nbsp;subtitle) |<sub></sub>|
|`mpeg_ts` |MPEG&nbsp;Transport&nbsp;Stream |<sub></sub>|
|[`msgpack`](#msgpack) |MessagePack |<sub></sub>|
|`ogg` |OGG&nbsp;file |<sub>`ogg_page` `vorbis_packet` `opus_packet` `flac_metadatablock` `flac_frame`</sub>|
|`ogg_page` |OGG&nbsp;page |<sub></sub>|
|`opus_packet` |Opus&nbsp;packet |<sub>`vorbis_comment`</sub>|
|[`pcap`](#pcap) |PCAP&nbsp;packet&nbsp;capture |<sub>`link_frame` `tcp_stream` `ipv4_packet`</sub>|
|`pcapng` |PCAPNG&nbsp;packet&nbsp;capture |<sub>`link_frame` `tcp_stream` `ipv4_packet`</sub>|
|`png` |Portable&nbsp;Network&nbsp;Graphics&nbsp;file |<sub>`icc_profile` `exif`</sub>|
|`prores_frame` |Apple&nbsp;ProRes&nbsp;frame |<sub></sub>|
|[`protobuf`](#protobuf) |Protobuf |<sub></sub>|
|`protobuf_widevine` |Widevine&nbsp;protobuf |<sub>`protobuf`</sub>|
|`pssh_playready` |PlayReady&nbsp;PSSH |<sub></sub>|
|[`rtmp`](#rtmp) |Real-Time&nbsp;Messaging&nbsp;Protocol |<sub>`amf0` `mpeg_asc`</sub>|
|`sll2_packet` |Linux&nbsp;cooked&nbsp;capture&nbsp;encapsulation&nbsp;v2 |<sub>`inet_packet`</sub>|
|`sll_packet` |Linux&nbsp;cooked&nbsp;capture&nbsp;encapsulation |<sub>`inet_packet`</sub>|
|`tar` |Tar&nbsp;archive |<sub>`probe`</sub>|
|`tcp_segment` |Transmission&nbsp;control&nbsp;protocol&nbsp;segment |<sub></sub>|
|`tiff` |Tag&nbsp;Image&nbsp;File&nbsp;Format |<sub>`icc_profile`</sub>|
|[`tls`](#tls) |Transport&nbsp;layer&nbsp;security |<sub>`asn1_ber`</sub>|
|`toml` |Tom's&nbsp;Obvious,&nbsp;Minimal&nbsp;Language |<sub></sub>|
|[`tzif`](#tzif) |Time&nbsp;Zone&nbsp;Information&nbsp;Format |<sub></sub>|
|`udp_datagram` |User&nbsp;datagram&nbsp;protocol |<sub>`udp_payload`</sub>|
|`vorbis_comment` |Vorbis&nbsp;comment |<sub>`flac_picture`</sub>|
|`vorbis_packet` |Vorbis&nbsp;packet |<sub>`vorbis_comment`</sub>|
|`vp8_frame` |VP8&nbsp;frame |<sub></sub>|
|`vp9_cfm` |VP9&nbsp;Codec&nbsp;Feature&nbsp;Metadata |<sub></sub>|
|`vp9_frame` |VP9&nbsp;frame |<sub></sub>|
|`vpx_ccr` |VPX&nbsp;Codec&nbsp;Configuration&nbsp;Record |<sub></sub>|
|[`wasm`](#wasm) |WebAssembly&nbsp;Binary&nbsp;Format |<sub></sub>|
|`wav` |WAV&nbsp;file |<sub>`id3v2` `id3v1` `id3v11`</sub>|
|`webp` |WebP&nbsp;image |<sub>`vp8_frame`</sub>|
|[`xml`](#xml) |Extensible&nbsp;Markup&nbsp;Language |<sub></sub>|
|`yaml` |YAML&nbsp;Ain't&nbsp;Markup&nbsp;Language |<sub></sub>|
|[`zip`](#zip) |ZIP&nbsp;archive |<sub>`probe`</sub>|
|`image` |Group |<sub>`gif` `jpeg` `mp4` `png` `tiff` `webp`</sub>|
|`inet_packet` |Group |<sub>`ipv4_packet` `ipv6_packet`</sub>|
|`ip_packet` |Group |<sub>`icmp` `icmpv6` `tcp_segment` `udp_datagram`</sub>|
|`link_frame` |Group |<sub>`bsd_loopback_frame` `ether8023_frame` `ipv4_packet` `ipv6_packet` `sll2_packet` `sll_packet`</sub>|
|`mp3_frame_tags` |Group |<sub>`mp3_frame_vbri` `mp3_frame_xing`</sub>|
|`probe` |Group |<sub>`adts` `aiff` `apple_bookmark` `ar` `avi` `avro_ocf` `bitcoin_blkdat` `bplist` `bzip2` `elf` `flac` `gif` `gzip` `jpeg` `json` `jsonl` `macho` `macho_fat` `matroska` `mp3` `mp4` `mpeg_ts` `ogg` `pcap` `pcapng` `png` `tar` `tiff` `toml` `tzif` `wasm` `wav` `webp` `xml` `yaml` `zip`</sub>|
|`tcp_stream` |Group |<sub>`dns_tcp` `rtmp` `tls`</sub>|
|`udp_payload` |Group |<sub>`dns`</sub>|
[#]: sh-end
@ -157,6 +163,47 @@ Decode value as aac_frame
... | aac_frame({object_type:1})
```
## apple_bookmark
Apple's `bookmarkData` format is used to encode information that can be resolved
into a `URL` object for a file even if the user moves or renames it. Can also
contain security scoping information for App Sandbox support.
These `bookmarkData` blobs are often found endcoded in data fields of Binary
Property Lists. Notable examples include:
- `com.apple.finder.plist` - contains an `FXRecentFolders` value, which is an
array of ten objects, each of which consists of a `name` and `file-bookmark`
field, which is a `bookmarkData` object for each recently accessed folder
location.
- `com.apple.LSSharedFileList.RecentApplications.sfl2` - `sfl2` files are
actually `plist` files of the `NSKeyedArchiver` format. They can be parsed the
same as `plist` files, but they have a more complicated tree-like structure
than would typically be found, which can make locating and retrieving specific
values difficult, even once it has been converted to a JSON representation.
For more information about these types of files, see Sarah Edwards' excellent
research on the subject (link in references).
`fq`'s `grep_by` function can be used to recursively descend through the decoded
tree, probing for and selecting any `bookmark` blobs, then converting them to
readable JSON with `torepr`:
```
fq 'grep_by(.type=="data" and .value[0:4] == "book") | .value | apple_bookmark |
torepr' <sfl2 file>
```
### Authors
- David McDonald
[@dgmcdona](https://github.com/dgmcdona)
[@river_rat_504](https://twitter.com/river_rat_504)
### References
- https://developer.apple.com/documentation/foundation/url/2143023-bookmarkdata
- https://mac-alias.readthedocs.io/en/latest/bookmark_fmt.html
- https://www.mac4n6.com/blog/2016/1/1/manual-analysis-of-nskeyedarchiver-formatted-plist-files-a-review-of-the-new-os-x-1011-recent-items
- https://michaellynn.github.io/2015/10/24/apples-bookmarkdata-exposed/
## asn1_ber
Supports decoding BER, CER and DER (X.690).
@ -168,7 +215,7 @@ Supports decoding BER, CER and DER (X.690).
### Can be used to decode certificates etc
```sh
$ fq -d bytes 'frompem | asn1_ber | d' cert.pem
$ fq -d bytes 'from_pem | asn1_ber | d' cert.pem
```
### Can decode nested values
@ -215,7 +262,7 @@ Decode value as avc_au
|Name |Default|Description|
|- |- |-|
|`decode_samples`|true |Decode supported media samples|
|`decode_samples`|true |Decode samples|
### Examples
@ -345,6 +392,8 @@ $ fq 'torepr.SomeTimeStamp | todescription' Info.plist
### Get JSON representation
`bplist` files can be converted to a JSON representation using the `torepr` filter:
```sh
$ fq torepr com.apple.UIAutomation.plist
{
@ -352,6 +401,28 @@ $ fq torepr com.apple.UIAutomation.plist
}
```
### Decoding NSKeyedArchiver serialized objects
A common way that Swift and Objective-C libraries on macOS serialize objects
is through the NSKeyedArchiver API, which flattens objects into a list of elements
and class descriptions that are reconstructed into an object graph using CFUID
elements in the property list. `fq` includes a function, `from_ns_keyed_archiver`,
which will rebuild this object graph into a friendly representation.
If no parameters are supplied, it will assume that there is a CFUID located at
`."$top".root` that specifies the root from which decoding should occur. If this
is not present, an error will be produced, asking the user to specify a root
object in the `.$objects` list from which to decode.
The following examples show how this might be used (in this case, within the `fq` REPL):
```
# Assume $top.root is present
bplist> from_ns_keyed_archiver
# Specify optional root
bplist> from_ns_keyed_archiver(1)
```
### Authors
- David McDonald
[@dgmcdona](https://github.com/dgmcdona)
@ -363,6 +434,10 @@ $ fq torepr com.apple.UIAutomation.plist
## bson
### Limitations:
- The decimal128 type is not supported for decoding, will just be treated as binary
### Convert represented value to JSON
```
@ -441,7 +516,7 @@ Decode value as csv
### TSV to CSV
```sh
$ fq -d csv -o comma="\t" tocsv file.tsv
$ fq -d csv -o comma="\t" to_csv file.tsv
```
### Convert rows to objects based on header row
@ -516,7 +591,7 @@ HTML is decoded in HTML5 mode and will always include `<html>`, `<body>` and `<h
See xml format for more examples and how to preserve element order and how to encode to xml.
There is no `tohtml` function, see `toxml` instead.
There is no `to_html` function, see `to_xml` instead.
### Element as object
@ -599,6 +674,24 @@ $ fq -d markdown '[.. | select(.type=="heading" and .level<=2)?.children[0]]' fi
```
## matroska
### Options
|Name |Default|Description|
|- |- |-|
|`decode_samples`|true |Decode samples|
### Examples
Decode file using matroska options
```
$ fq -d matroska -o decode_samples=true . file
```
Decode value as matroska
```
... | matroska({decode_samples:true})
```
### Lookup element using path
```sh
@ -626,17 +719,18 @@ $ fq 'grep_by(.id == "Tracks") | matroska_path' file.mkv
|- |- |-|
|`max_sync_seek` |32768 |Max byte distance to next sync|
|`max_unique_header_configs`|5 |Max number of unique frame header configs allowed|
|`max_unknown` |50 |Max percent (0-100) unknown bits|
### Examples
Decode file using mp3 options
```
$ fq -d mp3 -o max_sync_seek=32768 -o max_unique_header_configs=5 . file
$ fq -d mp3 -o max_sync_seek=32768 -o max_unique_header_configs=5 -o max_unknown=50 . file
```
Decode value as mp3
```
... | mp3({max_sync_seek:32768,max_unique_header_configs:5})
... | mp3({max_sync_seek:32768,max_unique_header_configs:5,max_unknown:50})
```
## mp4
@ -646,7 +740,7 @@ Decode value as mp3
|Name |Default|Description|
|- |- |-|
|`allow_truncated`|false |Allow box to be truncated|
|`decode_samples` |true |Decode supported media samples|
|`decode_samples` |true |Decode samples|
### Examples
@ -660,6 +754,31 @@ Decode value as mp4
... | mp4({allow_truncated:false,decode_samples:true})
```
### Speed up decoding by not decoding samples
```sh
# manually decode first sample as a aac_frame
$ fq -o decode_samples=false '.tracks[0].samples[0] | aac_frame | d' file.mp4
```
### Entries for first edit list as values
```sh
$ fq 'first(grep_by(.type=="elst").entries) | tovalue' file.mp4
```
### Whole box tree as JSON (exclude mdat data and tracks)
```sh
$ fq 'del(.tracks) | grep_by(.type=="mdat").data = "<excluded>" | tovalue' file.mp4
```
### Force decode a single box
```sh
$ fq -n '"AAAAHGVsc3QAAAAAAAAAAQAAADIAAAQAAAEAAA==" | from_base64 | mp4({force:true}) | d'
```
### Lookup mp4 box using a mp4 box path.
```sh
@ -674,25 +793,6 @@ $ fq 'mp4_path(".moov.trak[1]")' file.mp4
$ fq 'grep_by(.type == "trak") | mp4_path' file.mp4
```
### Force decode a single box
```sh
$ fq -n '"AAAAHGVsc3QAAAAAAAAAAQAAADIAAAQAAAEAAA==" | frombase64 | mp4({force:true}) | d'
```
### Speed up decoding by not decoding samples
```sh
# manually decode first sample as a aac_frame
$ fq -o decode_samples=false '.tracks[0].samples[0] | aac_frame | d' file.mp4
```
### Entries for first edit list as values
```sh
$ fq 'first(grep_by(.type=="elst").entries) | tovalue' file.mp4
```
### References
- [ISO/IEC base media file format (MPEG-4 Part 12)](https://en.wikipedia.org/wiki/ISO/IEC_base_media_file_format)
@ -745,6 +845,173 @@ fq '.tcp_connections[] | select(.server.port=="rtmp") | d' file.cap
- https://rtmp.veriskope.com/docs/spec/
- https://rtmp.veriskope.com/pdf/video_file_format_spec_v10.pdf
## tls
### Options
|Name |Default|Description|
|- |- |-|
|`keylog`| |NSS Key Log content|
### Examples
Decode file using tls options
```
$ fq -d tls -o keylog="" . file
```
Decode value as tls
```
... | tls({keylog:""})
```
Supports decoding of most standard records, messages and extensions. Can also decrypt most standard cipher suits in a PCAP with traffic in both directions if a NSS key log is provided.
### Decode and decrypt provding a PCAP and key log
Write traffic to a PCAP file:
```sh
$ tcpdump -i <iface> -w traffic.pcap
```
Make sure your curl TLS backend support `SSLKEYLOGFILE` and do:
```sh
$ SSLKEYLOGFILE=traffic.keylog curl --tls-max 1.2 https://host/path
```
Decode, decrypt and query. Uses `keylog=@<path>` to read option value from keylog file:
```sh
# decode and show whole tree
$ fq -o keylog=@traffic.keylog d traffic.pcap
# write unencrypted server response to a file.
# first .stream is the TCP stream, second .stream is TLS application data stream
#
# first TCP connections:
$ fq -o keylog=@traffic.keylog '.tcp_connections[0].server.stream.stream | tobytes' traffic.pcap > data
# first TLS connection:
$ fq -o keylog=@traffic.keylog 'first(grep_by(.server.stream | format == "tls")).server.stream.stream | tobytes' > data
```
### Supported cipher suites for decryption
`TLS_DH_ANON_EXPORT_WITH_DES40_CBC_SHA`,
`TLS_DH_ANON_EXPORT_WITH_RC4_40_MD5`,
`TLS_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA`,
`TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA`,
`TLS_DHE_DSS_WITH_AES_128_CBC_SHA`,
`TLS_DHE_DSS_WITH_AES_128_CBC_SHA256`,
`TLS_DHE_DSS_WITH_AES_128_GCM_SHA256`,
`TLS_DHE_DSS_WITH_AES_256_CBC_SHA`,
`TLS_DHE_DSS_WITH_AES_256_CBC_SHA256`,
`TLS_DHE_DSS_WITH_AES_256_GCM_SHA384`,
`TLS_DHE_DSS_WITH_DES_CBC_SHA`,
`TLS_DHE_DSS_WITH_RC4_128_SHA`,
`TLS_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA`,
`TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA`,
`TLS_DHE_RSA_WITH_AES_128_CBC_SHA`,
`TLS_DHE_RSA_WITH_AES_128_CBC_SHA256`,
`TLS_DHE_RSA_WITH_AES_128_GCM_SHA256`,
`TLS_DHE_RSA_WITH_AES_256_CBC_SHA`,
`TLS_DHE_RSA_WITH_AES_256_CBC_SHA256`,
`TLS_DHE_RSA_WITH_AES_256_GCM_SHA384`,
`TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256`,
`TLS_DHE_RSA_WITH_DES_CBC_SHA`,
`TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA`,
`TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA`,
`TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256`,
`TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256`,
`TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA`,
`TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384`,
`TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384`,
`TLS_ECDH_ECDSA_WITH_RC4_128_SHA`,
`TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA`,
`TLS_ECDH_RSA_WITH_AES_128_CBC_SHA`,
`TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256`,
`TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256`,
`TLS_ECDH_RSA_WITH_AES_256_CBC_SHA`,
`TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384`,
`TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384`,
`TLS_ECDH_RSA_WITH_RC4_128_SHA`,
`TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA`,
`TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA`,
`TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA`,
`TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256`,
`TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256`,
`TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA`,
`TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA`,
`TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384`,
`TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384eadAESGCM`,
`TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256`,
`TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305`,
`TLS_ECDHE_ECDSA_WITH_RC4_128_SHA`,
`TLS_ECDHE_ECDSA_WITH_RC4_128_SHA`,
`TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA`,
`TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256`,
`TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA`,
`TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA`,
`TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA`,
`TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA`,
`TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA`,
`TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256`,
`TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256`,
`TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA`,
`TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA`,
`TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384`,
`TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384`,
`TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256`,
`TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305`,
`TLS_ECDHE_RSA_WITH_RC4_128_SHA`,
`TLS_ECDHE_RSA_WITH_RC4_128_SHA`,
`TLS_PSK_WITH_AES_128_CBC_SHA`,
`TLS_PSK_WITH_AES_256_CBC_SHA`,
`TLS_PSK_WITH_RC4_128_SHA`,
`TLS_RSA_EXPORT_WITH_DES40_CBC_SHA`,
`TLS_RSA_EXPORT_WITH_RC4_40_MD5`,
`TLS_RSA_WITH_3DES_EDE_CBC_SHA`,
`TLS_RSA_WITH_3DES_EDE_CBC_SHA`,
`TLS_RSA_WITH_AES_128_CBC_SHA`,
`TLS_RSA_WITH_AES_128_CBC_SHA`,
`TLS_RSA_WITH_AES_128_CBC_SHA256`,
`TLS_RSA_WITH_AES_128_CBC_SHA256`,
`TLS_RSA_WITH_AES_128_GCM_SHA256`,
`TLS_RSA_WITH_AES_128_GCM_SHA256`,
`TLS_RSA_WITH_AES_256_CBC_SHA`,
`TLS_RSA_WITH_AES_256_CBC_SHA`,
`TLS_RSA_WITH_AES_256_CBC_SHA256`,
`TLS_RSA_WITH_AES_256_GCM_SHA384`,
`TLS_RSA_WITH_AES_256_GCM_SHA384`,
`TLS_RSA_WITH_DES_CBC_SHA`,
`TLS_RSA_WITH_RC4_128_MD5`,
`TLS_RSA_WITH_RC4_128_SHA`,
`TLS_RSA_WITH_RC4_128_SHA`
### References
- [RFC 5246: The Transport Layer Security (TLS) Protocol](https://www.rfc-editor.org/rfc/rfc5246)
- [RFC 6101: The Secure Sockets Layer (SSL) Protocol Version 3.0](https://www.rfc-editor.org/rfc/rfc)
## tzif
### Get last transition time
```sh
fq '.v2plusdatablock.transition_times[-1] | tovalue' tziffile
```
### Count leap second records
```sh
fq '.v2plusdatablock.leap_second_records | length' tziffile
```
### Authors
- Takashi Oguma
[@bitbears-dev](https://github.com/bitbears-dev)
[@0xb17bea125](https://twitter.com/0xb17bea125)
### References
- https://datatracker.ietf.org/doc/html/rfc8536
## wasm
### Count opcode usage
@ -792,7 +1059,7 @@ Which variant to use depends a bit what you want to do. The object variant might
to query for a specific value but array might be easier to use to generate xml or to query
after all elements of some kind etc.
Encoding is done using the `toxml` function and it will figure what variant that is used based on the input value.
Encoding is done using the `to_xml` function and it will figure what variant that is used based on the input value.
Is has two optional options `indent` and `attribute_prefix`.
### Elements as object
@ -835,7 +1102,7 @@ $ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq '.a.c["#text"]'
"ccc"
# decode to object and encode to xml
$ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq -r -d xml -o seq=true 'toxml({indent:2})'
$ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq -r -d xml -o seq=true 'to_xml({indent:2})'
<a>
<b></b>
<b>bbb</b>
@ -878,7 +1145,7 @@ $ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq -d xml -o array=true
]
# decode to array and encode to xml
$ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq -r -d xml -o array=true -o seq=true 'toxml({indent:2})'
$ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq -r -d xml -o array=true -o seq=true 'to_xml({indent:2})'
<a>
<b></b>
<b>bbb</b>

2132
doc/formats.svg
View File

File diff suppressed because it is too large Load Diff
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 137 KiB

After

Width:  |  Height:  |  Size: 142 KiB

2
doc/formats_diagram.jq
View File

@ -1,7 +1,7 @@
#!/usr/bin/env fq -rnf
def color:
tomd5 | [.[range(3)]] | map(band(.; 0x7f)+60 | toradix(16) | "0"[length:]+.) | join("");
to_md5 | [.[range(3)]] | map(band(.; 0x7f)+60 | to_radix(16) | "0"[length:]+.) | join("");
def _formats_dot:
def _record($title; $fields):

6
doc/presentations/bts2022/avc_sps_hdr_params.go
View File

@ -1,12 +1,12 @@
func avcHdrParameters(d *decode.D) {
cpbCnt := d.FieldUFn("cpb_cnt", uEV, scalar.UAdd(1))
cpbCnt := d.FieldUintFn("cpb_cnt", uEV, scalar.UAdd(1))
d.FieldU4("bit_rate_scale")
d.FieldU4("cpb_size_scale")
d.FieldArray("sched_sels", func(d *decode.D) {
for i := uint64(0); i < cpbCnt; i++ {
d.FieldStruct("sched_sel", func(d *decode.D) {
d.FieldUFn("bit_rate_value", uEV, scalar.UAdd(1))
d.FieldUFn("cpb_size_value", uEV, scalar.UAdd(1))
d.FieldUintFn("bit_rate_value", uEV, scalar.UAdd(1))
d.FieldUintFn("cpb_size_value", uEV, scalar.UAdd(1))
d.FieldBool("cbr_flag")
})
}

2
doc/presentations/bts2022/mp3.go
View File

@ -1,4 +1,4 @@
func decode(d *decode.D, _ any) any {
func decode(d *decode.D) any {
d.FieldArray("headers", func(d *decode.D) {
for !d.End() {
d.TryFieldFormat("header", headerGroup)

387
doc/usage.md
View File

@ -9,6 +9,7 @@ cat file | fq
fq . < file
fq . *.png *.mp3
fq '.frames[0]' *.mp3
fq '.frames[-1] | tobytes' file.mp3 > last_frame
```
### Common usages
@ -20,18 +21,58 @@ fq d file
fq display file
# display all bytes for each value
fq dd file
# same as
fq 'd({display_bytes: 0})' file
# display 200 bytes for each value
fq 'd({display_bytes: 200})' file
# recursively display decode tree without truncating
fq da file
# same as
fq 'd({array_truncate: 0})' file
# display a specific decode tree one level
fq '.path[1].to.value' file
# display a specific decode tree all levels
fq '.path[1].to.value | d' file
fq '.path[1].to.value | dd' file
fq '.path[1].to.value | da' file
# recursively and verbosely display decode tree
fq dv file
# same as
fq 'd({verbose: true})' file
# JSON repersenation for whole file
# JSON representation for whole file
fq tovalue file
# or use -V (--value-output) that does tovalue automatically
fq -V . file
# or -Vr if the value is a string and you want a "raw" string
fq -Vr .path.to.string file
# JSON but raw bit fields truncated
fq -o bits_format=truncate tovalue file
# JSON but raw bit fields as md5 hex string
fq -o bits_format=md5 tovalue file
# look up a path
fq '.some[1].path' file
# look up a path and output JSON
fq -V '.some[1].path' file
# can be a query that outputs multiple values
# this outputs first and last value in .same array and .path, three values in total
fq -V '.some[0,-1], .path' file
# grep whole tree by value
fq 'grep("^prefix")' file
fq 'grep(123)' file
# grep whole tree by condition
fq 'grep_by(. >= 100 and . =< 100)' file
# recursively look for values fulfilling some condition
fq '.. | select(.type=="trak")?' file
fq 'grep_by(.type=="trak")' file
# grep_by(f) is alias for .. | select(f)?, that is: recuse, select and ignore errors
# recursively look for decode value roots for a format
fq '.. | select(format=="jpeg")' file
@ -42,24 +83,45 @@ fq 'grep_by(format=="jpeg")' file
fq 'first(.. | select(format=="jpeg"))' file
fq 'first(grep_by(format=="jpeg"))' file
# recursively look for objects fullfilling condition
fq '.. | select(.type=="trak")?' file
fq 'grep_by(.type=="trak")' file
# grep whole tree
fq 'grep("^prefix")' file
fq 'grep(123)' file
fq 'grep_by(. >= 100 and . =< 100)' file
# decode file as mp4 and return a result even if there are some errors
fq -d mp4 file.mp4
# decode file as mp4 and also ignore validity assertions
fq -o force=true -d mp4 file.mp4
```
### CLI arguments
Most of jq's CLI arguments work with fq. But here are some additional ones specific to fq:
#### Decode format `--decode`, `-d NAME`
Force format to decode instead of probing.
`NAME` is a name of a format, ex `-d mp4`, see `-h formats` for list of formats.
#### Interactive REPL `--repl`,`-i`
Start interactive REPL.
Can be used with no input, one and multiple inputs, for example just `fq -i ` starts a REPL with `null` input, `fq -i 123` with the number 123 as input, `fq -i . a b` with two files as input. This also works with `--slurp`. In the REPL it is also possible to start a sub-REPLs by ending a query with `<query> | repl`, use ctrl-D to exit the sub-REPL. The sub-REPL will evaluate separately on each output from the query it was started. Use `[<query>] | repl` if you want to "slurp" into an array.
#### Set option `--options`,`-o KEY=VALUE|@PATH`
`KEY` is name of option
`VALUE` will be interpreted as a JSON value if possible otherwise a string, ex -o `name=abc` and `-o name='"abc"'` is the same.
`@PATH` will read string from file at `PATH`.
Specify a global option or a format option, ex: `-o decode_samples=false` would for some container decoders like `mp4` and `matroska` disable decoding of samples.
#### Value output `--value-output`, `-V`
Output JSON value instead of decode tree. Use `-Vr` if you want raw string (no quotes).
### Display output
`display` or `d` is the main function for displying values and is also the function that will be used if no other output function is explicitly used. If its input is a decode value it will output a dump and tree structure or otherwise it will output as JSON.
`display` or `d` is the main function for displaying values and is also the function that will be used if no other output function is explicitly used. If its input is a decode value it will output a dump and tree structure or otherwise it will output as JSON.
Below demonstrates some usages:
@ -67,17 +129,17 @@ First and second example does the same thing, inputs `"hello"` to `display`.
![fq demo](display_json.svg)
In the next few examples we select out the first "edit list" box in an mp4 file, it's a list of which part of media track to be included during playback, and displays it in various ways.
In the next few examples we select out the first "edit list" box in an mp4 file, it's a list of which part of the media track to be included during playback, and displays it in various ways.
Default if not explicitly used `display` will only show the root level:
![fq demo](display_decode_value.svg)
First row shows ruler with byte offset into the line and jq path for the value.
First row shows a ruler with byte offset into the line and jq path for the value.
The columns are:
- Start address for the line. For example we see that `type` starts at `0xd60`+`0x09`.
- Hex repersenation of input bits for value. Will show the whole byte even if the value only partially uses bits from it.
- Hex representation of input bits for value. Will show the whole byte even if the value only partially uses bits from it.
- ASCII representation of input bits for value. Will show the whole byte even if the value only partially uses bits from it.
- Tree structure of decoded value, symbolic value and description.
@ -96,7 +158,7 @@ Same but verbose `dv`:
In verbose mode bit ranges and array element names as shown.
Bit range uses `bytes.bits` notation. For example `type` start at byte `0xd69` bit `0` (left out if zero) and ends at `0xd6c` bit `7` (inclusive) and have byte size of `4`.
Bit range uses `bytes.bits` notation. For example `type` starts at byte `0xd69` bit `0` (left out if zero) and ends at `0xd6c` bit `7` (inclusive) and have byte size of `4`.
There are also some other `display` aliases:
- `da` same as `display({array_truncate: 0})` which will not truncate long arrays.
@ -179,15 +241,15 @@ fq '.frames[0:10] | map(tobytesrange.start)' file.mp3
#### Decode at range
```sh
# decode byte range 100 to end
# decode byte range 100 to end as mp3_frame
fq -d bytes '.[100:] | mp3_frame | d' file.mp3
# decode byte range 10 bytes into .somefield and preseve relative position in file
# decode byte range 10 bytes from .somefield and preserve relative position in file
fq '.somefield | tobytesrange[10:] | mp3_frame | d' file.mp3
```
#### Show AVC SPS difference between two mp4 files
`-n` tells fq to not have an implicit `input`, `f` is function to select out some interesting value, call `diff` with two arguments,
`-n` tells fq to not have an implicit `input`, `f` is a function to select out some interesting value, call `diff` with two arguments,
decoded value for `a.mp4` and `b.mp4` filtered thru `f`.
```sh
@ -196,7 +258,7 @@ fq -n 'def f: .. | select(format=="avc_sps"); diff(input|f; input|f)' a.mp4 b.mp
#### Extract first JPEG found in file
Recursively look for first value that is a `jpeg` decode value root. Use `tobytes` to get bytes for value. Redirect bytes to a file.
Recursively look for the first value that is a `jpeg` decode value root. Use `tobytes` to get bytes for value. Redirect bytes to a file.
```sh
fq 'first(.. | select(format=="jpeg")) | tobytes' file > file.jpeg
@ -204,7 +266,7 @@ fq 'first(.. | select(format=="jpeg")) | tobytes' file > file.jpeg
#### Sample size histogram
Recursively look for a all sample size boxes "stsz" and use `?` to ignore errors when doing `.type` on arrays etc. Save reference to box, count unique values, save the max, output the path to the box and output a historgram scaled to 0-100.
Recursively look for a all sample size boxes "stsz" and use `?` to ignore errors when doing `.type` on arrays etc. Save reference to box, count unique values, save the max, output the path to the box and output a histogram scaled to 0-100.
```sh
fq '.. | select(.type=="stsz")? as $stsz | .entries | count | max_by(.[1])[1] as $m | ($stsz | topath | path_to_expr), (.[] | "\(.[0]): \((100*.[1]/$m)*"=") \(.[1])") | println' file.mp4
@ -261,22 +323,27 @@ To get the most out of fq it's recommended to learn more about jq, here are some
- [jq wiki: Pitfalls](https://github.com/stedolan/jq/wiki/How-to:-Avoid-Pitfalls)
- [FAQ](https://github.com/stedolan/jq/wiki/FAQ)
Common beginner gotcha are:
For a more convenient jq experience these might be interesting:
- [jq Dash docset](https://github.com/wader/jq-dash-docset)
- [vscode-jq](https://github.com/wader/vscode-jq)
- [jq-lsp](https://github.com/wader/jq-lsp)
Common beginner gotchas are:
- jq's use of `;` and `,`. jq uses `;` as argument separator
and `,` as output separator. To call a function `f` with two arguments use `f(1; 2)`. If you do `f(1, 2)` you pass a
single argument `1, 2` (a lambda expression that output `1` and then output `2`) to `f`.
single argument `1, 2` (a lambda expression that outputs `1` and then outputs `2`) to `f`.
- Expressions can return or "output" zero or more values. This is how loops, foreach etc is
achieved.
- Expressions have one implicit input and output value. This how pipelines like `1 | . * 2` work.
## Types specific to fq
fq has two additional types compared to jq, decode value and binary. In standard jq expressions they will in most case behave as some standard jq type.
fq has two additional types compared to jq, decode value and binary. In standard jq expressions they will in most cases behave as some standard jq type.
### Decode value
This type is returned by decoders and it used to represent parts of the decoed input. It can act as all standard jq types, object, array, number, string etc.
This type is returned by decoders and it is used to represent parts of the decoded input. It can act as all standard jq types, object, array, number, string etc.
Each decode value has these properties:
- A bit range in the input
@ -298,18 +365,18 @@ Each non-compound decode value has these properties:
- `topath` is the jq path for the decode value
- `torepr` convert decode value to its representation if possible
The value of a decode value is the symbolic value if available and otherwise the actual value. To explicitly access the value use `tovalue`. In most expression this is not needed as it will be done automactically.
The value of a decode value is the symbolic value if available and otherwise the actual value. To explicitly access the value use `tovalue`. In most expressions this is not needed as it will be done automatically.
### Binary
Binaries are raw bits with a unit size, 1 (bits) or 8 (bytes), that can have a non-byte aligned size. Will act as byte padded strings in standard jq expressions.
Use `tobits` and `tobytes` to create them from a decode values, strings, numbers or binary arrays. `tobytes` will if needed zero pad most significant bits to be byte aligned.
Use `tobits` and `tobytes` to create them from decode values, strings, numbers or binary arrays. `tobytes` will, if needed zero pad most significant bits to be byte aligned.
There is also `tobitsrange` and `tobytesrange` which does the same thing but will preserve it's source range when displayed.
There is also `tobitsrange` and `tobytesrange` which does the same thing but will preserve its source range when displayed.
- `"string" | tobytes` produces a binary with UTF8 codepoint bytes.
- `1234 | tobits` produces a binary with the unsigned big-endian integer 1234 with enough bits to represent the number. Use `tobytes` to get the same but with enough bytes to represent the number. This is different to how numbers works inside binary arrays where they are limited to 0-255.
- `1234 | tobits` produces a binary with the unsigned big-endian integer 1234 with enough bits to represent the number. Use `tobytes` to get the same but with enough bytes to represent the number. This is different to how numbers work inside binary arrays where they are limited to 0-255.
- `["abc", 123, ...] | tobytes` produce a binary from a binary array. See [binary array](#binary-array) below.
- `.[index]` access bit or byte at index `index`. Index is in units.
- `[0x12, 0x34, 0x56] | tobytes[1]` is `0x35`
@ -350,13 +417,19 @@ TODO: padding and alignment
## Functions
All decode functions are available in two forms, just `<format>` (like `mp3`) that returns a decode value on error and `from_<format>` which throws error on decode error.
Note that jq sometimes uses the notation `name/0`, `name/1` etc in error messages and documentation which means `<function-name>/<arity>`. Same function names with different arity are treated as separate functions, but are usually related in some way in practice.
### Function added in fq
- All standard library functions from jq
- Adds a few new general functions:
- `print`, `println`, `printerr`, `printerrln` prints to stdout and stderr.
- `group` group values, same as `group_by(.)`.
- `streaks`, `streaks_by(f)` like `group` but groups streaks based on condition.
- `count`, `count_by(f)` like `group` but counts groups lengths.
- `debug(f)` like `debug` but uses arg to produce debug message. `{a: 123} | debug({a}) | ...`.
- `debug(f)` like `debug` but uses arg to produce a debug message. `{a: 123} | debug({a}) | ...`.
- `path_to_expr` from `["key", 1]` to `".key[1]"`.
- `expr_to_path` from `".key[1]"` to `["key", 1]`.
- `diff($a; $b)` produce diff object between two values.
@ -375,7 +448,7 @@ unary uses input and if more than one argument all as arguments ignoring the inp
- `toactual`, `toactual($opts)` actual value (usually the decoded value)
- `tosym`, `tosym($opts)` symbolic value (mapped etc)
- `todescription` description of value
- `torepr` convert decode value into what it reptresents. For example convert msgpack decode value
- `torepr` converts decode value into what it represents. For example convert msgpack decode value
into a value representing its JSON representation.
- All regexp functions work with binary as input and pattern argument with these differences
compared to when using string input:
@ -383,14 +456,14 @@ unary uses input and if more than one argument all as arguments ignoring the inp
- For `capture` the `.string` value is a binary.
- If pattern is a binary it will be matched literally and not as a regexp.
- If pattern is a binary or flags include "b" each input byte will be read as separate code points
- String function are not overloaded to support binary for now as some of them are bahaviours that might be confusing.
- String functions are not overloaded to support binary for now as some of them might have behaviors that might be confusing.
- `explode` is overloaded to work with binary. Will explode into array of the unit of the binary.
end of binary.
instead of possibly multi-byte UTF-8 codepoints. This allows to match raw bytes. Ex: `match("\u00ff"; "b")`
will match the byte `0xff` and not the UTF-8 encoded codepoint for 255, `match("[^\u00ff]"; "b")` will match
all non-`0xff` bytes.
- `grep` functions take 1 or 2 arguments. First is a scalar to match, where a string is
treated as a regexp. A binary will be matches exact bytes. Second argument are regexp
treated as a regexp. A binary will match exact bytes. Second argument are regexp
flags with addition that "b" will treat each byte in the input binary as a code point, this
makes it possible to match exact bytes.
- `grep($v)`, `grep($v; $flags)` recursively match value and binary
@ -399,39 +472,55 @@ unary uses input and if more than one argument all as arguments ignoring the inp
- `fgrep($v)`, `fgrep($v; $flags)` recursively match field name
- `grep_by(f)` recursively match using a filter. Ex: `grep_by(. > 180 and . < 200)`, `first(grep_by(format == "id3v2"))`.
- Binary:
- `tobits` - Transform input to binary with bit as unit, does not preserving source range, will start at zero.
- `tobits` - Transform input to binary with bit as unit, does not preserve source range, will start at zero.
- `tobitsrange` - Transform input to binary with bit as unit, preserves source range if possible.
- `tobytes` - Transform input to binary with byte as unit, does not preserving source range, will start at zero.
- `tobytes` - Transform input to binary with byte as unit, does not preserve source range, will start at zero.
- `tobytesrange` - Transform input binary with byte as unit, preserves source range if possible.
- `.[start:end]`, `.[:end]`, `.[start:]` - Slice binary from start to end preserving source range.
- `.[start:end]`, `.[:end]`, `.[start:]` - Slice binary from start to end preserve source range.
- `open` open file for reading
- All decode function takes a optional option argument. The only option currently is `force` to ignore decoder asserts.
- All decode functions take an optional option argument. The only option currently is `force` to ignore decoder asserts.
For example to decode as mp3 and ignore assets do `mp3({force: true})` or `decode("mp3"; {force: true})`, from command line
you currently have to do `fq -d bytes 'mp3({force: true})' file`.
- `decode`, `decode("<format>")`, `decode("<format>"; $opts)` decode format
- `probe`, `probe($opts)` probe and decode format
- `mp3`, `mp3($opts)`, ..., `<format>`, `<format>($opts)` same as `decode("<format>")`, `decode("<format>"; $opts)` decode as format
- `mp3`, `mp3($opts)`, ..., `<format>`, `<format>($opts)` same as `decode("<format>")`, `decode("<format>"; $opts)` decode as format and return decode value even on decode error.
- `from_mp3`, `from_mp3($opts)`, ..., `from_<format>`, `from_<format>($opts)` same as `decode("<format>")`, `decode("<format>"; $opts)` decode as format but throw error on decode error.
- Display shows hexdump/ASCII/tree for decode values and jq value for other types.
- `d`/`d($opts)` display value and truncate long arrays and binaries
- `da`/`da($opts)` display value and don't truncate arrays
- `dd`/`dd($opts)` display value and don't truncate arrays or binaries
- `dv`/`dv($opts)` verbosely display value and don't truncate arrays but truncate binaries
- `ddv`/`ddv($opts)` verbosely display value and don't truncate arrays or binaries
- `p`/`preview` show preview of field tree
- `hd`/`hexdump` hexdump value
- `repl`/`repl($opts)` nested REPL, must be last in a pipeline. `1 | repl`, can "slurp" outputs. Ex: `1, 2, 3 | repl`, `[1,2,3] | repl({compact: true})`.
- `slurp("<name>")` slurp outputs and save them to `$name`, must be last in pipeline. Will be available as global array `$name`. Ex `1,2,3 | slurp("a")`, `$a[]` same as `spew("a")`.
- `spew`/`spew("<name>")` output previously slurped values. `spew` outputs all slurps as an object, `spew("<name>")` outouts one slurp. Ex: `spew("a")`.
- `slurp("<name>")` slurp outputs and save them to `$name`, must be last in the pipeline. Will be available as a global array `$name`. Ex `1,2,3 | slurp("a")`, `$a[]` same as `spew("a")`.
- `spew`/`spew("<name>")` output previously slurped values. `spew` outputs all slurps as an object, `spew("<name>")` outputs one slurp. Ex: `spew("a")`.
- `paste` read string from stdin until ^D. Useful for pasting text.
- Ex: `paste | frompem | asn1_ber | repl` read from stdin then decode and start a new sub-REPL with result.
- Ex: `paste | from_pem | asn1_ber | repl` read from stdin then decode and start a new sub-REPL with result.
### Naming inconsistencies
jq's naming conversion is a bit inconsistent, some standard library functions are named `tojson` while others `from_entries`. fq follows this tradition a bit by but tries to use snake_case unless there is a good reason.
Here are all the non-snake_case functions added by. Most of them deal with decode and binary values which are new "primitive" types:
- `toactual`
- `tobits`
- `tobitsrange`
- `tobytes`
- `tobytesrange`
- `todescription`
- `topath`
- `torepr`
- `tosym`
- `tovalue`
### Encodings, serializations and hashes
In an addition to binary formats fq also support reading to and from encodings and serialization formats.
In addition to binary formats fq also support reading to and from encodings and serialization formats.
At the moment fq does not have any dedicated argument for serialization formats but raw string input `-R` slurp `-s` and raw string output `-r` can make things easier. The combination `-Rs` will read all inputs into one string (same as jq).
Note that `from*` functions output jq values and `to*` takes jq values as input so in some cases not all information will be properly preserved. For example for the element and attribute order might change and text and comment nodes might move or be merged. [yq](https://github.com/mikefarah/yq) might be a better tool if that is needed.
Note that `from*` functions output jq values and `to*` takes jq values as input so in some cases not all information will be properly preserved. For example, for the element and attribute order might change and text and comment nodes might move or be merged. [yq](https://github.com/mikefarah/yq) might be a better tool if that is needed.
Some example usages:
@ -444,19 +533,19 @@ $ fq '...' file.yml
$ fq -r 'tojson({indent:2})' file.yml
# add token to URL
$ echo -n "https://host.org" | fq -Rsr 'fromurl | .user.username="token" | tourl'
$ echo -n "https://host.org" | fq -Rsr 'from_url | .user.username="token" | tourl'
https://token@host.org
# top 3 hosts in src or href attributes:
# -d to decode as html, can't be probed as html5 parsers always produce some parse tree
# [...] to start collect values into an array
# .. | ."@src"?, ."@href"? | values, recurse and try (?) to get src and href attributes and filter out nulls
# fromurl.host | values, parse as url and filter out those without a host
# from_url.host | values, parse as url and filter out those without a host
# count to count unique values, returns [[key, count], ...]
# reverse sort by count and pick first 3
# map [key, count] tuples into {key: key, values: count}
# from_entries, convert into object
$ curl -s https://www.discogs.com/ | fq -d html '[.. | ."@src"?, ."@href"? | values | fromurl.host | values] | count | sort_by(-.[1])[0:3] | map({key: .[0], value: .[1]}) | from_entries'
$ curl -s https://www.discogs.com/ | fq -d html '[.. | ."@src"?, ."@href"? | values | from_url.host | values] | count | sort_by(-.[1])[0:3] | map({key: .[0], value: .[1]}) | from_entries'
{
"blog.discogs.com": 9,
"st.discogs.com": 10,
@ -469,7 +558,7 @@ $ fq -i . <(curl -sL https://github.com/stefangabos/world_countries/archive/mas
# select from interesting xml file
zip> .local_files[] | select(.file_name == "world_countries-master/data/countries/en/world.xml").uncompressed | repl
# convert xml into jq value
> .local_files[95].uncompressed string> fromxml | repl
> .local_files[95].uncompressed string> from_xml | repl
# sort countries by and select the first one
>> object> .countries.country | sort_by(."@name") | first | repl
# see what current input is
@ -481,7 +570,7 @@ zip> .local_files[] | select(.file_name == "world_countries-master/data/countrie
"@name": "Afghanistan"
}
# remove "@" prefix from keys and convert to YAML and print it
>>> object> with_entries(.key |= .[1:]) | toyaml | print
>>> object> with_entries(.key |= .[1:]) | to_yaml | print
alpha2: af
alpha3: afg
id: "4"
@ -493,21 +582,21 @@ name: Afghanistan
zip> ^D
```
- `fromxml`/`fromxml($opts)` Parse XML into jq value.<br>
- `from_xml`/`from_xml($opts)` Parse XML into jq value.<br>
`{seq: true}` preserve element ordering if more than one sibling.<br>
`{array: true}` use nested `[name, attributes, children]` arrays to represent elements. Attributes will be `null` if none and children will be `[]` if none, this is to make it easier to work it. `toxml` does not require this.<br>
- `fromhtml`/`fromhtml($opts)` Parse HTML into jq value.<br>
Similar to `fromxml` but parses html5 in non-script mode. Will always have a `html` root with `head` and `body` elements.<br>
`{array: true}` use nested `[name, attributes, children]` arrays to represent elements. Attributes will be `null` if none and children will be `[]` if none, this is to make it easier to work with as the array as 3 values. `to_xml` does not require this.<br>
- `from_html`/`from_html($opts)` Parse HTML into jq value.<br>
Similar to `from_xml` but parses html5 in non-script mode. Will always have a `html` root with `head` and `body` elements.<br>
`{array: true}` use nested arrays to represent elements.<br>
`{seq: true}` preserve element ordering if more than one sibling.<br>
- `toxml`/`toxml($opts})` Serialize jq value into XML.<br>
- `to_xml`/`to_xml($opts})` Serialize jq value into XML.<br>
`{indent: number}` indent child elements.<br>
Assumes object representation if input is an object, and nested arrays if input is an array.<br>
Will automatically add a root `doc` element if jq value has more then one root element.<br>
If a `#seq` is found on at least one element all siblings will be sort by sequence number. Attributes are always sorted.<br>
XML elements can be represented as jq value in two ways, as objects (inspired by [mxj](https://github.com/clbanning/mxj) and [xml.com's Converting Between XML and JSON
](https://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html)) or nested arrays. Both representations are lossy and might lose ordering of elements, text nodes and comments. In object representation `fromxml`, `fromhtml` and `toxml` support `{seq:true}` option to parse/serialize `{"#seq"=<number>}` attributes to preserve element sibling ordering.
](https://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html)) or nested arrays. Both representations are lossy and might lose ordering of elements, text nodes and comments. In object representation `from_xml`, `from_html` and `to_xml` support `{seq:true}` option to parse/serialize `{"#seq"=<number>}` attributes to preserve element sibling ordering.
The object version is denser and convenient to query, the nested arrays version is probably easier to use when generating XML.
@ -527,9 +616,9 @@ zip> ^D
- For explicit sibling ordering `#seq` keys with a number, can be negative, assumed zero if missing.
- Child element with only text as `<name>` key with text as value.
- Child element with more than just text as `<name>` key with value an object.
- Multiple child element sibling with same name as `name` key with value as array with strings and objects.
- Multiple child element siblings with same name as `name` key with value as array with strings and objects.
```jq
> $xml | fromxml
> $xml | from_xml
{
"doc": {
"child": [
@ -547,12 +636,12 @@ zip> ^D
```
With nested array representation, an array with these values `["<name>", {attributes...}, [children...]]`
- Index 0 is element name.
- Index 0 is an element name.
- Index 1 object attributes (including `#text` and `#comment` keys).
- Index 2 array of child elements.
#
```jq
> $xml | fromxml({array: true})
> $xml | from_xml({array: true})
[
"doc",
[
@ -580,7 +669,7 @@ zip> ^D
```
Parse and include `#seq` attributes if needed:
```jq
> $xml | fromxml({seq:true})
> $xml | from_xml({seq:true})
{
"doc": {
"child": [
@ -603,7 +692,7 @@ zip> ^D
````
Select values in `<doc>`, remove `<child>`, add a `<new>` element, serialize to xml with 2 space indent and print the string
```jq
> $xml | fromxml.doc | del(.child) | .new = "abc" | {root: .} | toxml({indent: 2}) | println
> $xml | from_xml.doc | del(.child) | .new = "abc" | {root: .} | to_xml({indent: 2}) | println
<root>
<new>abc</new>
<other>text</other>
@ -614,45 +703,47 @@ JSON and jq-flavoured JSON
- `fromjson` Parse JSON into jq value.
- `tojson`/`tojson($opt)` Serialize jq value into JSON.<br>
`{indent: number}` indent array/object values.<br>
- `fromjq` Parse jq-flavoured JSON into jq value.
- `tojq`/`tojq($opt)` Serialize jq value into jq-flavoured JSON<br>
- `from_jq` Parse jq-flavoured JSON into jq value.
- `to_jq`/`to_jq($opt)` Serialize jq value into jq-flavoured JSON<br>
`{indent: number}` indent array/object values.<br>
jq-flavoured JSON has optional key quotes, `#` comments and can have trailing comma in arrays.
- `fromjsonl` Parse JSON lines into jq array.
- `tojsonl` Serialize jq array into JSONL.
- `from_jsonl` Parse JSON lines into jq array.
- `to_jsonl` Serialize jq array into JSONL.
Note that `fromjson` and `tojson` use different naming conventions as they originate from jq's standard library.
YAML
- `fromyaml` Parse YAML into jq value.
- `toyaml` Serialize jq value into YAML.
- `from_yaml` Parse YAML into jq value.
- `to_yaml` Serialize jq value into YAML.
TOML
- `fromtoml` Parse TOML into jq value.
- `totoml` Serialize jq value into TOML.
- `from_toml` Parse TOML into jq value.
- `to_toml` Serialize jq value into TOML.
CSV
- `fromcsv`/`fromcvs($opts)` Parse CSV into jq value.<br>
- `from_csv`/`from_cvs($opts)` Parse CSV into jq value.<br>
`{comma: string}` field separator, default ",".<br>
`{comment: string}` comment line character, default "#".<br>
To work with tab separated values you can use `fromcvs({comma: "\t"})` or `fq -d csv -o 'comma="\t"'`
- `tocsv`/`tocsv($opts)` Serialize jq value into CSV.<br>
- `to_csv`/`to_csv($opts)` Serialize jq value into CSV.<br>
`{comma: string}` field separator, default ",".<br>
XML encoding
- `fromxmlentities` Decode XML entities.
- `toxmlentities` Encode XML entities.
- `from_xmlentities` Decode XML entities.
- `to_xmlentities` Encode XML entities.
URL parts and XML encodings
- `fromurlpath` Decode URL path component.
- `tourlpath` Encode URL path component. Whitespace as %20.
- `fromurlencode` Decode URL query encoding.
- `tourlencode` Encode URL to query encoding. Whitespace as "+".
- `fromurlquery` Decode URL query into object. For duplicates keys value will be an array.
- `tourlquery` Encode objet into query string.
- `fromurl` Decode URL into object.
- `from_urlpath` Decode URL path component.
- `to_urlpath` Encode URL path component. Whitespace as %20.
- `from_urlencode` Decode URL query encoding.
- `to_urlencode` Encode URL to query encoding. Whitespace as "+".
- `from_urlquery` Decode URL query into object. For duplicates keys value will be an array.
- `to_urlquery` Encode object into query string.
- `from_url` Decode URL into object.
```jq
> "schema://user:pass@host/path?key=value#fragment" | fromurl
> "schema://user:pass@host/path?key=value#fragment" | from_url
{
"fragment": "fragement",
"fragment": "fragment",
"host": "host",
"path": "/path",
"query": {
@ -666,43 +757,79 @@ URL parts and XML encodings
}
}
```
- `tourl` Encode object into URL string.
- `fromhex` Decode hexstring to binary.
- `to_url` Encode object into URL string.
Binary encodings like hex and base64
- `tohex` Encode binay into hexstring.
- `frombase64`/`frombase64($opts)` Decode base64 encodings into binary.<br>
- `from_hex` Decode hex string to binary.
- `to_hex` Encode binary into hex string.
- `from_base64`/`from_base64($opts)` Decode base64 encodings into binary.<br>
`{encoding:string}` encoding variant: `std` (default), `url`, `rawstd` or `rawurl`
- `tobase64`/`tobase64($opts)` Encode binary into base64 encodings.<br>
- `to_base64`/`to_base64($opts)` Encode binary into base64 encodings.<br>
`{encoding:string}` encoding variant: `std` (default), `url`, `rawstd` or `rawurl`
Hash functions
- `tomd4` Hash binary using md4.
- `tomd5` Hash binary using md5.
- `tosha1` Hash binary using sha1.
- `tosha256` Hash binary using sha256.
- `tosha512` Hash binary using sha512.
- `tosha3_224` Hash binary using sha3 224.
- `tosha3_256` Hash binary using sha3 256.
- `tosha3_384` Hash binary using sha3 384.
- `tosha3_512` Hash binary using sha3 512.
- `to_md4` Hash binary using md4.
- `to_md5` Hash binary using md5.
- `to_sha1` Hash binary using sha1.
- `to_sha256` Hash binary using sha256.
- `to_sha512` Hash binary using sha512.
- `to_sha3_224` Hash binary using sha3 224.
- `to_sha3_256` Hash binary using sha3 256.
- `to_sha3_384` Hash binary using sha3 384.
- `to_sha3_512` Hash binary using sha3 512.
Text encodings
- `toiso8859_1` Decode binary as ISO8859-1 into string.
- `fromiso8859_1` Encode string as ISO8859-1 into binary.
- `toutf8` Encode string as UTF8 into binary.
- `fromutf8` Decode binary as UTF8 into string.
- `toutf16` Encode string as UTF16 into binary.
- `fromutf16` Decode binary as UTF16 into string.
- `toutf16le` Encode string as UTF16 little-endian into binary.
- `fromutf16le` Decode binary as UTF16 little-endian into string.
- `toutf16be` Encode string as UTF16 big-endian into binary.
- `fromutf16be` Decode binary as UTF16 big-endian into string.
- `to_iso8859_1` Decode binary as ISO8859-1 into string.
- `from_iso8859_1` Encode string as ISO8859-1 into binary.
- `to_utf8` Encode string as UTF8 into binary.
- `from_utf8` Decode binary as UTF8 into string.
- `to_utf16` Encode string as UTF16 into binary.
- `from_utf16` Decode binary as UTF16 into string.
- `to_utf16le` Encode string as UTF16 little-endian into binary.
- `from_utf16le` Decode binary as UTF16 little-endian into string.
- `to_utf16be` Encode string as UTF16 big-endian into binary.
- `from_utf16be` Decode binary as UTF16 big-endian into string.
## Options
fq has some general options in addition to decode and decoders specific options. They all use the same `-o <name>=<value>` argument.
`<value>` is fuzzily parsed based on the type of the option. Ex: a string can be specified as `-o name=string` or `-o name="string"`.
### `-o bits_format=<string>`
How to represent raw binary as JSON.
- `-o bits_foramt=string` String with raw bytes (zero bit padded if size is not byte aligned). The string is binary safe internally in fq but bytes not representable as UTF-8 will be lost if turn to JSON.
- `-o bits_format=md5` MD5 hex string (zero bit padded).
- `-o bits_format=base64` Base64 string.
- `-p bits_foramt=truncate` Truncated string.
- `-o bits_format=snippet` Truncated Base64 string prefixed with bit length.
```sh
$ fq -V -o bits_format=base64 . file`
```
In query
```jq
tovalue({bits_format: "md5"})
```
### `-o skip_gaps=<boolean>`
Skip gaps fields (`gap0` etc) when using `tovalue` or `-V`. Note that this might affect array indexes if one more more gaps fields are skipped in an array.
```sh
$ fq -V -o skip_gaps=true . file`
```
In query
```jq
tovalue({skip_gaps: true})
```
## Color and unicode output
fq by default tries to use colors if possible, this can be disabled with `-M`. You can also
enable useage of unicode characters for improved output by setting the environment
enable usage of unicode characters for improved output by setting the environment
variable `CLIUNICODE`.
## Configuration
@ -714,7 +841,7 @@ To add own functions you can use `init.fq` that will be read from
## Use as script interpreter
fq can be used as a scrip interpreter:
fq can be used as a script interpreter:
`mp3_duration.jq`:
```jq
@ -726,35 +853,39 @@ fq can be used as a scrip interpreter:
- [gojq's differences to jq](https://github.com/itchyny/gojq#difference-to-jq),
notable is support for arbitrary-precision integers.
- Supports hexdecimal `0xab`, octal `0o77` and binary `0b101` integer literals.
- Try include `include "file?";` that don't fail if file is missing.
- Some values can act as a object with keys even when it's an array, number etc.
- Supports hexadecimal `0xab`, octal `0o77` and binary `0b101` integer literals.
- Try include `include "file?";` that doesn't fail if file is missing or has errors.
- Some values can act as an object with keys even when it's an array, number etc.
- There can be keys hidden from `keys` and `[]`.
- Some values are readonly and can't be updated.
## Decoded values
When you decode something you will get a decode value. A decode value work like
normal jq values but has special abilities and is used to represent a tree structure of the decoded
binary data. Each value always has a name, type and a bit range.
When decoding something, using `decode` or `mp3` etc, you a decode value is returned. They behave like
normal jq values but has special abilities and is used to represent the decoded structure. Each value
always has a name, type and a bit range.
A value has these special keys (TODO: remove, are internal)
- `_actual` decoded (not symbol mapped value)
- `_bits` bits in range as a binary
- `_buffer_root` first decode value for current buffer
- `_bytes` bits in range as binary using byte units
- `_description` description of value (optional)
- `_error` error message (optional)
- `_format` name of decoded format (optional, only format root)
- `_format_root` first decode value for current format
- `_gap` is a bit range gap (was not decoded)
- `_index` index in parent array (only for values in arrays)
- `_len` bit range length (TODO: rename)
- `_name` name of value
- `_value` jq value of value
- `_out` decoded out value
- `_parent` parent decode value
- `_path` jq path to decode value
- `_root` root decode value
- `_start` bit range start
- `_stop` bit range stop
- `_len` bit range length (TODO: rename)
- `_bits` bits in range as a binary
- `_bytes` bits in range as binary using byte units
- `_path` jq path to value
- `_unknown` value is un-decoded gap
- `_symbol` symbolic string representation of value (optional)
- `_description` longer description of value (optional)
- `_format` name of decoded format (optional)
- `_error` error message (optional)
- TODO: unknown gaps
- `_sym` symbolic value (optional)
## Own decoders and use as library
@ -775,11 +906,11 @@ Try add `select(...)?` to catch and ignore type errors in the select expression.
### Manual decode
Sometimes fq fails to decode or you know there is valid data buried inside some binary or maybe
you know the format of some unknown value. Then you can decode manually.
you know the format of some gap field. Then you can decode manually.
<pre>
# try decode a `mp3_frame` that failed to decode
$ fq -d mp3 '.unknown0 | mp3_frame' file.mp3
$ fq -d mp3 '.gap0 | mp3_frame' file.mp3
# skip first 10 bytes then decode as `mp3_frame`
$ fq -d bytes '.[10:] | mp3_frame' file.mp3
</pre>

10
format/all/all.fqtest
View File

@ -1,6 +1,7 @@
$ fq -n _registry.groups.probe
[
"adts",
"apple_bookmark",
"ar",
"avi",
"avro_ocf",
@ -22,9 +23,11 @@ $ fq -n _registry.groups.probe
"png",
"tar",
"tiff",
"tzif",
"wasm",
"webp",
"zip",
"aiff",
"mp3",
"mpeg_ts",
"wav",
@ -38,8 +41,10 @@ $ fq --help formats
aac_frame Advanced Audio Coding frame
adts Audio Data Transport Stream
adts_frame Audio Data Transport Stream frame
aiff Audio Interchange File Format
amf0 Action Message Format 0
apev2 APEv2 metadata tag
apple_bookmark Apple BookmarkData
ar Unix archive
asn1_ber ASN1 BER (basic encoding rules, also CER and DER)
av1_ccr AV1 Codec Configuration Record
@ -106,7 +111,8 @@ markdown Markdown
matroska Matroska file
mp3 MP3 file
mp3_frame MPEG audio layer 3 frame
mp3_frame_tags MP3 frame info/xing tags
mp3_frame_vbri MP3 frame Fraunhofer encoder variable bitrate tag
mp3_frame_xing MP3 frame Xing/Info tag
mp4 ISOBMFF, QuickTime and similar
mpeg_asc MPEG-4 Audio Specific Config
mpeg_es MPEG Elementary Stream
@ -131,7 +137,9 @@ sll_packet Linux cooked capture encapsulation
tar Tar archive
tcp_segment Transmission control protocol segment
tiff Tag Image File Format
tls Transport layer security
toml Tom's Obvious, Minimal Language
tzif Time Zone Information Format
udp_datagram User datagram protocol
vorbis_comment Vorbis comment
vorbis_packet Vorbis packet

9
format/all/all.go
View File

@ -1,10 +1,11 @@
// Package all registers all builtin formats with the default registry
//
//nolint:revive
package all
import (
_ "github.com/wader/fq/format/ape"
_ "github.com/wader/fq/format/apple/bookmark"
_ "github.com/wader/fq/format/apple/bplist"
_ "github.com/wader/fq/format/apple/macho"
_ "github.com/wader/fq/format/ar"
_ "github.com/wader/fq/format/asn1"
_ "github.com/wader/fq/format/av1"
@ -12,7 +13,6 @@ import (
_ "github.com/wader/fq/format/bencode"
_ "github.com/wader/fq/format/bitcoin"
_ "github.com/wader/fq/format/bits"
_ "github.com/wader/fq/format/bplist"
_ "github.com/wader/fq/format/bson"
_ "github.com/wader/fq/format/bzip2"
_ "github.com/wader/fq/format/cbor"
@ -29,7 +29,6 @@ import (
_ "github.com/wader/fq/format/inet"
_ "github.com/wader/fq/format/jpeg"
_ "github.com/wader/fq/format/json"
_ "github.com/wader/fq/format/macho"
_ "github.com/wader/fq/format/markdown"
_ "github.com/wader/fq/format/math"
_ "github.com/wader/fq/format/matroska"
@ -49,7 +48,9 @@ import (
_ "github.com/wader/fq/format/tar"
_ "github.com/wader/fq/format/text"
_ "github.com/wader/fq/format/tiff"
_ "github.com/wader/fq/format/tls"
_ "github.com/wader/fq/format/toml"
_ "github.com/wader/fq/format/tzif"
_ "github.com/wader/fq/format/vorbis"
_ "github.com/wader/fq/format/vpx"
_ "github.com/wader/fq/format/wasm"

25
format/ape/apev2.go
View File

@ -8,26 +8,27 @@ import (
"github.com/wader/fq/pkg/interp"
)
var imageFormat decode.Group
var imageGroup decode.Group
func init() {
interp.RegisterFormat(decode.Format{
Name: format.APEV2,
Description: "APEv2 metadata tag",
DecodeFn: apev2Decode,
Dependencies: []decode.Dependency{
{Names: []string{format.IMAGE}, Group: &imageFormat},
},
})
interp.RegisterFormat(
format.Apev2,
&decode.Format{
Description: "APEv2 metadata tag",
DecodeFn: apev2Decode,
Dependencies: []decode.Dependency{
{Groups: []*decode.Group{format.Image}, Out: &imageGroup},
},
})
}
func apev2Decode(d *decode.D, _ any) any {
func apev2Decode(d *decode.D) any {
d.Endian = decode.LittleEndian
headerFooterFn := func(d *decode.D, name string) uint64 {
var tagCount uint64
d.FieldStruct(name, func(d *decode.D) {
d.FieldUTF8("preamble", 8, d.AssertStr("APETAGEX"))
d.FieldUTF8("preamble", 8, d.StrAssert("APETAGEX"))
d.FieldU32("version")
d.FieldU32("tag_size")
tagCount = d.FieldU32("item_count")
@ -59,7 +60,7 @@ func apev2Decode(d *decode.D, _ any) any {
d.FramedFn(int64(itemSize)*8, func(d *decode.D) {
d.FieldUTF8Null("filename")
// assume image if binary
d.FieldFormatOrRaw("value", imageFormat, nil)
d.FieldFormatOrRaw("value", &imageGroup, nil)
})
} else {
d.FieldUTF8("value", int(itemSize))

422
format/apple/bookmark/apple_bookmark.go Normal file
View File

@ -0,0 +1,422 @@
package bookmarkdata
import (
"embed"
"time"
"github.com/wader/fq/format"
"github.com/wader/fq/format/apple"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
"github.com/wader/fq/pkg/scalar"
)
//go:embed apple_bookmark.jq apple_bookmark.md
var bookmarkFS embed.FS
func init() {
interp.RegisterFormat(format.Apple_Bookmark,
&decode.Format{
ProbeOrder: format.ProbeOrderBinUnique,
Description: "Apple BookmarkData",
Groups: []*decode.Group{format.Probe},
DecodeFn: bookmarkDecode,
Functions: []string{"torepr"},
})
interp.RegisterFS(bookmarkFS)
}
const (
dataTypeString = 0x0101
dataTypeData = 0x0201
dataTypeNumber8 = 0x0301
dataTypeNumber16 = 0x0302
dataTypeNumber32 = 0x0303
dataTypeNumber64 = 0x0304
dataTypeNumber32F = 0x0305
dataTypeNumber64F = 0x0306
dataTypeDate = 0x0400
dataTypeBooleanFalse = 0x0500
dataTypeBooleanTrue = 0x0501
dataTypeArray = 0x0601
dataTypeDictionary = 0x0701
dataTypeUUID = 0x0801
dataTypeURL = 0x0901
dataTypeRelativeURL = 0x0902
)
var dataTypeMap = scalar.UintMap{
dataTypeString: {Sym: "string", Description: "UTF-8 String"},
dataTypeData: {Sym: "data", Description: "Raw bytes"},
dataTypeNumber8: {Sym: "byte", Description: "(signed 8-bit) 1-byte number"},
dataTypeNumber16: {Sym: "short", Description: "(signed 16-bit) 2-byte number"},
dataTypeNumber32: {Sym: "int", Description: "(signed 32-bit) 4-byte number"},
dataTypeNumber64: {Sym: "long", Description: "(signed 64-bit) 8-byte number"},
dataTypeNumber32F: {Sym: "float", Description: "(32-bit float) IEEE single precision"},
dataTypeNumber64F: {Sym: "double", Description: "(64-bit float) IEEE double precision"},
dataTypeDate: {Sym: "date", Description: "Big-endian IEEE double precision seconds since 2001-01-01 00:00:00 UTC"},
dataTypeBooleanFalse: {Sym: "boolean_false", Description: "False"},
dataTypeBooleanTrue: {Sym: "boolean_true", Description: "True"},
dataTypeArray: {Sym: "array", Description: "Array of 4-byte offsets to data items"},
dataTypeDictionary: {Sym: "dictionary", Description: "Array of pairs of 4-byte (key, value) data item offsets"},
dataTypeUUID: {Sym: "uuid", Description: "Raw bytes"},
dataTypeURL: {Sym: "url", Description: "UTF-8 string"},
dataTypeRelativeURL: {Sym: "relative_url", Description: "4-byte offset to base URL, 4-byte offset to UTF-8 string"},
}
const (
elementTypeTargetURL = 0x1003
elementTypeTargetPath = 0x1004
elementTypeTargetCNIDPath = 0x1005
elementTypeTargetFlags = 0x1010
elementTypeTargetFilename = 0x1020
elementTypeCNID = 0x1030
elementTypeTargetCreationDate = 0x1040
elementTypeUnknown1 = 0x1054
elementTypeUnknown2 = 0x1055
elementTypeUnknown3 = 0x1056
elementTypeUnknown4 = 0x1101
elementTypeUnknown5 = 0x1102
elementTypeTOCPath = 0x2000
elementTypeVolumePath = 0x2002
elementTypeVolumeURL = 0x2005
elementTypeVolumeName = 0x2010
elementTypeVolumeUUID = 0x2011
elementTypeVolumeSize = 0x2012
elementTypeVolumeCreationDate = 0x2013
elementTypeVolumeFlags = 0x2020
elementTypeVolumeIsRoot = 0x2030
elementTypeVolumeBookmark = 0x2040
elementTypeVolumeMountPointURL = 0x2050
elementTypeUnknown6 = 0x2070
elementTypeContainingFolderIndex = 0xc001
elementTypeCreatorUsername = 0xc011
elementTypeCreatorUID = 0xc012
elementTypeFileReferenceFlag = 0xd001
elementTypeCreationOptions = 0xd010
elementTypeURLLengthArray = 0xe003
elementTypeDisplayName = 0xf017
elementTypeIconData = 0xf020
elementTypeIconImageData = 0xf021
elementTypeTypeBindingInfo = 0xf022
elementTypeBookmarkCreationTime = 0xf030
elementTypeSandboxRWExtension = 0xf080
elementTypeSandboxROExtension = 0xf081
)
var elementTypeMap = scalar.UintMap{
elementTypeTargetURL: {Sym: "target_url", Description: "A URL"},
elementTypeTargetPath: {Sym: "target_path", Description: "Array of individual path components"},
elementTypeTargetCNIDPath: {Sym: "target_cnid_path", Description: "Array of CNIDs"},
elementTypeTargetFlags: {Sym: "target_flags", Description: "flag bitfield"},
elementTypeTargetFilename: {Sym: "target_filename", Description: "String"},
elementTypeCNID: {Sym: "target_cnid", Description: "4-byte integer"},
elementTypeTargetCreationDate: {Sym: "target_creation_date", Description: "Date"},
elementTypeUnknown1: {Sym: "unknown1", Description: "Unknown"},
elementTypeUnknown2: {Sym: "unknown2", Description: "Unknown"},
elementTypeUnknown3: {Sym: "unknown3", Description: "Unknown"},
elementTypeUnknown4: {Sym: "unknown4", Description: "Unknown"},
elementTypeUnknown5: {Sym: "unknown5", Description: "Unknown"},
elementTypeTOCPath: {Sym: "toc_path", Description: "Array - see below"},
elementTypeVolumePath: {Sym: "volume_path", Description: "Array of individual path components"},
elementTypeVolumeURL: {Sym: "volume_url", Description: "URL of volume root"},
elementTypeVolumeName: {Sym: "volume_name", Description: "String"},
elementTypeVolumeUUID: {Sym: "volume_uuid", Description: "String UUID"},
elementTypeVolumeSize: {Sym: "volume_size", Description: "8-byte integer"},
elementTypeVolumeCreationDate: {Sym: "volume_creation_date", Description: "Date"},
elementTypeVolumeFlags: {Sym: "volume_flags", Description: "flag bitfield"},
elementTypeVolumeIsRoot: {Sym: "volume_is_root", Description: "True if the volume was the filesystem root"},
elementTypeVolumeBookmark: {Sym: "volume_bookmark", Description: "TOC identifier for disk image"},
elementTypeVolumeMountPointURL: {Sym: "volume_mount_point", Description: "URL"},
elementTypeUnknown6: {Sym: "unknown6", Description: "Unknown"},
elementTypeContainingFolderIndex: {Sym: "containing_folder_index", Description: "Integer index of containing folder in target path array"},
elementTypeCreatorUsername: {Sym: "creator_username", Description: "Name of user that created bookmark"},
elementTypeCreatorUID: {Sym: "creator_uid", Description: "UID of user that created bookmark"},
elementTypeFileReferenceFlag: {Sym: "file_reference_flag", Description: "True if creating URL was a file reference URL"},
elementTypeCreationOptions: {Sym: "creation_options", Description: "Integer containing flags passed to CFURLCreateBookmarkData"},
elementTypeURLLengthArray: {Sym: "url_length_array", Description: "Array of integers"},
elementTypeDisplayName: {Sym: "display_name", Description: "String"},
elementTypeIconData: {Sym: "icon_data", Description: "icns format data"},
elementTypeIconImageData: {Sym: "icon_image", Description: "Data"},
elementTypeTypeBindingInfo: {Sym: "type_binding_info", Description: "dnib byte array"},
elementTypeBookmarkCreationTime: {Sym: "bookmark_creation_time", Description: "64-bit float seconds since January 1st 2001"},
elementTypeSandboxRWExtension: {Sym: "sandbox_rw_extension", Description: "Looks like a hash with data and an access right"},
elementTypeSandboxROExtension: {Sym: "sandbox_ro_extension", Description: "Looks like a hash with data and an access right"},
}
const dataObjectLen = 24
func decodeFlagDataObject(d *decode.D, flagFn func(d *decode.D)) {
d.FieldStruct("record", func(d *decode.D) {
d.FieldU32("length", d.UintAssert(dataObjectLen))
d.FieldU32("raw_type", dataTypeMap, d.UintAssert(dataTypeData))
d.FieldValueStr("type", "flag_data")
d.FieldStruct("property_flags", flagFn)
d.FieldStruct("enabled_property_flags", flagFn)
d.FieldRawLen("reserved", 64)
})
}
func decodeTgtPropertyFlagBits(d *decode.D) {
start := d.Pos()
d.FieldBool("is_hidden")
d.FieldBool("is_user_immutable")
d.FieldBool("is_system_immutable")
d.FieldBool("is_package")
d.FieldBool("is_volume")
d.FieldBool("is_symbolic_link")
d.FieldBool("is_directory")
d.FieldBool("is_regular_file")
d.FieldBool("is_alias_file")
d.FieldBool("is_executable")
d.FieldBool("is_writeable")
d.FieldBool("is_readable")
d.FieldBool("can_set_hidden_extension")
d.FieldBool("is_compressed")
d.FieldBool("is_application")
d.FieldBool("has_hidden_extension")
d.FieldRawLen("reserved_bits_0", 7)
d.FieldBool("is_mount_trigger")
d.FieldRawLen("reserved", 64-(d.Pos()-start))
}
func decodeVolPropertyFlagBits(d *decode.D) {
d.FieldBool("is_internal")
d.FieldBool("is_removable")
d.FieldBool("is_ejectable")
d.FieldBool("is_quarantined")
d.FieldBool("is_read_only")
d.FieldBool("dont_browse")
d.FieldBool("is_automount")
d.FieldBool("is_local")
d.FieldBool("is_dvd")
d.FieldBool("is_cd")
d.FieldBool("is_idisk")
d.FieldBool("is_ipod")
d.FieldBool("is_local_idisk_mirror")
d.FieldBool("is_file_vault")
d.FieldBool("is_disk_image")
d.FieldBool("is_external")
d.FieldRawLen("reserved_0", 7)
d.FieldBool("is_device_file_system")
d.FieldRawLen("reserved_1", 8)
d.FieldBool("supports_read_dir_attr")
d.FieldBool("supports_copy_file")
d.FieldBool("supports_deny_modes")
d.FieldBool("supports_symbolic_links")
d.FieldBool("reserved_2")
d.FieldBool("supports_exchange")
d.FieldBool("supports_search_fs")
d.FieldBool("supports_persistent_ids")
d.FieldBool("supports_extended_security")
d.FieldBool("has_no_root_directory_times")
d.FieldBool("supports_flock")
d.FieldBool("supports_case_preserved_names")
d.FieldBool("supports_case_sensitive_names")
d.FieldBool("supports_fast_stat_fs")
d.FieldBool("supports_rename")
d.FieldBool("supports_journaling")
d.FieldBool("supports_zero_runs")
d.FieldBool("supports_sparse_files")
d.FieldBool("is_journaling")
d.FieldBool("reserved_3")
d.FieldBool("supports_path_from_id")
d.FieldBool("supports_mandatory_byte_range_locks")
d.FieldBool("supports_hard_links")
d.FieldBool("supports_2_tb_file_size")
d.FieldRawLen("reserved_4", 3)
d.FieldBool("has64_bit_object_ids")
d.FieldBool("supports_decmp_fs_compression")
d.FieldBool("supports_hidden_files")
d.FieldBool("supports_remote_events")
d.FieldBool("supports_volume_sizes")
}
var cocoaTimeEpochDate = time.Date(2001, time.January, 1, 0, 0, 0, 0, time.UTC)
type tocHeader struct {
nextTOCOffset uint64
numEntries uint64
entryArrayOffset int64
}
func (hdr *tocHeader) decodeEntries(d *decode.D) {
for k := uint64(0); k < hdr.numEntries; k++ {
d.FieldStruct("entry", func(d *decode.D) {
key := d.FieldU32("key", elementTypeMap)
// if the key has the top bit set, then (key & 0x7fffffff)
// gives the offset of a string record.
if key&0x80000000 != 0 {
d.FieldStruct("key_string", func(d *decode.D) {
d.SeekAbs(calcOffset(key&0x7fffffff), makeDecodeRecord())
})
}
recordOffset := calcOffset(d.FieldU32("offset_to_record"))
d.FieldU32("unused")
switch key {
case elementTypeTargetFlags:
d.SeekAbs(recordOffset, func(d *decode.D) { decodeFlagDataObject(d, decodeTgtPropertyFlagBits) })
case elementTypeVolumeFlags:
d.SeekAbs(recordOffset, func(d *decode.D) { decodeFlagDataObject(d, decodeVolPropertyFlagBits) })
default:
d.SeekAbs(recordOffset, makeDecodeRecord())
}
})
}
}
func decodeTOCHeader(d *decode.D) *tocHeader {
hdr := new(tocHeader)
d.FieldStruct("toc_header", func(d *decode.D) {
d.FieldU32("toc_size")
d.FieldU32("magic", d.UintAssert(0xfffffffe))
d.FieldU32("identifier")
hdr.nextTOCOffset = d.FieldU32("next_toc_offset")
hdr.numEntries = d.FieldU32("num_entries_in_toc")
hdr.entryArrayOffset = d.Pos()
})
return hdr
}
const (
arrayEntrySize = 4
dictEntrySize = 4
)
func makeDecodeRecord() func(d *decode.D) {
var pld apple.PosLoopDetector[int64]
var decodeRecord func(d *decode.D)
decodeRecord = func(d *decode.D) {
defer pld.PushAndPop(
d.Pos(),
func() { d.Fatalf("infinite recursion detected in record decode function") },
)()
d.FieldStruct("record", func(d *decode.D) {
n := int(d.FieldU32("length"))
typ := d.FieldU32("type", dataTypeMap)
switch typ {
case dataTypeString:
d.FieldUTF8("data", n)
d.FieldRawLen("alignment_bytes", 32-((d.Pos()+32)%32))
case dataTypeData:
d.FieldRawLen("data", int64(n*8))
case dataTypeNumber8:
d.FieldS8("data")
case dataTypeNumber16:
d.FieldS16("data")
case dataTypeNumber32:
d.FieldS32("data")
case dataTypeNumber64:
d.FieldS64("data")
case dataTypeNumber32F:
d.FieldF32("data")
case dataTypeNumber64F:
d.FieldF64("data")
case dataTypeDate:
d.FieldF64BE("data", scalar.FltActualDate(cocoaTimeEpochDate, time.RFC3339))
case dataTypeBooleanFalse:
case dataTypeBooleanTrue:
case dataTypeArray:
d.FieldStructNArray("data", "element", int64(n/arrayEntrySize), func(d *decode.D) {
offset := calcOffset(d.FieldU32("offset"))
d.SeekAbs(offset, decodeRecord)
})
case dataTypeDictionary:
d.FieldStructNArray("data", "element", int64(n/dictEntrySize), func(d *decode.D) {
keyOffset := calcOffset(d.FieldU32("key_offset"))
d.FieldStruct("key", func(d *decode.D) {
d.SeekAbs(keyOffset, decodeRecord)
})
valueOffset := calcOffset(d.FieldU32("value_offset"))
d.FieldStruct("value", func(d *decode.D) {
d.SeekAbs(valueOffset, decodeRecord)
})
})
case dataTypeUUID:
d.FieldRawLen("data", int64(n*8))
case dataTypeURL:
d.FieldUTF8("data", n)
case dataTypeRelativeURL:
baseOffset := d.FieldU32("base_url_offset")
d.FieldStruct("base_url", func(d *decode.D) {
d.SeekAbs(int64(baseOffset), decodeRecord)
})
suffixOffset := d.FieldU32("suffix_offset")
d.FieldStruct("suffix", func(d *decode.D) {
d.SeekAbs(int64(suffixOffset), decodeRecord)
})
}
})
}
return decodeRecord
}
const reservedSize = 32
const headerEnd = 48
const headerEndBitPos = headerEnd * 8
// all offsets are calculated relative to the end of the bookmark header
func calcOffset(i uint64) int64 { return int64(8 * (i + headerEnd)) }
func bookmarkDecode(d *decode.D) any {
// all fields are little-endian with the exception of the Date datatype.
d.Endian = decode.LittleEndian
// decode bookmarkdata header, one at the top of each "file",
// although these may be nested inside of binary plists
d.FieldStruct("header", func(d *decode.D) {
d.FieldUTF8("magic", 4, d.StrAssert("book", "alis"))
d.FieldU32("total_size")
d.FieldU32("unknown")
d.FieldU32("header_size", d.UintAssert(48))
d.FieldRawLen("reserved", reservedSize*8)
})
tocOffset := calcOffset(d.FieldU32("first_toc_offset"))
var currentHdr *tocHeader
var tocHeaders []*tocHeader
hdrCount := 0
d.FieldArrayLoop("toc_headers", func() bool {
return tocOffset != headerEndBitPos || hdrCount > 100
}, func(d *decode.D) {
d.SeekAbs(tocOffset, func(d *decode.D) {
currentHdr = decodeTOCHeader(d)
tocOffset = calcOffset(currentHdr.nextTOCOffset)
tocHeaders = append(tocHeaders, currentHdr)
})
hdrCount++
})
// now that we've collected all toc headers, iterate through each one's
// entries and decode associated records.
d.FieldArray("bookmark_entries",
func(d *decode.D) {
for _, hdr := range tocHeaders {
d.SeekAbs(hdr.entryArrayOffset, hdr.decodeEntries)
}
})
return nil
}

38
format/apple/bookmark/apple_bookmark.jq Normal file
View File

@ -0,0 +1,38 @@
def _apple_bookmark_torepr:
def _f:
if .type == "string" then .data | tovalue
elif .type == "data" then .data | tovalue
elif .type == "byte" then .data | tovalue
elif .type == "short" then .data | tovalue
elif .type == "int" then .data | tovalue
elif .type == "long" then .data | tovalue
elif .type == "float" then .data | tovalue
elif .type == "double" then .data | tovalue
elif .type == "date" then .data | tovalue
elif .type == "boolean_false" then false
elif .type == "boolean_true" then true
elif .type == "flag_data" then
( .enabled_property_flags as $eflags
| .property_flags
| to_entries
| map(select($eflags[.key] and (.key | startswith("reserved") | not)))
| from_entries
)
elif .type == "array" then
( .data
| map(.record | _f)
)
elif .type == "dictionary" then
( .data
| map({key: (.key | _f), value: (.value | _f)})
| from_entries
)
elif .type == "uuid" then .data | tovalue
elif .type == "url" then .data | tovalue
elif .type == "relative_url" then
.data | map(.record.data)
end;
( .bookmark_entries
| map({key: (.key_string?.record.data // .key|tostring), value: (.record | _f)})
| from_entries
);

38
format/apple/bookmark/apple_bookmark.md Normal file
View File

@ -0,0 +1,38 @@
Apple's `bookmarkData` format is used to encode information that can be resolved
into a `URL` object for a file even if the user moves or renames it. Can also
contain security scoping information for App Sandbox support.
These `bookmarkData` blobs are often found endcoded in data fields of Binary
Property Lists. Notable examples include:
- `com.apple.finder.plist` - contains an `FXRecentFolders` value, which is an
array of ten objects, each of which consists of a `name` and `file-bookmark`
field, which is a `bookmarkData` object for each recently accessed folder
location.
- `com.apple.LSSharedFileList.RecentApplications.sfl2` - `sfl2` files are
actually `plist` files of the `NSKeyedArchiver` format. They can be parsed the
same as `plist` files, but they have a more complicated tree-like structure
than would typically be found, which can make locating and retrieving specific
values difficult, even once it has been converted to a JSON representation.
For more information about these types of files, see Sarah Edwards' excellent
research on the subject (link in references).
`fq`'s `grep_by` function can be used to recursively descend through the decoded
tree, probing for and selecting any `bookmark` blobs, then converting them to
readable JSON with `torepr`:
```
fq 'grep_by(.type=="data" and .value[0:4] == "book") | .value | apple_bookmark |
torepr' <sfl2 file>
```
### Authors
- David McDonald
[@dgmcdona](https://github.com/dgmcdona)
[@river_rat_504](https://twitter.com/river_rat_504)
### References
- https://developer.apple.com/documentation/foundation/url/2143023-bookmarkdata
- https://mac-alias.readthedocs.io/en/latest/bookmark_fmt.html
- https://www.mac4n6.com/blog/2016/1/1/manual-analysis-of-nskeyedarchiver-formatted-plist-files-a-review-of-the-new-os-x-1011-recent-items
- https://michaellynn.github.io/2015/10/24/apples-bookmarkdata-exposed/

42
format/apple/bookmark/testdata/help_applebookmark.fqtest vendored Normal file
View File

@ -0,0 +1,42 @@
$ fq -h apple_bookmark
apple_bookmark: Apple BookmarkData decoder
Decode examples
===============
# Decode file as apple_bookmark
$ fq -d apple_bookmark . file
# Decode value as apple_bookmark
... | apple_bookmark
Apple's bookmarkData format is used to encode information that can be resolved into a URL object for a file even if the user moves or
renames it. Can also contain security scoping information for App Sandbox support.
These bookmarkData blobs are often found endcoded in data fields of Binary Property Lists. Notable examples include:
- com.apple.finder.plist - contains an FXRecentFolders value, which is an array of ten objects, each of which consists of a name and
file-bookmark field, which is a bookmarkData object for each recently accessed folder location.
- com.apple.LSSharedFileList.RecentApplications.sfl2 - sfl2 files are actually plist files of the NSKeyedArchiver format. They can be
parsed the same as plist files, but they have a more complicated tree-like structure than would typically be found, which can make
locating and retrieving specific values difficult, even once it has been converted to a JSON representation. For more information
about these types of files, see Sarah Edwards' excellent research on the subject (link in references).
fq's grep_by function can be used to recursively descend through the decoded tree, probing for and selecting any bookmark blobs, then
converting them to readable JSON with torepr:
fq 'grep_by(.type=="data" and .value[0:4] == "book") | .value | apple_bookmark |
torepr' <sfl2 file>
Authors
=======
- David McDonald @dgmcdona (https://github.com/dgmcdona) @river_rat_504 (https://twitter.com/river_rat_504)
References
==========
- https://developer.apple.com/documentation/foundation/url/2143023-bookmarkdata
- https://mac-alias.readthedocs.io/en/latest/bookmark_fmt.html
-
https://www.mac4n6.com/blog/2016/1/1/manual-analysis-of-nskeyedarchiver-formatted-plist-files-a-review-of-the-new-os-x-1011-recent-items
- https://michaellynn.github.io/2015/10/24/apples-bookmarkdata-exposed/

BIN
format/apple/bookmark/testdata/loop.book vendored Normal file
View File

Binary file not shown.

19
format/apple/bookmark/testdata/loop.fqtest vendored Normal file
View File

@ -0,0 +1,19 @@
$ fq . -d apple_bookmark loop.book
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: loop.book (apple_bookmark)
| | | error: apple_bookmark: error at position 0x6c: infinite recursion detected in record decode function
0x000|62 6f 6f 6b 30 02 00 00 00 00 04 10 30 00 00 00|book0.......0...| header{}:
* |until 0x2f.7 (48) | |
0x030|50 01 00 00 |P... | first_toc_offset: 336
0x030| 04 00 00 00 03 03 00 00 00 00 00 20| ........... | gap0: raw bits
0x040|0c 00 00 00 01 01 00 00 41 70 70 6c 69 63 61 74|........Applicat|
* |until 0x6b.7 (56) | |
0x060| 08 00 00 00| ....| bookmark_entries[0:1]:
0x070|01 06 00 00 3c 00 00 00 24 00 00 00 08 00 00 00|....<...$.......|
* |until 0x19f.7 (308) | |
0x070| 24 00 00 00 08 00 00 00| $.......| gap1: raw bits
0x080|04 03 00 00 67 f5 50 03 00 00 00 00 08 00 00 00|....g.P.........|
* |until 0x17f.7 (264) | |
0x180|a8 00 00 00 fe ff ff ff 01 00 00 00 00 00 00 00|................| toc_headers[0:1]:
0x190|0d 00 00 00 |.... |
0x1a0|05 10 00 00 6c 00 00 00 00 00 00 00 10 10 00 00|....l...........| gap2: raw bits
* |until 0x22f.7 (end) (144) | |

BIN
format/apple/bookmark/testdata/sample1.book vendored Normal file
View File

Binary file not shown.

336
format/apple/bookmark/testdata/sample1.fqtest vendored Normal file
View File

@ -0,0 +1,336 @@
$ fq dv sample1.book
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: sample1.book (apple_bookmark) 0x0-0x22f.7 (560)
| | | header{}: 0x0-0x2f.7 (48)
0x000|62 6f 6f 6b |book | magic: "book" (valid) 0x0-0x3.7 (4)
0x000| 30 02 00 00 | 0... | total_size: 560 0x4-0x7.7 (4)
0x000| 00 00 04 10 | .... | unknown: 268697600 0x8-0xb.7 (4)
0x000| 30 00 00 00| 0...| header_size: 48 (valid) 0xc-0xf.7 (4)
0x010|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| reserved: raw bits 0x10-0x2f.7 (32)
0x020|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
0x030|50 01 00 00 |P... | first_toc_offset: 336 0x30-0x33.7 (4)
| | | bookmark_entries[0:13]: 0x34-0x22f.7 (508)
| | | [0]{}: entry 0x40-0x19f.7 (352)
| | | record{}: 0x40-0x7b.7 (60)
| | | data[0:2]: 0x40-0x7b.7 (60)
| | | [0]{}: element 0x40-0x77.7 (56)
| | | record{}: 0x40-0x57.7 (24)
0x040|0c 00 00 00 |.... | length: 12 0x40-0x43.7 (4)
0x040| 01 01 00 00 | .... | type: "string" (257) (UTF-8 String) 0x44-0x47.7 (4)
0x040| 41 70 70 6c 69 63 61 74| Applicat| data: "Applications" 0x48-0x53.7 (12)
0x050|69 6f 6e 73 |ions |
0x050| 0d 00 00 00 | .... | alignment_bytes: raw bits 0x54-0x57.7 (4)
0x070| 10 00 00 00 | .... | offset: 16 0x74-0x77.7 (4)
| | | [1]{}: element 0x54-0x7b.7 (40)
| | | record{}: 0x54-0x6b.7 (24)
0x050| 0d 00 00 00 | .... | length: 13 0x54-0x57.7 (4)
0x050| 01 01 00 00 | .... | type: "string" (257) (UTF-8 String) 0x58-0x5b.7 (4)
0x050| 42 69 74 77| Bitw| data: "Bitwarden.app" 0x5c-0x68.7 (13)
0x060|61 72 64 65 6e 2e 61 70 70 |arden.app |
0x060| 00 00 00 | ... | alignment_bytes: raw bits 0x69-0x6b.7 (3)
0x070| 24 00 00 00 | $... | offset: 36 0x78-0x7b.7 (4)
0x060| 08 00 00 00| ....| length: 8 0x6c-0x6f.7 (4)
0x070|01 06 00 00 |.... | type: "array" (1537) (Array of 4-byte offsets to data items) 0x70-0x73.7 (4)
0x190| 04 10 00 00 | .... | key: "target_path" (4100) (Array of individual path components) 0x194-0x197.7 (4)
0x190| 3c 00 00 00 | <... | offset_to_record: 60 0x198-0x19b.7 (4)
0x190| 00 00 00 00| ....| unused: 0 0x19c-0x19f.7 (4)
| | | [1]{}: entry 0x7c-0x1ab.7 (304)
| | | record{}: 0x7c-0xab.7 (48)
| | | data[0:2]: 0x7c-0xab.7 (48)
| | | [0]{}: element 0x7c-0xa7.7 (44)
| | | record{}: 0x7c-0x8b.7 (16)
0x070| 08 00 00 00| ....| length: 8 0x7c-0x7f.7 (4)
0x080|04 03 00 00 |.... | type: "long" (772) ((signed 64-bit) 8-byte number) 0x80-0x83.7 (4)
0x080| 67 f5 50 03 00 00 00 00 | g.P..... | data: 55637351 0x84-0x8b.7 (8)
0x0a0| 4c 00 00 00 | L... | offset: 76 0xa4-0xa7.7 (4)
| | | [1]{}: element 0x8c-0xab.7 (32)
| | | record{}: 0x8c-0x9b.7 (16)
0x080| 08 00 00 00| ....| length: 8 0x8c-0x8f.7 (4)
0x090|04 03 00 00 |.... | type: "long" (772) ((signed 64-bit) 8-byte number) 0x90-0x93.7 (4)
0x090| fb e9 3d 03 00 00 00 00 | ..=..... | data: 54389243 0x94-0x9b.7 (8)
0x0a0| 5c 00 00 00 | \... | offset: 92 0xa8-0xab.7 (4)
0x090| 08 00 00 00| ....| length: 8 0x9c-0x9f.7 (4)
0x0a0|01 06 00 00 |.... | type: "array" (1537) (Array of 4-byte offsets to data items) 0xa0-0xa3.7 (4)
0x1a0|05 10 00 00 |.... | key: "target_cnid_path" (4101) (Array of CNIDs) 0x1a0-0x1a3.7 (4)
0x1a0| 6c 00 00 00 | l... | offset_to_record: 108 0x1a4-0x1a7.7 (4)
0x1a0| 00 00 00 00 | .... | unused: 0 0x1a8-0x1ab.7 (4)
| | | [2]{}: entry 0xbc-0x1b7.7 (252)
| | | record{}: 0xbc-0xdb.7 (32)
0x0b0| 18 00 00 00| ....| length: 24 (valid) 0xbc-0xbf.7 (4)
0x0c0|01 02 00 00 |.... | raw_type: "data" (513) (valid) 0xc0-0xc3.7 (4)
| | | type: "flag_data" 0xc4-NA (0)
| | | property_flags{}: 0xc4-0xcb.7 (8)
0x0c0| 02 | . | is_hidden: false 0xc4-0xc4 (0.1)
0x0c0| 02 | . | is_user_immutable: false 0xc4.1-0xc4.1 (0.1)
0x0c0| 02 | . | is_system_immutable: false 0xc4.2-0xc4.2 (0.1)
0x0c0| 02 | . | is_package: false 0xc4.3-0xc4.3 (0.1)
0x0c0| 02 | . | is_volume: false 0xc4.4-0xc4.4 (0.1)
0x0c0| 02 | . | is_symbolic_link: false 0xc4.5-0xc4.5 (0.1)
0x0c0| 02 | . | is_directory: true 0xc4.6-0xc4.6 (0.1)
0x0c0| 02 | . | is_regular_file: false 0xc4.7-0xc4.7 (0.1)
0x0c0| 00 | . | is_alias_file: false 0xc5-0xc5 (0.1)
0x0c0| 00 | . | is_executable: false 0xc5.1-0xc5.1 (0.1)
0x0c0| 00 | . | is_writeable: false 0xc5.2-0xc5.2 (0.1)
0x0c0| 00 | . | is_readable: false 0xc5.3-0xc5.3 (0.1)
0x0c0| 00 | . | can_set_hidden_extension: false 0xc5.4-0xc5.4 (0.1)
0x0c0| 00 | . | is_compressed: false 0xc5.5-0xc5.5 (0.1)
0x0c0| 00 | . | is_application: false 0xc5.6-0xc5.6 (0.1)
0x0c0| 00 | . | has_hidden_extension: false 0xc5.7-0xc5.7 (0.1)
0x0c0| 00 | . | reserved_bits_0: raw bits 0xc6-0xc6.6 (0.7)
0x0c0| 00 | . | is_mount_trigger: false 0xc6.7-0xc6.7 (0.1)
0x0c0| 00 00 00 00 00 | ..... | reserved: raw bits 0xc7-0xcb.7 (5)
| | | enabled_property_flags{}: 0xcc-0xd3.7 (8)
0x0c0| 0f | . | is_hidden: false 0xcc-0xcc (0.1)
0x0c0| 0f | . | is_user_immutable: false 0xcc.1-0xcc.1 (0.1)
0x0c0| 0f | . | is_system_immutable: false 0xcc.2-0xcc.2 (0.1)
0x0c0| 0f | . | is_package: false 0xcc.3-0xcc.3 (0.1)
0x0c0| 0f | . | is_volume: true 0xcc.4-0xcc.4 (0.1)
0x0c0| 0f | . | is_symbolic_link: true 0xcc.5-0xcc.5 (0.1)
0x0c0| 0f | . | is_directory: true 0xcc.6-0xcc.6 (0.1)
0x0c0| 0f | . | is_regular_file: true 0xcc.7-0xcc.7 (0.1)
0x0c0| 00 | . | is_alias_file: false 0xcd-0xcd (0.1)
0x0c0| 00 | . | is_executable: false 0xcd.1-0xcd.1 (0.1)
0x0c0| 00 | . | is_writeable: false 0xcd.2-0xcd.2 (0.1)
0x0c0| 00 | . | is_readable: false 0xcd.3-0xcd.3 (0.1)
0x0c0| 00 | . | can_set_hidden_extension: false 0xcd.4-0xcd.4 (0.1)
0x0c0| 00 | . | is_compressed: false 0xcd.5-0xcd.5 (0.1)
0x0c0| 00 | . | is_application: false 0xcd.6-0xcd.6 (0.1)
0x0c0| 00 | . | has_hidden_extension: false 0xcd.7-0xcd.7 (0.1)
0x0c0| 00 | . | reserved_bits_0: raw bits 0xce-0xce.6 (0.7)
0x0c0| 00 | . | is_mount_trigger: false 0xce.7-0xce.7 (0.1)
0x0c0| 00| .| reserved: raw bits 0xcf-0xd3.7 (5)
0x0d0|00 00 00 00 |.... |
0x0d0| 00 00 00 00 00 00 00 00 | ........ | reserved: raw bits 0xd4-0xdb.7 (8)
0x1a0| 10 10 00 00| ....| key: "target_flags" (4112) (flag bitfield) 0x1ac-0x1af.7 (4)
0x1b0|8c 00 00 00 |.... | offset_to_record: 140 0x1b0-0x1b3.7 (4)
0x1b0| 00 00 00 00 | .... | unused: 0 0x1b4-0x1b7.7 (4)
| | | [3]{}: entry 0xac-0x1c3.7 (280)
| | | record{}: 0xac-0xbb.7 (16)
0x0a0| 08 00 00 00| ....| length: 8 0xac-0xaf.7 (4)
0x0b0|00 04 00 00 |.... | type: "date" (1024) (Big-endian IEEE double precision seconds since 2001-01-01 00:00:00 UTC) 0xb0-0xb3.7 (4)
0x0b0| 41 c4 81 02 d0 00 00 00 | A....... | data: 6.87998368e+08 (2022-10-20T22:39:28Z) 0xb4-0xbb.7 (8)
0x1b0| 40 10 00 00 | @... | key: "target_creation_date" (4160) (Date) 0x1b8-0x1bb.7 (4)
0x1b0| 7c 00 00 00| |...| offset_to_record: 124 0x1bc-0x1bf.7 (4)
0x1c0|00 00 00 00 |.... | unused: 0 0x1c0-0x1c3.7 (4)
| | | [4]{}: entry 0x16c-0x1cf.7 (100)
| | | record{}: 0x16c-0x177.7 (12)
0x160| 01 00 00 00| ....| length: 1 0x16c-0x16f.7 (4)
0x170|01 01 00 00 |.... | type: "string" (257) (UTF-8 String) 0x170-0x173.7 (4)
0x170| 2f | / | data: "/" 0x174-0x174.7 (1)
0x170| 00 00 00 | ... | alignment_bytes: raw bits 0x175-0x177.7 (3)
0x1c0| 02 20 00 00 | . .. | key: "volume_path" (8194) (Array of individual path components) 0x1c4-0x1c7.7 (4)
0x1c0| 3c 01 00 00 | <... | offset_to_record: 316 0x1c8-0x1cb.7 (4)
0x1c0| 00 00 00 00| ....| unused: 0 0x1cc-0x1cf.7 (4)
| | | [5]{}: entry 0xdc-0x1db.7 (256)
| | | record{}: 0xdc-0xeb.7 (16)
0x0d0| 08 00 00 00| ....| length: 8 0xdc-0xdf.7 (4)
0x0e0|01 09 00 00 |.... | type: "url" (2305) (UTF-8 string) 0xe0-0xe3.7 (4)
0x0e0| 66 69 6c 65 3a 2f 2f 2f | file:/// | data: "file:///" 0xe4-0xeb.7 (8)
0x1d0|05 20 00 00 |. .. | key: "volume_url" (8197) (URL of volume root) 0x1d0-0x1d3.7 (4)
0x1d0| ac 00 00 00 | .... | offset_to_record: 172 0x1d4-0x1d7.7 (4)
0x1d0| 00 00 00 00 | .... | unused: 0 0x1d8-0x1db.7 (4)
| | | [6]{}: entry 0xec-0x1e7.7 (252)
| | | record{}: 0xec-0x103.7 (24)
0x0e0| 0c 00 00 00| ....| length: 12 0xec-0xef.7 (4)
0x0f0|01 01 00 00 |.... | type: "string" (257) (UTF-8 String) 0xf0-0xf3.7 (4)
0x0f0| 4d 61 63 69 6e 74 6f 73 68 20 48 44| Macintosh HD| data: "Macintosh HD" 0xf4-0xff.7 (12)
0x100|08 00 00 00 |.... | alignment_bytes: raw bits 0x100-0x103.7 (4)
0x1d0| 10 20 00 00| . ..| key: "volume_name" (8208) (String) 0x1dc-0x1df.7 (4)
0x1e0|bc 00 00 00 |.... | offset_to_record: 188 0x1e0-0x1e3.7 (4)
0x1e0| 00 00 00 00 | .... | unused: 0 0x1e4-0x1e7.7 (4)
| | | [7]{}: entry 0x120-0x1f3.7 (212)
| | | record{}: 0x120-0x14f.7 (48)
0x120|24 00 00 00 |$... | length: 36 0x120-0x123.7 (4)
0x120| 01 01 00 00 | .... | type: "string" (257) (UTF-8 String) 0x124-0x127.7 (4)
0x120| 41 41 41 41 41 41 41 41| AAAAAAAA| data: "AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA" 0x128-0x14b.7 (36)
0x130|2d 41 41 41 41 2d 41 41 41 41 2d 41 41 41 41 2d|-AAAA-AAAA-AAAA-|
0x140|41 41 41 41 41 41 41 41 41 41 41 41 |AAAAAAAAAAAA |
0x140| 18 00 00 00| ....| alignment_bytes: raw bits 0x14c-0x14f.7 (4)
0x1e0| 11 20 00 00 | . .. | key: "volume_uuid" (8209) (String UUID) 0x1e8-0x1eb.7 (4)
0x1e0| f0 00 00 00| ....| offset_to_record: 240 0x1ec-0x1ef.7 (4)
0x1f0|00 00 00 00 |.... | unused: 0 0x1f0-0x1f3.7 (4)
| | | [8]{}: entry 0x100-0x1ff.7 (256)
| | | record{}: 0x100-0x10f.7 (16)
0x100|08 00 00 00 |.... | length: 8 0x100-0x103.7 (4)
0x100| 04 03 00 00 | .... | type: "long" (772) ((signed 64-bit) 8-byte number) 0x104-0x107.7 (4)
0x100| 00 a0 20 68 74 00 00 00| .. ht...| data: 499963174912 0x108-0x10f.7 (8)
0x1f0| 12 20 00 00 | . .. | key: "volume_size" (8210) (8-byte integer) 0x1f4-0x1f7.7 (4)
0x1f0| d0 00 00 00 | .... | offset_to_record: 208 0x1f8-0x1fb.7 (4)
0x1f0| 00 00 00 00| ....| unused: 0 0x1fc-0x1ff.7 (4)
| | | [9]{}: entry 0x110-0x20b.7 (252)
| | | record{}: 0x110-0x11f.7 (16)
0x110|08 00 00 00 |.... | length: 8 0x110-0x113.7 (4)
0x110| 00 04 00 00 | .... | type: "date" (1024) (Big-endian IEEE double precision seconds since 2001-01-01 00:00:00 UTC) 0x114-0x117.7 (4)
0x110| 41 c1 de 44 80 00 00 00| A..D....| data: 5.995584e+08 (2020-01-01T08:00:00Z) 0x118-0x11f.7 (8)
0x200|13 20 00 00 |. .. | key: "volume_creation_date" (8211) (Date) 0x200-0x203.7 (4)
0x200| e0 00 00 00 | .... | offset_to_record: 224 0x204-0x207.7 (4)
0x200| 00 00 00 00 | .... | unused: 0 0x208-0x20b.7 (4)
| | | [10]{}: entry 0x14c-0x217.7 (204)
| | | record{}: 0x14c-0x16b.7 (32)
0x140| 18 00 00 00| ....| length: 24 (valid) 0x14c-0x14f.7 (4)
0x150|01 02 00 00 |.... | raw_type: "data" (513) (valid) 0x150-0x153.7 (4)
| | | type: "flag_data" 0x154-NA (0)
| | | property_flags{}: 0x154-0x15b.7 (8)
0x150| 81 | . | is_internal: true 0x154-0x154 (0.1)
0x150| 81 | . | is_removable: false 0x154.1-0x154.1 (0.1)
0x150| 81 | . | is_ejectable: false 0x154.2-0x154.2 (0.1)
0x150| 81 | . | is_quarantined: false 0x154.3-0x154.3 (0.1)
0x150| 81 | . | is_read_only: false 0x154.4-0x154.4 (0.1)
0x150| 81 | . | dont_browse: false 0x154.5-0x154.5 (0.1)
0x150| 81 | . | is_automount: false 0x154.6-0x154.6 (0.1)
0x150| 81 | . | is_local: true 0x154.7-0x154.7 (0.1)
0x150| 00 | . | is_dvd: false 0x155-0x155 (0.1)
0x150| 00 | . | is_cd: false 0x155.1-0x155.1 (0.1)
0x150| 00 | . | is_idisk: false 0x155.2-0x155.2 (0.1)
0x150| 00 | . | is_ipod: false 0x155.3-0x155.3 (0.1)
0x150| 00 | . | is_local_idisk_mirror: false 0x155.4-0x155.4 (0.1)
0x150| 00 | . | is_file_vault: false 0x155.5-0x155.5 (0.1)
0x150| 00 | . | is_disk_image: false 0x155.6-0x155.6 (0.1)
0x150| 00 | . | is_external: false 0x155.7-0x155.7 (0.1)
0x150| 00 | . | reserved_0: raw bits 0x156-0x156.6 (0.7)
0x150| 00 | . | is_device_file_system: false 0x156.7-0x156.7 (0.1)
0x150| 00 | . | reserved_1: raw bits 0x157-0x157.7 (1)
0x150| 01 | . | supports_read_dir_attr: false 0x158-0x158 (0.1)
0x150| 01 | . | supports_copy_file: false 0x158.1-0x158.1 (0.1)
0x150| 01 | . | supports_deny_modes: false 0x158.2-0x158.2 (0.1)
0x150| 01 | . | supports_symbolic_links: false 0x158.3-0x158.3 (0.1)
0x150| 01 | . | reserved_2: false 0x158.4-0x158.4 (0.1)
0x150| 01 | . | supports_exchange: false 0x158.5-0x158.5 (0.1)
0x150| 01 | . | supports_search_fs: false 0x158.6-0x158.6 (0.1)
0x150| 01 | . | supports_persistent_ids: true 0x158.7-0x158.7 (0.1)
0x150| 00 | . | supports_extended_security: false 0x159-0x159 (0.1)
0x150| 00 | . | has_no_root_directory_times: false 0x159.1-0x159.1 (0.1)
0x150| 00 | . | supports_flock: false 0x159.2-0x159.2 (0.1)
0x150| 00 | . | supports_case_preserved_names: false 0x159.3-0x159.3 (0.1)
0x150| 00 | . | supports_case_sensitive_names: false 0x159.4-0x159.4 (0.1)
0x150| 00 | . | supports_fast_stat_fs: false 0x159.5-0x159.5 (0.1)
0x150| 00 | . | supports_rename: false 0x159.6-0x159.6 (0.1)
0x150| 00 | . | supports_journaling: false 0x159.7-0x159.7 (0.1)
0x150| 00 | . | supports_zero_runs: false 0x15a-0x15a (0.1)
0x150| 00 | . | supports_sparse_files: false 0x15a.1-0x15a.1 (0.1)
0x150| 00 | . | is_journaling: false 0x15a.2-0x15a.2 (0.1)
0x150| 00 | . | reserved_3: false 0x15a.3-0x15a.3 (0.1)
0x150| 00 | . | supports_path_from_id: false 0x15a.4-0x15a.4 (0.1)
0x150| 00 | . | supports_mandatory_byte_range_locks: false 0x15a.5-0x15a.5 (0.1)
0x150| 00 | . | supports_hard_links: false 0x15a.6-0x15a.6 (0.1)
0x150| 00 | . | supports_2_tb_file_size: false 0x15a.7-0x15a.7 (0.1)
0x150| 00 | . | reserved_4: raw bits 0x15b-0x15b.2 (0.3)
0x150| 00 | . | has64_bit_object_ids: false 0x15b.3-0x15b.3 (0.1)
0x150| 00 | . | supports_decmp_fs_compression: false 0x15b.4-0x15b.4 (0.1)
0x150| 00 | . | supports_hidden_files: false 0x15b.5-0x15b.5 (0.1)
0x150| 00 | . | supports_remote_events: false 0x15b.6-0x15b.6 (0.1)
0x150| 00 | . | supports_volume_sizes: false 0x15b.7-0x15b.7 (0.1)
| | | enabled_property_flags{}: 0x15c-0x163.7 (8)
0x150| ef | . | is_internal: true 0x15c-0x15c (0.1)
0x150| ef | . | is_removable: true 0x15c.1-0x15c.1 (0.1)
0x150| ef | . | is_ejectable: true 0x15c.2-0x15c.2 (0.1)
0x150| ef | . | is_quarantined: false 0x15c.3-0x15c.3 (0.1)
0x150| ef | . | is_read_only: true 0x15c.4-0x15c.4 (0.1)
0x150| ef | . | dont_browse: true 0x15c.5-0x15c.5 (0.1)
0x150| ef | . | is_automount: true 0x15c.6-0x15c.6 (0.1)
0x150| ef | . | is_local: true 0x15c.7-0x15c.7 (0.1)
0x150| 13 | . | is_dvd: false 0x15d-0x15d (0.1)
0x150| 13 | . | is_cd: false 0x15d.1-0x15d.1 (0.1)
0x150| 13 | . | is_idisk: false 0x15d.2-0x15d.2 (0.1)
0x150| 13 | . | is_ipod: true 0x15d.3-0x15d.3 (0.1)
0x150| 13 | . | is_local_idisk_mirror: false 0x15d.4-0x15d.4 (0.1)
0x150| 13 | . | is_file_vault: false 0x15d.5-0x15d.5 (0.1)
0x150| 13 | . | is_disk_image: true 0x15d.6-0x15d.6 (0.1)
0x150| 13 | . | is_external: true 0x15d.7-0x15d.7 (0.1)
0x150| 00 | . | reserved_0: raw bits 0x15e-0x15e.6 (0.7)
0x150| 00 | . | is_device_file_system: false 0x15e.7-0x15e.7 (0.1)
0x150| 00| .| reserved_1: raw bits 0x15f-0x15f.7 (1)
0x160|01 |. | supports_read_dir_attr: false 0x160-0x160 (0.1)
0x160|01 |. | supports_copy_file: false 0x160.1-0x160.1 (0.1)
0x160|01 |. | supports_deny_modes: false 0x160.2-0x160.2 (0.1)
0x160|01 |. | supports_symbolic_links: false 0x160.3-0x160.3 (0.1)
0x160|01 |. | reserved_2: false 0x160.4-0x160.4 (0.1)
0x160|01 |. | supports_exchange: false 0x160.5-0x160.5 (0.1)
0x160|01 |. | supports_search_fs: false 0x160.6-0x160.6 (0.1)
0x160|01 |. | supports_persistent_ids: true 0x160.7-0x160.7 (0.1)
0x160| 00 | . | supports_extended_security: false 0x161-0x161 (0.1)
0x160| 00 | . | has_no_root_directory_times: false 0x161.1-0x161.1 (0.1)
0x160| 00 | . | supports_flock: false 0x161.2-0x161.2 (0.1)
0x160| 00 | . | supports_case_preserved_names: false 0x161.3-0x161.3 (0.1)
0x160| 00 | . | supports_case_sensitive_names: false 0x161.4-0x161.4 (0.1)
0x160| 00 | . | supports_fast_stat_fs: false 0x161.5-0x161.5 (0.1)
0x160| 00 | . | supports_rename: false 0x161.6-0x161.6 (0.1)
0x160| 00 | . | supports_journaling: false 0x161.7-0x161.7 (0.1)
0x160| 00 | . | supports_zero_runs: false 0x162-0x162 (0.1)
0x160| 00 | . | supports_sparse_files: false 0x162.1-0x162.1 (0.1)
0x160| 00 | . | is_journaling: false 0x162.2-0x162.2 (0.1)
0x160| 00 | . | reserved_3: false 0x162.3-0x162.3 (0.1)
0x160| 00 | . | supports_path_from_id: false 0x162.4-0x162.4 (0.1)
0x160| 00 | . | supports_mandatory_byte_range_locks: false 0x162.5-0x162.5 (0.1)
0x160| 00 | . | supports_hard_links: false 0x162.6-0x162.6 (0.1)
0x160| 00 | . | supports_2_tb_file_size: false 0x162.7-0x162.7 (0.1)
0x160| 00 | . | reserved_4: raw bits 0x163-0x163.2 (0.3)
0x160| 00 | . | has64_bit_object_ids: false 0x163.3-0x163.3 (0.1)
0x160| 00 | . | supports_decmp_fs_compression: false 0x163.4-0x163.4 (0.1)
0x160| 00 | . | supports_hidden_files: false 0x163.5-0x163.5 (0.1)
0x160| 00 | . | supports_remote_events: false 0x163.6-0x163.6 (0.1)
0x160| 00 | . | supports_volume_sizes: false 0x163.7-0x163.7 (0.1)
0x160| 00 00 00 00 00 00 00 00 | ........ | reserved: raw bits 0x164-0x16b.7 (8)
0x200| 20 20 00 00| ..| key: "volume_flags" (8224) (flag bitfield) 0x20c-0x20f.7 (4)
0x210|1c 01 00 00 |.... | offset_to_record: 284 0x210-0x213.7 (4)
0x210| 00 00 00 00 | .... | unused: 0 0x214-0x217.7 (4)
| | | [11]{}: entry 0x178-0x223.7 (172)
| | | record{}: 0x178-0x17f.7 (8)
0x170| 00 00 00 00 | .... | length: 0 0x178-0x17b.7 (4)
0x170| 01 05 00 00| ....| type: "boolean_true" (1281) (True) 0x17c-0x17f.7 (4)
0x210| 30 20 00 00 | 0 .. | key: "volume_is_root" (8240) (True if the volume was the filesystem root) 0x218-0x21b.7 (4)
0x210| 48 01 00 00| H...| offset_to_record: 328 0x21c-0x21f.7 (4)
0x220|00 00 00 00 |.... | unused: 0 0x220-0x223.7 (4)
| | | [12]{}: entry 0x34-0x22f.7 (508)
| | | record{}: 0x34-0x3f.7 (12)
0x030| 04 00 00 00 | .... | length: 4 0x34-0x37.7 (4)
0x030| 03 03 00 00 | .... | type: "int" (771) ((signed 32-bit) 4-byte number) 0x38-0x3b.7 (4)
0x030| 00 00 00 20| ... | data: 536870912 0x3c-0x3f.7 (4)
0x220| 10 d0 00 00 | .... | key: "creation_options" (53264) (Integer containing flags passed to CFURLCreateBookmarkData) 0x224-0x227.7 (4)
0x220| 04 00 00 00 | .... | offset_to_record: 4 0x228-0x22b.7 (4)
0x220| 00 00 00 00| ....| unused: 0 0x22c-0x22f.7 (4)
| | | toc_headers[0:1]: 0x180-0x193.7 (20)
| | | [0]{}: toc_header 0x180-0x193.7 (20)
0x180|a8 00 00 00 |.... | toc_size: 168 0x180-0x183.7 (4)
0x180| fe ff ff ff | .... | magic: 4294967294 (valid) 0x184-0x187.7 (4)
0x180| 01 00 00 00 | .... | identifier: 1 0x188-0x18b.7 (4)
0x180| 00 00 00 00| ....| next_toc_offset: 0 0x18c-0x18f.7 (4)
0x190|0d 00 00 00 |.... | num_entries_in_toc: 13 0x190-0x193.7 (4)
$ fq torepr sample1.book
{
"creation_options": 536870912,
"target_cnid_path": [
55637351,
54389243
],
"target_creation_date": 687998368,
"target_flags": {
"is_directory": true,
"is_regular_file": false,
"is_symbolic_link": false,
"is_volume": false
},
"target_path": [
"Applications",
"Bitwarden.app"
],
"volume_creation_date": 599558400,
"volume_flags": {
"dont_browse": false,
"is_automount": false,
"is_disk_image": false,
"is_ejectable": false,
"is_external": false,
"is_internal": true,
"is_ipod": false,
"is_local": true,
"is_read_only": false,
"is_removable": false,
"supports_persistent_ids": true
},
"volume_is_root": true,
"volume_name": "Macintosh HD",
"volume_path": "/",
"volume_size": 499963174912,
"volume_url": "file:///",
"volume_uuid": "AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA"
}

BIN
format/apple/bookmark/testdata/sample2.book vendored Normal file
View File

Binary file not shown.

398
format/apple/bookmark/testdata/sample2.fqtest vendored Normal file
View File

@ -0,0 +1,398 @@
$ fq dv sample2.book
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: sample2.book (apple_bookmark) 0x0-0x2ab.7 (684)
| | | header{}: 0x0-0x2f.7 (48)
0x000|62 6f 6f 6b |book | magic: "book" (valid) 0x0-0x3.7 (4)
0x000| ac 02 00 00 | .... | total_size: 684 0x4-0x7.7 (4)
0x000| 00 00 04 10 | .... | unknown: 268697600 0x8-0xb.7 (4)
0x000| 30 00 00 00| 0...| header_size: 48 (valid) 0xc-0xf.7 (4)
0x010|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| reserved: raw bits 0x10-0x2f.7 (32)
0x020|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
0x030|90 01 00 00 |.... | first_toc_offset: 400 0x30-0x33.7 (4)
| | | bookmark_entries[0:18]: 0x34-0x2ab.7 (632)
| | | [0]{}: entry 0x40-0x1df.7 (416)
| | | record{}: 0x40-0x8b.7 (76)
| | | data[0:3]: 0x40-0x8b.7 (76)
| | | [0]{}: element 0x40-0x83.7 (68)
| | | record{}: 0x40-0x4f.7 (16)
0x040|05 00 00 00 |.... | length: 5 0x40-0x43.7 (4)
0x040| 01 01 00 00 | .... | type: "string" (257) (UTF-8 String) 0x44-0x47.7 (4)
0x040| 55 73 65 72 73 | Users | data: "Users" 0x48-0x4c.7 (5)
0x040| 00 00 00| ...| alignment_bytes: raw bits 0x4d-0x4f.7 (3)
0x080|10 00 00 00 |.... | offset: 16 0x80-0x83.7 (4)
| | | [1]{}: element 0x50-0x87.7 (56)
| | | record{}: 0x50-0x67.7 (24)
0x050|0d 00 00 00 |.... | length: 13 0x50-0x53.7 (4)
0x050| 01 01 00 00 | .... | type: "string" (257) (UTF-8 String) 0x54-0x57.7 (4)
0x050| 64 61 76 69 64 6d 63 64| davidmcd| data: "davidmcdonald" 0x58-0x64.7 (13)
0x060|6f 6e 61 6c 64 |onald |
0x060| 00 00 00 | ... | alignment_bytes: raw bits 0x65-0x67.7 (3)
0x080| 20 00 00 00 | ... | offset: 32 0x84-0x87.7 (4)
| | | [2]{}: element 0x68-0x8b.7 (36)
| | | record{}: 0x68-0x77.7 (16)
0x060| 07 00 00 00 | .... | length: 7 0x68-0x6b.7 (4)
0x060| 01 01 00 00| ....| type: "string" (257) (UTF-8 String) 0x6c-0x6f.7 (4)
0x070|68 78 73 74 6f 72 65 |hxstore | data: "hxstore" 0x70-0x76.7 (7)
0x070| 00 | . | alignment_bytes: raw bits 0x77-0x77.7 (1)
0x080| 38 00 00 00 | 8... | offset: 56 0x88-0x8b.7 (4)
0x070| 0c 00 00 00 | .... | length: 12 0x78-0x7b.7 (4)
0x070| 01 06 00 00| ....| type: "array" (1537) (Array of 4-byte offsets to data items) 0x7c-0x7f.7 (4)
0x1d0| 04 10 00 00 | .... | key: "target_path" (4100) (Array of individual path components) 0x1d4-0x1d7.7 (4)
0x1d0| 48 00 00 00 | H... | offset_to_record: 72 0x1d8-0x1db.7 (4)
0x1d0| 00 00 00 00| ....| unused: 0 0x1dc-0x1df.7 (4)
| | | [1]{}: entry 0x8c-0x1eb.7 (352)
| | | record{}: 0x8c-0xcf.7 (68)
| | | data[0:3]: 0x8c-0xcf.7 (68)
| | | [0]{}: element 0x8c-0xc7.7 (60)
| | | record{}: 0x8c-0x9b.7 (16)
0x080| 08 00 00 00| ....| length: 8 0x8c-0x8f.7 (4)
0x090|04 03 00 00 |.... | type: "long" (772) ((signed 64-bit) 8-byte number) 0x90-0x93.7 (4)
0x090| 0b 5b 00 00 00 00 00 00 | .[...... | data: 23307 0x94-0x9b.7 (8)
0x0c0| 5c 00 00 00 | \... | offset: 92 0xc4-0xc7.7 (4)
| | | [1]{}: element 0x9c-0xcb.7 (48)
| | | record{}: 0x9c-0xab.7 (16)
0x090| 08 00 00 00| ....| length: 8 0x9c-0x9f.7 (4)
0x0a0|04 03 00 00 |.... | type: "long" (772) ((signed 64-bit) 8-byte number) 0xa0-0xa3.7 (4)
0x0a0| 1c 9e 08 00 00 00 00 00 | ........ | data: 564764 0xa4-0xab.7 (8)
0x0c0| 6c 00 00 00 | l... | offset: 108 0xc8-0xcb.7 (4)
| | | [2]{}: element 0xac-0xcf.7 (36)
| | | record{}: 0xac-0xbb.7 (16)
0x0a0| 08 00 00 00| ....| length: 8 0xac-0xaf.7 (4)
0x0b0|04 03 00 00 |.... | type: "long" (772) ((signed 64-bit) 8-byte number) 0xb0-0xb3.7 (4)
0x0b0| c7 c5 44 03 00 00 00 00 | ..D..... | data: 54838727 0xb4-0xbb.7 (8)
0x0c0| 7c 00 00 00| |...| offset: 124 0xcc-0xcf.7 (4)
0x0b0| 0c 00 00 00| ....| length: 12 0xbc-0xbf.7 (4)
0x0c0|01 06 00 00 |.... | type: "array" (1537) (Array of 4-byte offsets to data items) 0xc0-0xc3.7 (4)
0x1e0|05 10 00 00 |.... | key: "target_cnid_path" (4101) (Array of CNIDs) 0x1e0-0x1e3.7 (4)
0x1e0| 8c 00 00 00 | .... | offset_to_record: 140 0x1e4-0x1e7.7 (4)
0x1e0| 00 00 00 00 | .... | unused: 0 0x1e8-0x1eb.7 (4)
| | | [2]{}: entry 0xe0-0x1f7.7 (280)
| | | record{}: 0xe0-0xff.7 (32)
0x0e0|18 00 00 00 |.... | length: 24 (valid) 0xe0-0xe3.7 (4)
0x0e0| 01 02 00 00 | .... | raw_type: "data" (513) (valid) 0xe4-0xe7.7 (4)
| | | type: "flag_data" 0xe8-NA (0)
| | | property_flags{}: 0xe8-0xef.7 (8)
0x0e0| 02 | . | is_hidden: false 0xe8-0xe8 (0.1)
0x0e0| 02 | . | is_user_immutable: false 0xe8.1-0xe8.1 (0.1)
0x0e0| 02 | . | is_system_immutable: false 0xe8.2-0xe8.2 (0.1)
0x0e0| 02 | . | is_package: false 0xe8.3-0xe8.3 (0.1)
0x0e0| 02 | . | is_volume: false 0xe8.4-0xe8.4 (0.1)
0x0e0| 02 | . | is_symbolic_link: false 0xe8.5-0xe8.5 (0.1)
0x0e0| 02 | . | is_directory: true 0xe8.6-0xe8.6 (0.1)
0x0e0| 02 | . | is_regular_file: false 0xe8.7-0xe8.7 (0.1)
0x0e0| 00 | . | is_alias_file: false 0xe9-0xe9 (0.1)
0x0e0| 00 | . | is_executable: false 0xe9.1-0xe9.1 (0.1)
0x0e0| 00 | . | is_writeable: false 0xe9.2-0xe9.2 (0.1)
0x0e0| 00 | . | is_readable: false 0xe9.3-0xe9.3 (0.1)
0x0e0| 00 | . | can_set_hidden_extension: false 0xe9.4-0xe9.4 (0.1)
0x0e0| 00 | . | is_compressed: false 0xe9.5-0xe9.5 (0.1)
0x0e0| 00 | . | is_application: false 0xe9.6-0xe9.6 (0.1)
0x0e0| 00 | . | has_hidden_extension: false 0xe9.7-0xe9.7 (0.1)
0x0e0| 00 | . | reserved_bits_0: raw bits 0xea-0xea.6 (0.7)
0x0e0| 00 | . | is_mount_trigger: false 0xea.7-0xea.7 (0.1)
0x0e0| 00 00 00 00 00| .....| reserved: raw bits 0xeb-0xef.7 (5)
| | | enabled_property_flags{}: 0xf0-0xf7.7 (8)
0x0f0|1f |. | is_hidden: false 0xf0-0xf0 (0.1)
0x0f0|1f |. | is_user_immutable: false 0xf0.1-0xf0.1 (0.1)
0x0f0|1f |. | is_system_immutable: false 0xf0.2-0xf0.2 (0.1)
0x0f0|1f |. | is_package: true 0xf0.3-0xf0.3 (0.1)
0x0f0|1f |. | is_volume: true 0xf0.4-0xf0.4 (0.1)
0x0f0|1f |. | is_symbolic_link: true 0xf0.5-0xf0.5 (0.1)
0x0f0|1f |. | is_directory: true 0xf0.6-0xf0.6 (0.1)
0x0f0|1f |. | is_regular_file: true 0xf0.7-0xf0.7 (0.1)
0x0f0| 02 | . | is_alias_file: false 0xf1-0xf1 (0.1)
0x0f0| 02 | . | is_executable: false 0xf1.1-0xf1.1 (0.1)
0x0f0| 02 | . | is_writeable: false 0xf1.2-0xf1.2 (0.1)
0x0f0| 02 | . | is_readable: false 0xf1.3-0xf1.3 (0.1)
0x0f0| 02 | . | can_set_hidden_extension: false 0xf1.4-0xf1.4 (0.1)
0x0f0| 02 | . | is_compressed: false 0xf1.5-0xf1.5 (0.1)
0x0f0| 02 | . | is_application: true 0xf1.6-0xf1.6 (0.1)
0x0f0| 02 | . | has_hidden_extension: false 0xf1.7-0xf1.7 (0.1)
0x0f0| 00 | . | reserved_bits_0: raw bits 0xf2-0xf2.6 (0.7)
0x0f0| 00 | . | is_mount_trigger: false 0xf2.7-0xf2.7 (0.1)
0x0f0| 00 00 00 00 00 | ..... | reserved: raw bits 0xf3-0xf7.7 (5)
0x0f0| 1a 02 00 00 00 00 00 00| ........| reserved: raw bits 0xf8-0xff.7 (8)
0x1e0| 10 10 00 00| ....| key: "target_flags" (4112) (flag bitfield) 0x1ec-0x1ef.7 (4)
0x1f0|b0 00 00 00 |.... | offset_to_record: 176 0x1f0-0x1f3.7 (4)
0x1f0| 00 00 00 00 | .... | unused: 0 0x1f4-0x1f7.7 (4)
| | | [3]{}: entry 0xd0-0x203.7 (308)
| | | record{}: 0xd0-0xdf.7 (16)
0x0d0|08 00 00 00 |.... | length: 8 0xd0-0xd3.7 (4)
0x0d0| 00 04 00 00 | .... | type: "date" (1024) (Big-endian IEEE double precision seconds since 2001-01-01 00:00:00 UTC) 0xd4-0xd7.7 (4)
0x0d0| 41 c4 8a 17 83 4d f0 44| A....M.D| data: 6.891886146088948e+08 (2022-11-03T17:16:54Z) 0xd8-0xdf.7 (8)
0x1f0| 40 10 00 00 | @... | key: "target_creation_date" (4160) (Date) 0x1f8-0x1fb.7 (4)
0x1f0| a0 00 00 00| ....| offset_to_record: 160 0x1fc-0x1ff.7 (4)
0x200|00 00 00 00 |.... | unused: 0 0x200-0x203.7 (4)
| | | [4]{}: entry 0x1ac-0x20f.7 (100)
| | | record{}: 0x1ac-0x1b7.7 (12)
0x1a0| 01 00 00 00| ....| length: 1 0x1ac-0x1af.7 (4)
0x1b0|01 01 00 00 |.... | type: "string" (257) (UTF-8 String) 0x1b0-0x1b3.7 (4)
0x1b0| 2f | / | data: "/" 0x1b4-0x1b4.7 (1)
0x1b0| 00 00 00 | ... | alignment_bytes: raw bits 0x1b5-0x1b7.7 (3)
0x200| 02 20 00 00 | . .. | key: "volume_path" (8194) (Array of individual path components) 0x204-0x207.7 (4)
0x200| 7c 01 00 00 | |... | offset_to_record: 380 0x208-0x20b.7 (4)
0x200| 00 00 00 00| ....| unused: 0 0x20c-0x20f.7 (4)
| | | [5]{}: entry 0x11c-0x21b.7 (256)
| | | record{}: 0x11c-0x12b.7 (16)
0x110| 08 00 00 00| ....| length: 8 0x11c-0x11f.7 (4)
0x120|01 09 00 00 |.... | type: "url" (2305) (UTF-8 string) 0x120-0x123.7 (4)
0x120| 66 69 6c 65 3a 2f 2f 2f | file:/// | data: "file:///" 0x124-0x12b.7 (8)
0x210|05 20 00 00 |. .. | key: "volume_url" (8197) (URL of volume root) 0x210-0x213.7 (4)
0x210| ec 00 00 00 | .... | offset_to_record: 236 0x214-0x217.7 (4)
0x210| 00 00 00 00 | .... | unused: 0 0x218-0x21b.7 (4)
| | | [6]{}: entry 0x12c-0x227.7 (252)
| | | record{}: 0x12c-0x143.7 (24)
0x120| 0c 00 00 00| ....| length: 12 0x12c-0x12f.7 (4)
0x130|01 01 00 00 |.... | type: "string" (257) (UTF-8 String) 0x130-0x133.7 (4)
0x130| 4d 61 63 69 6e 74 6f 73 68 20 48 44| Macintosh HD| data: "Macintosh HD" 0x134-0x13f.7 (12)
0x140|08 00 00 00 |.... | alignment_bytes: raw bits 0x140-0x143.7 (4)
0x210| 10 20 00 00| . ..| key: "volume_name" (8208) (String) 0x21c-0x21f.7 (4)
0x220|fc 00 00 00 |.... | offset_to_record: 252 0x220-0x223.7 (4)
0x220| 00 00 00 00 | .... | unused: 0 0x224-0x227.7 (4)
| | | [7]{}: entry 0x160-0x233.7 (212)
| | | record{}: 0x160-0x18f.7 (48)
0x160|24 00 00 00 |$... | length: 36 0x160-0x163.7 (4)
0x160| 01 01 00 00 | .... | type: "string" (257) (UTF-8 String) 0x164-0x167.7 (4)
0x160| 38 41 38 32 39 41 30 33| 8A829A03| data: "8A829A03-9B24-4085-8051-18978712E4BE" 0x168-0x18b.7 (36)
0x170|2d 39 42 32 34 2d 34 30 38 35 2d 38 30 35 31 2d|-9B24-4085-8051-|
0x180|31 38 39 37 38 37 31 32 45 34 42 45 |18978712E4BE |
0x180| 18 00 00 00| ....| alignment_bytes: raw bits 0x18c-0x18f.7 (4)
0x220| 11 20 00 00 | . .. | key: "volume_uuid" (8209) (String UUID) 0x228-0x22b.7 (4)
0x220| 30 01 00 00| 0...| offset_to_record: 304 0x22c-0x22f.7 (4)
0x230|00 00 00 00 |.... | unused: 0 0x230-0x233.7 (4)
| | | [8]{}: entry 0x140-0x23f.7 (256)
| | | record{}: 0x140-0x14f.7 (16)
0x140|08 00 00 00 |.... | length: 8 0x140-0x143.7 (4)
0x140| 04 03 00 00 | .... | type: "long" (772) ((signed 64-bit) 8-byte number) 0x144-0x147.7 (4)
0x140| 00 a0 20 68 74 00 00 00| .. ht...| data: 499963174912 0x148-0x14f.7 (8)
0x230| 12 20 00 00 | . .. | key: "volume_size" (8210) (8-byte integer) 0x234-0x237.7 (4)
0x230| 10 01 00 00 | .... | offset_to_record: 272 0x238-0x23b.7 (4)
0x230| 00 00 00 00| ....| unused: 0 0x23c-0x23f.7 (4)
| | | [9]{}: entry 0x150-0x24b.7 (252)
| | | record{}: 0x150-0x15f.7 (16)
0x150|08 00 00 00 |.... | length: 8 0x150-0x153.7 (4)
0x150| 00 04 00 00 | .... | type: "date" (1024) (Big-endian IEEE double precision seconds since 2001-01-01 00:00:00 UTC) 0x154-0x157.7 (4)
0x150| 41 c1 de 44 80 00 00 00| A..D....| data: 5.995584e+08 (2020-01-01T08:00:00Z) 0x158-0x15f.7 (8)
0x240|13 20 00 00 |. .. | key: "volume_creation_date" (8211) (Date) 0x240-0x243.7 (4)
0x240| 20 01 00 00 | ... | offset_to_record: 288 0x244-0x247.7 (4)
0x240| 00 00 00 00 | .... | unused: 0 0x248-0x24b.7 (4)
| | | [10]{}: entry 0x18c-0x257.7 (204)
| | | record{}: 0x18c-0x1ab.7 (32)
0x180| 18 00 00 00| ....| length: 24 (valid) 0x18c-0x18f.7 (4)
0x190|01 02 00 00 |.... | raw_type: "data" (513) (valid) 0x190-0x193.7 (4)
| | | type: "flag_data" 0x194-NA (0)
| | | property_flags{}: 0x194-0x19b.7 (8)
0x190| 81 | . | is_internal: true 0x194-0x194 (0.1)
0x190| 81 | . | is_removable: false 0x194.1-0x194.1 (0.1)
0x190| 81 | . | is_ejectable: false 0x194.2-0x194.2 (0.1)
0x190| 81 | . | is_quarantined: false 0x194.3-0x194.3 (0.1)
0x190| 81 | . | is_read_only: false 0x194.4-0x194.4 (0.1)
0x190| 81 | . | dont_browse: false 0x194.5-0x194.5 (0.1)
0x190| 81 | . | is_automount: false 0x194.6-0x194.6 (0.1)
0x190| 81 | . | is_local: true 0x194.7-0x194.7 (0.1)
0x190| 00 | . | is_dvd: false 0x195-0x195 (0.1)
0x190| 00 | . | is_cd: false 0x195.1-0x195.1 (0.1)
0x190| 00 | . | is_idisk: false 0x195.2-0x195.2 (0.1)
0x190| 00 | . | is_ipod: false 0x195.3-0x195.3 (0.1)
0x190| 00 | . | is_local_idisk_mirror: false 0x195.4-0x195.4 (0.1)
0x190| 00 | . | is_file_vault: false 0x195.5-0x195.5 (0.1)
0x190| 00 | . | is_disk_image: false 0x195.6-0x195.6 (0.1)
0x190| 00 | . | is_external: false 0x195.7-0x195.7 (0.1)
0x190| 00 | . | reserved_0: raw bits 0x196-0x196.6 (0.7)
0x190| 00 | . | is_device_file_system: false 0x196.7-0x196.7 (0.1)
0x190| 00 | . | reserved_1: raw bits 0x197-0x197.7 (1)
0x190| 01 | . | supports_read_dir_attr: false 0x198-0x198 (0.1)
0x190| 01 | . | supports_copy_file: false 0x198.1-0x198.1 (0.1)
0x190| 01 | . | supports_deny_modes: false 0x198.2-0x198.2 (0.1)
0x190| 01 | . | supports_symbolic_links: false 0x198.3-0x198.3 (0.1)
0x190| 01 | . | reserved_2: false 0x198.4-0x198.4 (0.1)
0x190| 01 | . | supports_exchange: false 0x198.5-0x198.5 (0.1)
0x190| 01 | . | supports_search_fs: false 0x198.6-0x198.6 (0.1)
0x190| 01 | . | supports_persistent_ids: true 0x198.7-0x198.7 (0.1)
0x190| 00 | . | supports_extended_security: false 0x199-0x199 (0.1)
0x190| 00 | . | has_no_root_directory_times: false 0x199.1-0x199.1 (0.1)
0x190| 00 | . | supports_flock: false 0x199.2-0x199.2 (0.1)
0x190| 00 | . | supports_case_preserved_names: false 0x199.3-0x199.3 (0.1)
0x190| 00 | . | supports_case_sensitive_names: false 0x199.4-0x199.4 (0.1)
0x190| 00 | . | supports_fast_stat_fs: false 0x199.5-0x199.5 (0.1)
0x190| 00 | . | supports_rename: false 0x199.6-0x199.6 (0.1)
0x190| 00 | . | supports_journaling: false 0x199.7-0x199.7 (0.1)
0x190| 00 | . | supports_zero_runs: false 0x19a-0x19a (0.1)
0x190| 00 | . | supports_sparse_files: false 0x19a.1-0x19a.1 (0.1)
0x190| 00 | . | is_journaling: false 0x19a.2-0x19a.2 (0.1)
0x190| 00 | . | reserved_3: false 0x19a.3-0x19a.3 (0.1)
0x190| 00 | . | supports_path_from_id: false 0x19a.4-0x19a.4 (0.1)
0x190| 00 | . | supports_mandatory_byte_range_locks: false 0x19a.5-0x19a.5 (0.1)
0x190| 00 | . | supports_hard_links: false 0x19a.6-0x19a.6 (0.1)
0x190| 00 | . | supports_2_tb_file_size: false 0x19a.7-0x19a.7 (0.1)
0x190| 00 | . | reserved_4: raw bits 0x19b-0x19b.2 (0.3)
0x190| 00 | . | has64_bit_object_ids: false 0x19b.3-0x19b.3 (0.1)
0x190| 00 | . | supports_decmp_fs_compression: false 0x19b.4-0x19b.4 (0.1)
0x190| 00 | . | supports_hidden_files: false 0x19b.5-0x19b.5 (0.1)
0x190| 00 | . | supports_remote_events: false 0x19b.6-0x19b.6 (0.1)
0x190| 00 | . | supports_volume_sizes: false 0x19b.7-0x19b.7 (0.1)
| | | enabled_property_flags{}: 0x19c-0x1a3.7 (8)
0x190| ef | . | is_internal: true 0x19c-0x19c (0.1)
0x190| ef | . | is_removable: true 0x19c.1-0x19c.1 (0.1)
0x190| ef | . | is_ejectable: true 0x19c.2-0x19c.2 (0.1)
0x190| ef | . | is_quarantined: false 0x19c.3-0x19c.3 (0.1)
0x190| ef | . | is_read_only: true 0x19c.4-0x19c.4 (0.1)
0x190| ef | . | dont_browse: true 0x19c.5-0x19c.5 (0.1)
0x190| ef | . | is_automount: true 0x19c.6-0x19c.6 (0.1)
0x190| ef | . | is_local: true 0x19c.7-0x19c.7 (0.1)
0x190| 13 | . | is_dvd: false 0x19d-0x19d (0.1)
0x190| 13 | . | is_cd: false 0x19d.1-0x19d.1 (0.1)
0x190| 13 | . | is_idisk: false 0x19d.2-0x19d.2 (0.1)
0x190| 13 | . | is_ipod: true 0x19d.3-0x19d.3 (0.1)
0x190| 13 | . | is_local_idisk_mirror: false 0x19d.4-0x19d.4 (0.1)
0x190| 13 | . | is_file_vault: false 0x19d.5-0x19d.5 (0.1)
0x190| 13 | . | is_disk_image: true 0x19d.6-0x19d.6 (0.1)
0x190| 13 | . | is_external: true 0x19d.7-0x19d.7 (0.1)
0x190| 00 | . | reserved_0: raw bits 0x19e-0x19e.6 (0.7)
0x190| 00 | . | is_device_file_system: false 0x19e.7-0x19e.7 (0.1)
0x190| 00| .| reserved_1: raw bits 0x19f-0x19f.7 (1)
0x1a0|01 |. | supports_read_dir_attr: false 0x1a0-0x1a0 (0.1)
0x1a0|01 |. | supports_copy_file: false 0x1a0.1-0x1a0.1 (0.1)
0x1a0|01 |. | supports_deny_modes: false 0x1a0.2-0x1a0.2 (0.1)
0x1a0|01 |. | supports_symbolic_links: false 0x1a0.3-0x1a0.3 (0.1)
0x1a0|01 |. | reserved_2: false 0x1a0.4-0x1a0.4 (0.1)
0x1a0|01 |. | supports_exchange: false 0x1a0.5-0x1a0.5 (0.1)
0x1a0|01 |. | supports_search_fs: false 0x1a0.6-0x1a0.6 (0.1)
0x1a0|01 |. | supports_persistent_ids: true 0x1a0.7-0x1a0.7 (0.1)
0x1a0| 00 | . | supports_extended_security: false 0x1a1-0x1a1 (0.1)
0x1a0| 00 | . | has_no_root_directory_times: false 0x1a1.1-0x1a1.1 (0.1)
0x1a0| 00 | . | supports_flock: false 0x1a1.2-0x1a1.2 (0.1)
0x1a0| 00 | . | supports_case_preserved_names: false 0x1a1.3-0x1a1.3 (0.1)
0x1a0| 00 | . | supports_case_sensitive_names: false 0x1a1.4-0x1a1.4 (0.1)
0x1a0| 00 | . | supports_fast_stat_fs: false 0x1a1.5-0x1a1.5 (0.1)
0x1a0| 00 | . | supports_rename: false 0x1a1.6-0x1a1.6 (0.1)
0x1a0| 00 | . | supports_journaling: false 0x1a1.7-0x1a1.7 (0.1)
0x1a0| 00 | . | supports_zero_runs: false 0x1a2-0x1a2 (0.1)
0x1a0| 00 | . | supports_sparse_files: false 0x1a2.1-0x1a2.1 (0.1)
0x1a0| 00 | . | is_journaling: false 0x1a2.2-0x1a2.2 (0.1)
0x1a0| 00 | . | reserved_3: false 0x1a2.3-0x1a2.3 (0.1)
0x1a0| 00 | . | supports_path_from_id: false 0x1a2.4-0x1a2.4 (0.1)
0x1a0| 00 | . | supports_mandatory_byte_range_locks: false 0x1a2.5-0x1a2.5 (0.1)
0x1a0| 00 | . | supports_hard_links: false 0x1a2.6-0x1a2.6 (0.1)
0x1a0| 00 | . | supports_2_tb_file_size: false 0x1a2.7-0x1a2.7 (0.1)
0x1a0| 00 | . | reserved_4: raw bits 0x1a3-0x1a3.2 (0.3)
0x1a0| 00 | . | has64_bit_object_ids: false 0x1a3.3-0x1a3.3 (0.1)
0x1a0| 00 | . | supports_decmp_fs_compression: false 0x1a3.4-0x1a3.4 (0.1)
0x1a0| 00 | . | supports_hidden_files: false 0x1a3.5-0x1a3.5 (0.1)
0x1a0| 00 | . | supports_remote_events: false 0x1a3.6-0x1a3.6 (0.1)
0x1a0| 00 | . | supports_volume_sizes: false 0x1a3.7-0x1a3.7 (0.1)
0x1a0| 00 00 00 00 00 00 00 00 | ........ | reserved: raw bits 0x1a4-0x1ab.7 (8)
0x240| 20 20 00 00| ..| key: "volume_flags" (8224) (flag bitfield) 0x24c-0x24f.7 (4)
0x250|5c 01 00 00 |\... | offset_to_record: 348 0x250-0x253.7 (4)
0x250| 00 00 00 00 | .... | unused: 0 0x254-0x257.7 (4)
| | | [11]{}: entry 0x1b8-0x263.7 (172)
| | | record{}: 0x1b8-0x1bf.7 (8)
0x1b0| 00 00 00 00 | .... | length: 0 0x1b8-0x1bb.7 (4)
0x1b0| 01 05 00 00| ....| type: "boolean_true" (1281) (True) 0x1bc-0x1bf.7 (4)
0x250| 30 20 00 00 | 0 .. | key: "volume_is_root" (8240) (True if the volume was the filesystem root) 0x258-0x25b.7 (4)
0x250| 88 01 00 00| ....| offset_to_record: 392 0x25c-0x25f.7 (4)
0x260|00 00 00 00 |.... | unused: 0 0x260-0x263.7 (4)
| | | [12]{}: entry 0x100-0x26f.7 (368)
| | | record{}: 0x100-0x10f.7 (16)
0x100|08 00 00 00 |.... | length: 8 0x100-0x103.7 (4)
0x100| 04 03 00 00 | .... | type: "long" (772) ((signed 64-bit) 8-byte number) 0x104-0x107.7 (4)
0x100| 01 00 00 00 00 00 00 00| ........| data: 1 0x108-0x10f.7 (8)
0x260| 01 c0 00 00 | .... | key: "containing_folder_index" (49153) (Integer index of containing folder in target path array) 0x264-0x267.7 (4)
0x260| d0 00 00 00 | .... | offset_to_record: 208 0x268-0x26b.7 (4)
0x260| 00 00 00 00| ....| unused: 0 0x26c-0x26f.7 (4)
| | | [13]{}: entry 0x50-0x27b.7 (556)
| | | record{}: 0x50-0x67.7 (24)
0x050|0d 00 00 00 |.... | length: 13 0x50-0x53.7 (4)
0x050| 01 01 00 00 | .... | type: "string" (257) (UTF-8 String) 0x54-0x57.7 (4)
0x050| 64 61 76 69 64 6d 63 64| davidmcd| data: "davidmcdonald" 0x58-0x64.7 (13)
0x060|6f 6e 61 6c 64 |onald |
0x060| 00 00 00 | ... | alignment_bytes: raw bits 0x65-0x67.7 (3)
0x270|11 c0 00 00 |.... | key: "creator_username" (49169) (Name of user that created bookmark) 0x270-0x273.7 (4)
0x270| 20 00 00 00 | ... | offset_to_record: 32 0x274-0x277.7 (4)
0x270| 00 00 00 00 | .... | unused: 0 0x278-0x27b.7 (4)
| | | [14]{}: entry 0x110-0x287.7 (376)
| | | record{}: 0x110-0x11b.7 (12)
0x110|04 00 00 00 |.... | length: 4 0x110-0x113.7 (4)
0x110| 03 03 00 00 | .... | type: "int" (771) ((signed 32-bit) 4-byte number) 0x114-0x117.7 (4)
0x110| f5 01 00 00 | .... | data: 501 0x118-0x11b.7 (4)
0x270| 12 c0 00 00| ....| key: "creator_uid" (49170) (UID of user that created bookmark) 0x27c-0x27f.7 (4)
0x280|e0 00 00 00 |.... | offset_to_record: 224 0x280-0x283.7 (4)
0x280| 00 00 00 00 | .... | unused: 0 0x284-0x287.7 (4)
| | | [15]{}: entry 0x34-0x293.7 (608)
| | | record{}: 0x34-0x3f.7 (12)
0x030| 04 00 00 00 | .... | length: 4 0x34-0x37.7 (4)
0x030| 03 03 00 00 | .... | type: "int" (771) ((signed 32-bit) 4-byte number) 0x38-0x3b.7 (4)
0x030| 00 00 00 20| ... | data: 536870912 0x3c-0x3f.7 (4)
0x280| 10 d0 00 00 | .... | key: "creation_options" (53264) (Integer containing flags passed to CFURLCreateBookmarkData) 0x288-0x28b.7 (4)
0x280| 04 00 00 00| ....| offset_to_record: 4 0x28c-0x28f.7 (4)
0x290|00 00 00 00 |.... | unused: 0 0x290-0x293.7 (4)
| | | [16]{}: entry 0x68-0x29f.7 (568)
| | | record{}: 0x68-0x77.7 (16)
0x060| 07 00 00 00 | .... | length: 7 0x68-0x6b.7 (4)
0x060| 01 01 00 00| ....| type: "string" (257) (UTF-8 String) 0x6c-0x6f.7 (4)
0x070|68 78 73 74 6f 72 65 |hxstore | data: "hxstore" 0x70-0x76.7 (7)
0x070| 00 | . | alignment_bytes: raw bits 0x77-0x77.7 (1)
0x290| 17 f0 00 00 | .... | key: "display_name" (61463) (String) 0x294-0x297.7 (4)
0x290| 38 00 00 00 | 8... | offset_to_record: 56 0x298-0x29b.7 (4)
0x290| 00 00 00 00| ....| unused: 0 0x29c-0x29f.7 (4)
| | | [17]{}: entry 0x1b8-0x2ab.7 (244)
| | | record{}: 0x1b8-0x1bf.7 (8)
0x1b0| 00 00 00 00 | .... | length: 0 0x1b8-0x1bb.7 (4)
0x1b0| 01 05 00 00| ....| type: "boolean_true" (1281) (True) 0x1bc-0x1bf.7 (4)
0x2a0|0f 00 0f 00 |.... | key: 983055 0x2a0-0x2a3.7 (4)
0x2a0| 88 01 00 00 | .... | offset_to_record: 392 0x2a4-0x2a7.7 (4)
0x2a0| 00 00 00 00| | ....| | unused: 0 0x2a8-0x2ab.7 (4)
| | | toc_headers[0:1]: 0x1c0-0x1d3.7 (20)
| | | [0]{}: toc_header 0x1c0-0x1d3.7 (20)
0x1c0|e4 00 00 00 |.... | toc_size: 228 0x1c0-0x1c3.7 (4)
0x1c0| fe ff ff ff | .... | magic: 4294967294 (valid) 0x1c4-0x1c7.7 (4)
0x1c0| 01 00 00 00 | .... | identifier: 1 0x1c8-0x1cb.7 (4)
0x1c0| 00 00 00 00| ....| next_toc_offset: 0 0x1cc-0x1cf.7 (4)
0x1d0|12 00 00 00 |.... | num_entries_in_toc: 18 0x1d0-0x1d3.7 (4)
$ fq torepr sample2.book
{
"983055": true,
"containing_folder_index": 1,
"creation_options": 536870912,
"creator_uid": 501,
"creator_username": "davidmcdonald",
"display_name": "hxstore",
"target_cnid_path": [
23307,
564764,
54838727
],
"target_creation_date": 689188614.6088948,
"target_flags": {
"is_application": false,
"is_directory": true,
"is_package": false,
"is_regular_file": false,
"is_symbolic_link": false,
"is_volume": false
},
"target_path": [
"Users",
"davidmcdonald",
"hxstore"
],
"volume_creation_date": 599558400,
"volume_flags": {
"dont_browse": false,
"is_automount": false,
"is_disk_image": false,
"is_ejectable": false,
"is_external": false,
"is_internal": true,
"is_ipod": false,
"is_local": true,
"is_read_only": false,
"is_removable": false,
"supports_persistent_ids": true
},
"volume_is_root": true,
"volume_name": "Macintosh HD",
"volume_path": "/",
"volume_size": 499963174912,
"volume_url": "file:///",
"volume_uuid": "8A829A03-9B24-4085-8051-18978712E4BE"
}

BIN
format/apple/bookmark/testdata/sample3.book vendored Normal file
View File

Binary file not shown.

446
format/apple/bookmark/testdata/sample3.fqtest vendored Normal file
View File

@ -0,0 +1,446 @@
$ fq dv sample3.book
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: sample3.book (apple_bookmark) 0x0-0x333.7 (820)
| | | header{}: 0x0-0x2f.7 (48)
0x000|62 6f 6f 6b |book | magic: "book" (valid) 0x0-0x3.7 (4)
0x000| 34 03 00 00 | 4... | total_size: 820 0x4-0x7.7 (4)
0x000| 00 00 04 10 | .... | unknown: 268697600 0x8-0xb.7 (4)
0x000| 30 00 00 00| 0...| header_size: 48 (valid) 0xc-0xf.7 (4)
0x010|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| reserved: raw bits 0x10-0x2f.7 (32)
0x020|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
0x030|18 02 00 00 |.... | first_toc_offset: 536 0x30-0x33.7 (4)
| | | bookmark_entries[0:18]: 0x34-0x333.7 (768)
| | | [0]{}: entry 0x40-0x267.7 (552)
| | | record{}: 0x40-0xd7.7 (152)
| | | data[0:6]: 0x40-0xd7.7 (152)
| | | [0]{}: element 0x40-0xc3.7 (132)
| | | record{}: 0x40-0x4f.7 (16)
0x040|05 00 00 00 |.... | length: 5 0x40-0x43.7 (4)
0x040| 01 01 00 00 | .... | type: "string" (257) (UTF-8 String) 0x44-0x47.7 (4)
0x040| 55 73 65 72 73 | Users | data: "Users" 0x48-0x4c.7 (5)
0x040| 00 00 00| ...| alignment_bytes: raw bits 0x4d-0x4f.7 (3)
0x0c0|10 00 00 00 |.... | offset: 16 0xc0-0xc3.7 (4)
| | | [1]{}: element 0x50-0xc7.7 (120)
| | | record{}: 0x50-0x67.7 (24)
0x050|0d 00 00 00 |.... | length: 13 0x50-0x53.7 (4)
0x050| 01 01 00 00 | .... | type: "string" (257) (UTF-8 String) 0x54-0x57.7 (4)
0x050| 64 61 76 69 64 6d 63 64| davidmcd| data: "davidmcdonald" 0x58-0x64.7 (13)
0x060|6f 6e 61 6c 64 |onald |
0x060| 00 00 00 | ... | alignment_bytes: raw bits 0x65-0x67.7 (3)
0x0c0| 20 00 00 00 | ... | offset: 32 0xc4-0xc7.7 (4)
| | | [2]{}: element 0x68-0xcb.7 (100)
| | | record{}: 0x68-0x7b.7 (20)
0x060| 09 00 00 00 | .... | length: 9 0x68-0x6b.7 (4)
0x060| 01 01 00 00| ....| type: "string" (257) (UTF-8 String) 0x6c-0x6f.7 (4)
0x070|44 6f 63 75 6d 65 6e 74 73 |Documents | data: "Documents" 0x70-0x78.7 (9)
0x070| 00 00 00 | ... | alignment_bytes: raw bits 0x79-0x7b.7 (3)
0x0c0| 38 00 00 00 | 8... | offset: 56 0xc8-0xcb.7 (4)
| | | [3]{}: element 0x7c-0xcf.7 (84)
| | | record{}: 0x7c-0x8b.7 (16)
0x070| 07 00 00 00| ....| length: 7 0x7c-0x7f.7 (4)
0x080|01 01 00 00 |.... | type: "string" (257) (UTF-8 String) 0x80-0x83.7 (4)
0x080| 41 72 64 75 69 6e 6f | Arduino | data: "Arduino" 0x84-0x8a.7 (7)
0x080| 00 | . | alignment_bytes: raw bits 0x8b-0x8b.7 (1)
0x0c0| 4c 00 00 00| L...| offset: 76 0xcc-0xcf.7 (4)
| | | [4]{}: element 0x8c-0xd3.7 (72)
| | | record{}: 0x8c-0xa7.7 (28)
0x080| 12 00 00 00| ....| length: 18 0x8c-0x8f.7 (4)
0x090|01 01 00 00 |.... | type: "string" (257) (UTF-8 String) 0x90-0x93.7 (4)
0x090| 67 65 6e 65 72 61 74 65 64 5f 65 78| generated_ex| data: "generated_examples" 0x94-0xa5.7 (18)
0x0a0|61 6d 70 6c 65 73 |amples |
0x0a0| 00 00 | .. | alignment_bytes: raw bits 0xa6-0xa7.7 (2)
0x0d0|5c 00 00 00 |\... | offset: 92 0xd0-0xd3.7 (4)
| | | [5]{}: element 0xa8-0xd7.7 (48)
| | | record{}: 0xa8-0xb7.7 (16)
0x0a0| 05 00 00 00 | .... | length: 5 0xa8-0xab.7 (4)
0x0a0| 01 01 00 00| ....| type: "string" (257) (UTF-8 String) 0xac-0xaf.7 (4)
0x0b0|42 6c 69 6e 6b |Blink | data: "Blink" 0xb0-0xb4.7 (5)
0x0b0| 00 00 00 | ... | alignment_bytes: raw bits 0xb5-0xb7.7 (3)
0x0d0| 78 00 00 00 | x... | offset: 120 0xd4-0xd7.7 (4)
0x0b0| 18 00 00 00 | .... | length: 24 0xb8-0xbb.7 (4)
0x0b0| 01 06 00 00| ....| type: "array" (1537) (Array of 4-byte offsets to data items) 0xbc-0xbf.7 (4)
0x250| 04 10 00 00| ....| key: "target_path" (4100) (Array of individual path components) 0x25c-0x25f.7 (4)
0x260|88 00 00 00 |.... | offset_to_record: 136 0x260-0x263.7 (4)
0x260| 00 00 00 00 | .... | unused: 0 0x264-0x267.7 (4)
| | | [1]{}: entry 0xd8-0x273.7 (412)
| | | record{}: 0xd8-0x157.7 (128)
| | | data[0:6]: 0xd8-0x157.7 (128)
| | | [0]{}: element 0xd8-0x143.7 (108)
| | | record{}: 0xd8-0xe7.7 (16)
0x0d0| 08 00 00 00 | .... | length: 8 0xd8-0xdb.7 (4)
0x0d0| 04 03 00 00| ....| type: "long" (772) ((signed 64-bit) 8-byte number) 0xdc-0xdf.7 (4)
0x0e0|0b 5b 00 00 00 00 00 00 |.[...... | data: 23307 0xe0-0xe7.7 (8)
0x140|a8 00 00 00 |.... | offset: 168 0x140-0x143.7 (4)
| | | [1]{}: element 0xe8-0x147.7 (96)
| | | record{}: 0xe8-0xf7.7 (16)
0x0e0| 08 00 00 00 | .... | length: 8 0xe8-0xeb.7 (4)
0x0e0| 04 03 00 00| ....| type: "long" (772) ((signed 64-bit) 8-byte number) 0xec-0xef.7 (4)
0x0f0|1c 9e 08 00 00 00 00 00 |........ | data: 564764 0xf0-0xf7.7 (8)
0x140| b8 00 00 00 | .... | offset: 184 0x144-0x147.7 (4)
| | | [2]{}: element 0xf8-0x14b.7 (84)
| | | record{}: 0xf8-0x107.7 (16)
0x0f0| 08 00 00 00 | .... | length: 8 0xf8-0xfb.7 (4)
0x0f0| 04 03 00 00| ....| type: "long" (772) ((signed 64-bit) 8-byte number) 0xfc-0xff.7 (4)
0x100|52 e1 0d 00 00 00 00 00 |R....... | data: 909650 0x100-0x107.7 (8)
0x140| c8 00 00 00 | .... | offset: 200 0x148-0x14b.7 (4)
| | | [3]{}: element 0x108-0x14f.7 (72)
| | | record{}: 0x108-0x117.7 (16)
0x100| 08 00 00 00 | .... | length: 8 0x108-0x10b.7 (4)
0x100| 04 03 00 00| ....| type: "long" (772) ((signed 64-bit) 8-byte number) 0x10c-0x10f.7 (4)
0x110|b0 1f 0e 00 00 00 00 00 |........ | data: 925616 0x110-0x117.7 (8)
0x140| d8 00 00 00| ....| offset: 216 0x14c-0x14f.7 (4)
| | | [4]{}: element 0x118-0x153.7 (60)
| | | record{}: 0x118-0x127.7 (16)
0x110| 08 00 00 00 | .... | length: 8 0x118-0x11b.7 (4)
0x110| 04 03 00 00| ....| type: "long" (772) ((signed 64-bit) 8-byte number) 0x11c-0x11f.7 (4)
0x120|91 2b 2d 02 00 00 00 00 |.+-..... | data: 36514705 0x120-0x127.7 (8)
0x150|e8 00 00 00 |.... | offset: 232 0x150-0x153.7 (4)
| | | [5]{}: element 0x128-0x157.7 (48)
| | | record{}: 0x128-0x137.7 (16)
0x120| 08 00 00 00 | .... | length: 8 0x128-0x12b.7 (4)
0x120| 04 03 00 00| ....| type: "long" (772) ((signed 64-bit) 8-byte number) 0x12c-0x12f.7 (4)
0x130|92 2b 2d 02 00 00 00 00 |.+-..... | data: 36514706 0x130-0x137.7 (8)
0x150| f8 00 00 00 | .... | offset: 248 0x154-0x157.7 (4)
0x130| 18 00 00 00 | .... | length: 24 0x138-0x13b.7 (4)
0x130| 01 06 00 00| ....| type: "array" (1537) (Array of 4-byte offsets to data items) 0x13c-0x13f.7 (4)
0x260| 05 10 00 00 | .... | key: "target_cnid_path" (4101) (Array of CNIDs) 0x268-0x26b.7 (4)
0x260| 08 01 00 00| ....| offset_to_record: 264 0x26c-0x26f.7 (4)
0x270|00 00 00 00 |.... | unused: 0 0x270-0x273.7 (4)
| | | [2]{}: entry 0x168-0x27f.7 (280)
| | | record{}: 0x168-0x187.7 (32)
0x160| 18 00 00 00 | .... | length: 24 (valid) 0x168-0x16b.7 (4)
0x160| 01 02 00 00| ....| raw_type: "data" (513) (valid) 0x16c-0x16f.7 (4)
| | | type: "flag_data" 0x170-NA (0)
| | | property_flags{}: 0x170-0x177.7 (8)
0x170|02 |. | is_hidden: false 0x170-0x170 (0.1)
0x170|02 |. | is_user_immutable: false 0x170.1-0x170.1 (0.1)
0x170|02 |. | is_system_immutable: false 0x170.2-0x170.2 (0.1)
0x170|02 |. | is_package: false 0x170.3-0x170.3 (0.1)
0x170|02 |. | is_volume: false 0x170.4-0x170.4 (0.1)
0x170|02 |. | is_symbolic_link: false 0x170.5-0x170.5 (0.1)
0x170|02 |. | is_directory: true 0x170.6-0x170.6 (0.1)
0x170|02 |. | is_regular_file: false 0x170.7-0x170.7 (0.1)
0x170| 00 | . | is_alias_file: false 0x171-0x171 (0.1)
0x170| 00 | . | is_executable: false 0x171.1-0x171.1 (0.1)
0x170| 00 | . | is_writeable: false 0x171.2-0x171.2 (0.1)
0x170| 00 | . | is_readable: false 0x171.3-0x171.3 (0.1)
0x170| 00 | . | can_set_hidden_extension: false 0x171.4-0x171.4 (0.1)
0x170| 00 | . | is_compressed: false 0x171.5-0x171.5 (0.1)
0x170| 00 | . | is_application: false 0x171.6-0x171.6 (0.1)
0x170| 00 | . | has_hidden_extension: false 0x171.7-0x171.7 (0.1)
0x170| 00 | . | reserved_bits_0: raw bits 0x172-0x172.6 (0.7)
0x170| 00 | . | is_mount_trigger: false 0x172.7-0x172.7 (0.1)
0x170| 00 00 00 00 00 | ..... | reserved: raw bits 0x173-0x177.7 (5)
| | | enabled_property_flags{}: 0x178-0x17f.7 (8)
0x170| 1f | . | is_hidden: false 0x178-0x178 (0.1)
0x170| 1f | . | is_user_immutable: false 0x178.1-0x178.1 (0.1)
0x170| 1f | . | is_system_immutable: false 0x178.2-0x178.2 (0.1)
0x170| 1f | . | is_package: true 0x178.3-0x178.3 (0.1)
0x170| 1f | . | is_volume: true 0x178.4-0x178.4 (0.1)
0x170| 1f | . | is_symbolic_link: true 0x178.5-0x178.5 (0.1)
0x170| 1f | . | is_directory: true 0x178.6-0x178.6 (0.1)
0x170| 1f | . | is_regular_file: true 0x178.7-0x178.7 (0.1)
0x170| 02 | . | is_alias_file: false 0x179-0x179 (0.1)
0x170| 02 | . | is_executable: false 0x179.1-0x179.1 (0.1)
0x170| 02 | . | is_writeable: false 0x179.2-0x179.2 (0.1)
0x170| 02 | . | is_readable: false 0x179.3-0x179.3 (0.1)
0x170| 02 | . | can_set_hidden_extension: false 0x179.4-0x179.4 (0.1)
0x170| 02 | . | is_compressed: false 0x179.5-0x179.5 (0.1)
0x170| 02 | . | is_application: true 0x179.6-0x179.6 (0.1)
0x170| 02 | . | has_hidden_extension: false 0x179.7-0x179.7 (0.1)
0x170| 00 | . | reserved_bits_0: raw bits 0x17a-0x17a.6 (0.7)
0x170| 00 | . | is_mount_trigger: false 0x17a.7-0x17a.7 (0.1)
0x170| 00 00 00 00 00| .....| reserved: raw bits 0x17b-0x17f.7 (5)
0x180|1a 02 00 00 00 00 00 00 |........ | reserved: raw bits 0x180-0x187.7 (8)
0x270| 10 10 00 00 | .... | key: "target_flags" (4112) (flag bitfield) 0x274-0x277.7 (4)
0x270| 38 01 00 00 | 8... | offset_to_record: 312 0x278-0x27b.7 (4)
0x270| 00 00 00 00| ....| unused: 0 0x27c-0x27f.7 (4)
| | | [3]{}: entry 0x158-0x28b.7 (308)
| | | record{}: 0x158-0x167.7 (16)
0x150| 08 00 00 00 | .... | length: 8 0x158-0x15b.7 (4)
0x150| 00 04 00 00| ....| type: "date" (1024) (Big-endian IEEE double precision seconds since 2001-01-01 00:00:00 UTC) 0x15c-0x15f.7 (4)
0x160|41 c4 11 9d 61 74 66 30 |A...atf0 | data: 6.733974429093685e+08 (2022-05-04T22:50:42Z) 0x160-0x167.7 (8)
0x280|40 10 00 00 |@... | key: "target_creation_date" (4160) (Date) 0x280-0x283.7 (4)
0x280| 28 01 00 00 | (... | offset_to_record: 296 0x284-0x287.7 (4)
0x280| 00 00 00 00 | .... | unused: 0 0x288-0x28b.7 (4)
| | | [4]{}: entry 0x234-0x297.7 (100)
| | | record{}: 0x234-0x23f.7 (12)
0x230| 01 00 00 00 | .... | length: 1 0x234-0x237.7 (4)
0x230| 01 01 00 00 | .... | type: "string" (257) (UTF-8 String) 0x238-0x23b.7 (4)
0x230| 2f | / | data: "/" 0x23c-0x23c.7 (1)
0x230| 00 00 00| ...| alignment_bytes: raw bits 0x23d-0x23f.7 (3)
0x280| 02 20 00 00| . ..| key: "volume_path" (8194) (Array of individual path components) 0x28c-0x28f.7 (4)
0x290|04 02 00 00 |.... | offset_to_record: 516 0x290-0x293.7 (4)
0x290| 00 00 00 00 | .... | unused: 0 0x294-0x297.7 (4)
| | | [5]{}: entry 0x1a4-0x2a3.7 (256)
| | | record{}: 0x1a4-0x1b3.7 (16)
0x1a0| 08 00 00 00 | .... | length: 8 0x1a4-0x1a7.7 (4)
0x1a0| 01 09 00 00 | .... | type: "url" (2305) (UTF-8 string) 0x1a8-0x1ab.7 (4)
0x1a0| 66 69 6c 65| file| data: "file:///" 0x1ac-0x1b3.7 (8)
0x1b0|3a 2f 2f 2f |:/// |
0x290| 05 20 00 00 | . .. | key: "volume_url" (8197) (URL of volume root) 0x298-0x29b.7 (4)
0x290| 74 01 00 00| t...| offset_to_record: 372 0x29c-0x29f.7 (4)
0x2a0|00 00 00 00 |.... | unused: 0 0x2a0-0x2a3.7 (4)
| | | [6]{}: entry 0x1b4-0x2af.7 (252)
| | | record{}: 0x1b4-0x1cb.7 (24)
0x1b0| 0c 00 00 00 | .... | length: 12 0x1b4-0x1b7.7 (4)
0x1b0| 01 01 00 00 | .... | type: "string" (257) (UTF-8 String) 0x1b8-0x1bb.7 (4)
0x1b0| 4d 61 63 69| Maci| data: "Macintosh HD" 0x1bc-0x1c7.7 (12)
0x1c0|6e 74 6f 73 68 20 48 44 |ntosh HD |
0x1c0| 08 00 00 00 | .... | alignment_bytes: raw bits 0x1c8-0x1cb.7 (4)
0x2a0| 10 20 00 00 | . .. | key: "volume_name" (8208) (String) 0x2a4-0x2a7.7 (4)
0x2a0| 84 01 00 00 | .... | offset_to_record: 388 0x2a8-0x2ab.7 (4)
0x2a0| 00 00 00 00| ....| unused: 0 0x2ac-0x2af.7 (4)
| | | [7]{}: entry 0x1e8-0x2bb.7 (212)
| | | record{}: 0x1e8-0x217.7 (48)
0x1e0| 24 00 00 00 | $... | length: 36 0x1e8-0x1eb.7 (4)
0x1e0| 01 01 00 00| ....| type: "string" (257) (UTF-8 String) 0x1ec-0x1ef.7 (4)
0x1f0|38 41 38 32 39 41 30 33 2d 39 42 32 34 2d 34 30|8A829A03-9B24-40| data: "8A829A03-9B24-4085-8051-18978712E4BE" 0x1f0-0x213.7 (36)
* |until 0x213.7 (36) | |
0x210| 18 00 00 00 | .... | alignment_bytes: raw bits 0x214-0x217.7 (4)
0x2b0|11 20 00 00 |. .. | key: "volume_uuid" (8209) (String UUID) 0x2b0-0x2b3.7 (4)
0x2b0| b8 01 00 00 | .... | offset_to_record: 440 0x2b4-0x2b7.7 (4)
0x2b0| 00 00 00 00 | .... | unused: 0 0x2b8-0x2bb.7 (4)
| | | [8]{}: entry 0x1c8-0x2c7.7 (256)
| | | record{}: 0x1c8-0x1d7.7 (16)
0x1c0| 08 00 00 00 | .... | length: 8 0x1c8-0x1cb.7 (4)
0x1c0| 04 03 00 00| ....| type: "long" (772) ((signed 64-bit) 8-byte number) 0x1cc-0x1cf.7 (4)
0x1d0|00 a0 20 68 74 00 00 00 |.. ht... | data: 499963174912 0x1d0-0x1d7.7 (8)
0x2b0| 12 20 00 00| . ..| key: "volume_size" (8210) (8-byte integer) 0x2bc-0x2bf.7 (4)
0x2c0|98 01 00 00 |.... | offset_to_record: 408 0x2c0-0x2c3.7 (4)
0x2c0| 00 00 00 00 | .... | unused: 0 0x2c4-0x2c7.7 (4)
| | | [9]{}: entry 0x1d8-0x2d3.7 (252)
| | | record{}: 0x1d8-0x1e7.7 (16)
0x1d0| 08 00 00 00 | .... | length: 8 0x1d8-0x1db.7 (4)
0x1d0| 00 04 00 00| ....| type: "date" (1024) (Big-endian IEEE double precision seconds since 2001-01-01 00:00:00 UTC) 0x1dc-0x1df.7 (4)
0x1e0|41 c1 de 44 80 00 00 00 |A..D.... | data: 5.995584e+08 (2020-01-01T08:00:00Z) 0x1e0-0x1e7.7 (8)
0x2c0| 13 20 00 00 | . .. | key: "volume_creation_date" (8211) (Date) 0x2c8-0x2cb.7 (4)
0x2c0| a8 01 00 00| ....| offset_to_record: 424 0x2cc-0x2cf.7 (4)
0x2d0|00 00 00 00 |.... | unused: 0 0x2d0-0x2d3.7 (4)
| | | [10]{}: entry 0x214-0x2df.7 (204)
| | | record{}: 0x214-0x233.7 (32)
0x210| 18 00 00 00 | .... | length: 24 (valid) 0x214-0x217.7 (4)
0x210| 01 02 00 00 | .... | raw_type: "data" (513) (valid) 0x218-0x21b.7 (4)
| | | type: "flag_data" 0x21c-NA (0)
| | | property_flags{}: 0x21c-0x223.7 (8)
0x210| 81 | . | is_internal: true 0x21c-0x21c (0.1)
0x210| 81 | . | is_removable: false 0x21c.1-0x21c.1 (0.1)
0x210| 81 | . | is_ejectable: false 0x21c.2-0x21c.2 (0.1)
0x210| 81 | . | is_quarantined: false 0x21c.3-0x21c.3 (0.1)
0x210| 81 | . | is_read_only: false 0x21c.4-0x21c.4 (0.1)
0x210| 81 | . | dont_browse: false 0x21c.5-0x21c.5 (0.1)
0x210| 81 | . | is_automount: false 0x21c.6-0x21c.6 (0.1)
0x210| 81 | . | is_local: true 0x21c.7-0x21c.7 (0.1)
0x210| 00 | . | is_dvd: false 0x21d-0x21d (0.1)
0x210| 00 | . | is_cd: false 0x21d.1-0x21d.1 (0.1)
0x210| 00 | . | is_idisk: false 0x21d.2-0x21d.2 (0.1)
0x210| 00 | . | is_ipod: false 0x21d.3-0x21d.3 (0.1)
0x210| 00 | . | is_local_idisk_mirror: false 0x21d.4-0x21d.4 (0.1)
0x210| 00 | . | is_file_vault: false 0x21d.5-0x21d.5 (0.1)
0x210| 00 | . | is_disk_image: false 0x21d.6-0x21d.6 (0.1)
0x210| 00 | . | is_external: false 0x21d.7-0x21d.7 (0.1)
0x210| 00 | . | reserved_0: raw bits 0x21e-0x21e.6 (0.7)
0x210| 00 | . | is_device_file_system: false 0x21e.7-0x21e.7 (0.1)
0x210| 00| .| reserved_1: raw bits 0x21f-0x21f.7 (1)
0x220|01 |. | supports_read_dir_attr: false 0x220-0x220 (0.1)
0x220|01 |. | supports_copy_file: false 0x220.1-0x220.1 (0.1)
0x220|01 |. | supports_deny_modes: false 0x220.2-0x220.2 (0.1)
0x220|01 |. | supports_symbolic_links: false 0x220.3-0x220.3 (0.1)
0x220|01 |. | reserved_2: false 0x220.4-0x220.4 (0.1)
0x220|01 |. | supports_exchange: false 0x220.5-0x220.5 (0.1)
0x220|01 |. | supports_search_fs: false 0x220.6-0x220.6 (0.1)
0x220|01 |. | supports_persistent_ids: true 0x220.7-0x220.7 (0.1)
0x220| 00 | . | supports_extended_security: false 0x221-0x221 (0.1)
0x220| 00 | . | has_no_root_directory_times: false 0x221.1-0x221.1 (0.1)
0x220| 00 | . | supports_flock: false 0x221.2-0x221.2 (0.1)
0x220| 00 | . | supports_case_preserved_names: false 0x221.3-0x221.3 (0.1)
0x220| 00 | . | supports_case_sensitive_names: false 0x221.4-0x221.4 (0.1)
0x220| 00 | . | supports_fast_stat_fs: false 0x221.5-0x221.5 (0.1)
0x220| 00 | . | supports_rename: false 0x221.6-0x221.6 (0.1)
0x220| 00 | . | supports_journaling: false 0x221.7-0x221.7 (0.1)
0x220| 00 | . | supports_zero_runs: false 0x222-0x222 (0.1)
0x220| 00 | . | supports_sparse_files: false 0x222.1-0x222.1 (0.1)
0x220| 00 | . | is_journaling: false 0x222.2-0x222.2 (0.1)
0x220| 00 | . | reserved_3: false 0x222.3-0x222.3 (0.1)
0x220| 00 | . | supports_path_from_id: false 0x222.4-0x222.4 (0.1)
0x220| 00 | . | supports_mandatory_byte_range_locks: false 0x222.5-0x222.5 (0.1)
0x220| 00 | . | supports_hard_links: false 0x222.6-0x222.6 (0.1)
0x220| 00 | . | supports_2_tb_file_size: false 0x222.7-0x222.7 (0.1)
0x220| 00 | . | reserved_4: raw bits 0x223-0x223.2 (0.3)
0x220| 00 | . | has64_bit_object_ids: false 0x223.3-0x223.3 (0.1)
0x220| 00 | . | supports_decmp_fs_compression: false 0x223.4-0x223.4 (0.1)
0x220| 00 | . | supports_hidden_files: false 0x223.5-0x223.5 (0.1)
0x220| 00 | . | supports_remote_events: false 0x223.6-0x223.6 (0.1)
0x220| 00 | . | supports_volume_sizes: false 0x223.7-0x223.7 (0.1)
| | | enabled_property_flags{}: 0x224-0x22b.7 (8)
0x220| ef | . | is_internal: true 0x224-0x224 (0.1)
0x220| ef | . | is_removable: true 0x224.1-0x224.1 (0.1)
0x220| ef | . | is_ejectable: true 0x224.2-0x224.2 (0.1)
0x220| ef | . | is_quarantined: false 0x224.3-0x224.3 (0.1)
0x220| ef | . | is_read_only: true 0x224.4-0x224.4 (0.1)
0x220| ef | . | dont_browse: true 0x224.5-0x224.5 (0.1)
0x220| ef | . | is_automount: true 0x224.6-0x224.6 (0.1)
0x220| ef | . | is_local: true 0x224.7-0x224.7 (0.1)
0x220| 13 | . | is_dvd: false 0x225-0x225 (0.1)
0x220| 13 | . | is_cd: false 0x225.1-0x225.1 (0.1)
0x220| 13 | . | is_idisk: false 0x225.2-0x225.2 (0.1)
0x220| 13 | . | is_ipod: true 0x225.3-0x225.3 (0.1)
0x220| 13 | . | is_local_idisk_mirror: false 0x225.4-0x225.4 (0.1)
0x220| 13 | . | is_file_vault: false 0x225.5-0x225.5 (0.1)
0x220| 13 | . | is_disk_image: true 0x225.6-0x225.6 (0.1)
0x220| 13 | . | is_external: true 0x225.7-0x225.7 (0.1)
0x220| 00 | . | reserved_0: raw bits 0x226-0x226.6 (0.7)
0x220| 00 | . | is_device_file_system: false 0x226.7-0x226.7 (0.1)
0x220| 00 | . | reserved_1: raw bits 0x227-0x227.7 (1)
0x220| 01 | . | supports_read_dir_attr: false 0x228-0x228 (0.1)
0x220| 01 | . | supports_copy_file: false 0x228.1-0x228.1 (0.1)
0x220| 01 | . | supports_deny_modes: false 0x228.2-0x228.2 (0.1)
0x220| 01 | . | supports_symbolic_links: false 0x228.3-0x228.3 (0.1)
0x220| 01 | . | reserved_2: false 0x228.4-0x228.4 (0.1)
0x220| 01 | . | supports_exchange: false 0x228.5-0x228.5 (0.1)
0x220| 01 | . | supports_search_fs: false 0x228.6-0x228.6 (0.1)
0x220| 01 | . | supports_persistent_ids: true 0x228.7-0x228.7 (0.1)
0x220| 00 | . | supports_extended_security: false 0x229-0x229 (0.1)
0x220| 00 | . | has_no_root_directory_times: false 0x229.1-0x229.1 (0.1)
0x220| 00 | . | supports_flock: false 0x229.2-0x229.2 (0.1)
0x220| 00 | . | supports_case_preserved_names: false 0x229.3-0x229.3 (0.1)
0x220| 00 | . | supports_case_sensitive_names: false 0x229.4-0x229.4 (0.1)
0x220| 00 | . | supports_fast_stat_fs: false 0x229.5-0x229.5 (0.1)
0x220| 00 | . | supports_rename: false 0x229.6-0x229.6 (0.1)
0x220| 00 | . | supports_journaling: false 0x229.7-0x229.7 (0.1)
0x220| 00 | . | supports_zero_runs: false 0x22a-0x22a (0.1)
0x220| 00 | . | supports_sparse_files: false 0x22a.1-0x22a.1 (0.1)
0x220| 00 | . | is_journaling: false 0x22a.2-0x22a.2 (0.1)
0x220| 00 | . | reserved_3: false 0x22a.3-0x22a.3 (0.1)
0x220| 00 | . | supports_path_from_id: false 0x22a.4-0x22a.4 (0.1)
0x220| 00 | . | supports_mandatory_byte_range_locks: false 0x22a.5-0x22a.5 (0.1)
0x220| 00 | . | supports_hard_links: false 0x22a.6-0x22a.6 (0.1)
0x220| 00 | . | supports_2_tb_file_size: false 0x22a.7-0x22a.7 (0.1)
0x220| 00 | . | reserved_4: raw bits 0x22b-0x22b.2 (0.3)
0x220| 00 | . | has64_bit_object_ids: false 0x22b.3-0x22b.3 (0.1)
0x220| 00 | . | supports_decmp_fs_compression: false 0x22b.4-0x22b.4 (0.1)
0x220| 00 | . | supports_hidden_files: false 0x22b.5-0x22b.5 (0.1)
0x220| 00 | . | supports_remote_events: false 0x22b.6-0x22b.6 (0.1)
0x220| 00 | . | supports_volume_sizes: false 0x22b.7-0x22b.7 (0.1)
0x220| 00 00 00 00| ....| reserved: raw bits 0x22c-0x233.7 (8)
0x230|00 00 00 00 |.... |
0x2d0| 20 20 00 00 | .. | key: "volume_flags" (8224) (flag bitfield) 0x2d4-0x2d7.7 (4)
0x2d0| e4 01 00 00 | .... | offset_to_record: 484 0x2d8-0x2db.7 (4)
0x2d0| 00 00 00 00| ....| unused: 0 0x2dc-0x2df.7 (4)
| | | [11]{}: entry 0x240-0x2eb.7 (172)
| | | record{}: 0x240-0x247.7 (8)
0x240|00 00 00 00 |.... | length: 0 0x240-0x243.7 (4)
0x240| 01 05 00 00 | .... | type: "boolean_true" (1281) (True) 0x244-0x247.7 (4)
0x2e0|30 20 00 00 |0 .. | key: "volume_is_root" (8240) (True if the volume was the filesystem root) 0x2e0-0x2e3.7 (4)
0x2e0| 10 02 00 00 | .... | offset_to_record: 528 0x2e4-0x2e7.7 (4)
0x2e0| 00 00 00 00 | .... | unused: 0 0x2e8-0x2eb.7 (4)
| | | [12]{}: entry 0x188-0x2f7.7 (368)
| | | record{}: 0x188-0x197.7 (16)
0x180| 08 00 00 00 | .... | length: 8 0x188-0x18b.7 (4)
0x180| 04 03 00 00| ....| type: "long" (772) ((signed 64-bit) 8-byte number) 0x18c-0x18f.7 (4)
0x190|04 00 00 00 00 00 00 00 |........ | data: 4 0x190-0x197.7 (8)
0x2e0| 01 c0 00 00| ....| key: "containing_folder_index" (49153) (Integer index of containing folder in target path array) 0x2ec-0x2ef.7 (4)
0x2f0|58 01 00 00 |X... | offset_to_record: 344 0x2f0-0x2f3.7 (4)
0x2f0| 00 00 00 00 | .... | unused: 0 0x2f4-0x2f7.7 (4)
| | | [13]{}: entry 0x50-0x303.7 (692)
| | | record{}: 0x50-0x67.7 (24)
0x050|0d 00 00 00 |.... | length: 13 0x50-0x53.7 (4)
0x050| 01 01 00 00 | .... | type: "string" (257) (UTF-8 String) 0x54-0x57.7 (4)
0x050| 64 61 76 69 64 6d 63 64| davidmcd| data: "davidmcdonald" 0x58-0x64.7 (13)
0x060|6f 6e 61 6c 64 |onald |
0x060| 00 00 00 | ... | alignment_bytes: raw bits 0x65-0x67.7 (3)
0x2f0| 11 c0 00 00 | .... | key: "creator_username" (49169) (Name of user that created bookmark) 0x2f8-0x2fb.7 (4)
0x2f0| 20 00 00 00| ...| offset_to_record: 32 0x2fc-0x2ff.7 (4)
0x300|00 00 00 00 |.... | unused: 0 0x300-0x303.7 (4)
| | | [14]{}: entry 0x198-0x30f.7 (376)
| | | record{}: 0x198-0x1a3.7 (12)
0x190| 04 00 00 00 | .... | length: 4 0x198-0x19b.7 (4)
0x190| 03 03 00 00| ....| type: "int" (771) ((signed 32-bit) 4-byte number) 0x19c-0x19f.7 (4)
0x1a0|f5 01 00 00 |.... | data: 501 0x1a0-0x1a3.7 (4)
0x300| 12 c0 00 00 | .... | key: "creator_uid" (49170) (UID of user that created bookmark) 0x304-0x307.7 (4)
0x300| 68 01 00 00 | h... | offset_to_record: 360 0x308-0x30b.7 (4)
0x300| 00 00 00 00| ....| unused: 0 0x30c-0x30f.7 (4)
| | | [15]{}: entry 0x34-0x31b.7 (744)
| | | record{}: 0x34-0x3f.7 (12)
0x030| 04 00 00 00 | .... | length: 4 0x34-0x37.7 (4)
0x030| 03 03 00 00 | .... | type: "int" (771) ((signed 32-bit) 4-byte number) 0x38-0x3b.7 (4)
0x030| 00 00 00 20| ... | data: 536870912 0x3c-0x3f.7 (4)
0x310|10 d0 00 00 |.... | key: "creation_options" (53264) (Integer containing flags passed to CFURLCreateBookmarkData) 0x310-0x313.7 (4)
0x310| 04 00 00 00 | .... | offset_to_record: 4 0x314-0x317.7 (4)
0x310| 00 00 00 00 | .... | unused: 0 0x318-0x31b.7 (4)
| | | [16]{}: entry 0xa8-0x327.7 (640)
| | | record{}: 0xa8-0xb7.7 (16)
0x0a0| 05 00 00 00 | .... | length: 5 0xa8-0xab.7 (4)
0x0a0| 01 01 00 00| ....| type: "string" (257) (UTF-8 String) 0xac-0xaf.7 (4)
0x0b0|42 6c 69 6e 6b |Blink | data: "Blink" 0xb0-0xb4.7 (5)
0x0b0| 00 00 00 | ... | alignment_bytes: raw bits 0xb5-0xb7.7 (3)
0x310| 17 f0 00 00| ....| key: "display_name" (61463) (String) 0x31c-0x31f.7 (4)
0x320|78 00 00 00 |x... | offset_to_record: 120 0x320-0x323.7 (4)
0x320| 00 00 00 00 | .... | unused: 0 0x324-0x327.7 (4)
| | | [17]{}: entry 0x240-0x333.7 (244)
| | | record{}: 0x240-0x247.7 (8)
0x240|00 00 00 00 |.... | length: 0 0x240-0x243.7 (4)
0x240| 01 05 00 00 | .... | type: "boolean_true" (1281) (True) 0x244-0x247.7 (4)
0x320| 0f 00 0f 00 | .... | key: 983055 0x328-0x32b.7 (4)
0x320| 10 02 00 00| ....| offset_to_record: 528 0x32c-0x32f.7 (4)
0x330|00 00 00 00| |....| | unused: 0 0x330-0x333.7 (4)
| | | toc_headers[0:1]: 0x248-0x25b.7 (20)
| | | [0]{}: toc_header 0x248-0x25b.7 (20)
0x240| e4 00 00 00 | .... | toc_size: 228 0x248-0x24b.7 (4)
0x240| fe ff ff ff| ....| magic: 4294967294 (valid) 0x24c-0x24f.7 (4)
0x250|01 00 00 00 |.... | identifier: 1 0x250-0x253.7 (4)
0x250| 00 00 00 00 | .... | next_toc_offset: 0 0x254-0x257.7 (4)
0x250| 12 00 00 00 | .... | num_entries_in_toc: 18 0x258-0x25b.7 (4)
$ fq torepr sample3.book
{
"983055": true,
"containing_folder_index": 4,
"creation_options": 536870912,
"creator_uid": 501,
"creator_username": "davidmcdonald",
"display_name": "Blink",
"target_cnid_path": [
23307,
564764,
909650,
925616,
36514705,
36514706
],
"target_creation_date": 673397442.9093685,
"target_flags": {
"is_application": false,
"is_directory": true,
"is_package": false,
"is_regular_file": false,
"is_symbolic_link": false,
"is_volume": false
},
"target_path": [
"Users",
"davidmcdonald",
"Documents",
"Arduino",
"generated_examples",
"Blink"
],
"volume_creation_date": 599558400,
"volume_flags": {
"dont_browse": false,
"is_automount": false,
"is_disk_image": false,
"is_ejectable": false,
"is_external": false,
"is_internal": true,
"is_ipod": false,
"is_local": true,
"is_read_only": false,
"is_removable": false,
"supports_persistent_ids": true
},
"volume_is_root": true,
"volume_name": "Macintosh HD",
"volume_path": "/",
"volume_size": 499963174912,
"volume_url": "file:///",
"volume_uuid": "8A829A03-9B24-4085-8051-18978712E4BE"
}

BIN
format/apple/bookmark/testdata/sample4.book vendored Normal file
View File

Binary file not shown.

430
format/apple/bookmark/testdata/sample4.fqtest vendored Normal file
View File

@ -0,0 +1,430 @@
$ fq dv sample4.book
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: sample4.book (apple_bookmark) 0x0-0x2f3.7 (756)
| | | header{}: 0x0-0x2f.7 (48)
0x000|62 6f 6f 6b |book | magic: "book" (valid) 0x0-0x3.7 (4)
0x000| f4 02 00 00 | .... | total_size: 756 0x4-0x7.7 (4)
0x000| 00 00 04 10 | .... | unknown: 268697600 0x8-0xb.7 (4)
0x000| 30 00 00 00| 0...| header_size: 48 (valid) 0xc-0xf.7 (4)
0x010|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| reserved: raw bits 0x10-0x2f.7 (32)
0x020|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
0x030|d8 01 00 00 |.... | first_toc_offset: 472 0x30-0x33.7 (4)
| | | bookmark_entries[0:18]: 0x34-0x2f3.7 (704)
| | | [0]{}: entry 0x40-0x227.7 (488)
| | | record{}: 0x40-0xab.7 (108)
| | | data[0:5]: 0x40-0xab.7 (108)
| | | [0]{}: element 0x40-0x9b.7 (92)
| | | record{}: 0x40-0x4f.7 (16)
0x040|05 00 00 00 |.... | length: 5 0x40-0x43.7 (4)
0x040| 01 01 00 00 | .... | type: "string" (257) (UTF-8 String) 0x44-0x47.7 (4)
0x040| 55 73 65 72 73 | Users | data: "Users" 0x48-0x4c.7 (5)
0x040| 00 00 00| ...| alignment_bytes: raw bits 0x4d-0x4f.7 (3)
0x090| 10 00 00 00 | .... | offset: 16 0x98-0x9b.7 (4)
| | | [1]{}: element 0x50-0x9f.7 (80)
| | | record{}: 0x50-0x67.7 (24)
0x050|0d 00 00 00 |.... | length: 13 0x50-0x53.7 (4)
0x050| 01 01 00 00 | .... | type: "string" (257) (UTF-8 String) 0x54-0x57.7 (4)
0x050| 64 61 76 69 64 6d 63 64| davidmcd| data: "davidmcdonald" 0x58-0x64.7 (13)
0x060|6f 6e 61 6c 64 |onald |
0x060| 00 00 00 | ... | alignment_bytes: raw bits 0x65-0x67.7 (3)
0x090| 20 00 00 00| ...| offset: 32 0x9c-0x9f.7 (4)
| | | [2]{}: element 0x68-0xa3.7 (60)
| | | record{}: 0x68-0x73.7 (12)
0x060| 02 00 00 00 | .... | length: 2 0x68-0x6b.7 (4)
0x060| 01 01 00 00| ....| type: "string" (257) (UTF-8 String) 0x6c-0x6f.7 (4)
0x070|67 6f |go | data: "go" 0x70-0x71.7 (2)
0x070| 00 00 | .. | alignment_bytes: raw bits 0x72-0x73.7 (2)
0x0a0|38 00 00 00 |8... | offset: 56 0xa0-0xa3.7 (4)
| | | [3]{}: element 0x74-0xa7.7 (52)
| | | record{}: 0x74-0x7f.7 (12)
0x070| 03 00 00 00 | .... | length: 3 0x74-0x77.7 (4)
0x070| 01 01 00 00 | .... | type: "string" (257) (UTF-8 String) 0x78-0x7b.7 (4)
0x070| 73 72 63 | src | data: "src" 0x7c-0x7e.7 (3)
0x070| 00| .| alignment_bytes: raw bits 0x7f-0x7f.7 (1)
0x0a0| 44 00 00 00 | D... | offset: 68 0xa4-0xa7.7 (4)
| | | [4]{}: element 0x80-0xab.7 (44)
| | | record{}: 0x80-0x8f.7 (16)
0x080|06 00 00 00 |.... | length: 6 0x80-0x83.7 (4)
0x080| 01 01 00 00 | .... | type: "string" (257) (UTF-8 String) 0x84-0x87.7 (4)
0x080| 74 68 65 73 69 73 | thesis | data: "thesis" 0x88-0x8d.7 (6)
0x080| 00 00| ..| alignment_bytes: raw bits 0x8e-0x8f.7 (2)
0x0a0| 50 00 00 00 | P... | offset: 80 0xa8-0xab.7 (4)
0x090|14 00 00 00 |.... | length: 20 0x90-0x93.7 (4)
0x090| 01 06 00 00 | .... | type: "array" (1537) (Array of 4-byte offsets to data items) 0x94-0x97.7 (4)
0x210| 04 10 00 00| ....| key: "target_path" (4100) (Array of individual path components) 0x21c-0x21f.7 (4)
0x220|60 00 00 00 |`... | offset_to_record: 96 0x220-0x223.7 (4)
0x220| 00 00 00 00 | .... | unused: 0 0x224-0x227.7 (4)
| | | [1]{}: entry 0xac-0x233.7 (392)
| | | record{}: 0xac-0x117.7 (108)
| | | data[0:5]: 0xac-0x117.7 (108)
| | | [0]{}: element 0xac-0x107.7 (92)
| | | record{}: 0xac-0xbb.7 (16)
0x0a0| 08 00 00 00| ....| length: 8 0xac-0xaf.7 (4)
0x0b0|04 03 00 00 |.... | type: "long" (772) ((signed 64-bit) 8-byte number) 0xb0-0xb3.7 (4)
0x0b0| 0b 5b 00 00 00 00 00 00 | .[...... | data: 23307 0xb4-0xbb.7 (8)
0x100| 7c 00 00 00 | |... | offset: 124 0x104-0x107.7 (4)
| | | [1]{}: element 0xbc-0x10b.7 (80)
| | | record{}: 0xbc-0xcb.7 (16)
0x0b0| 08 00 00 00| ....| length: 8 0xbc-0xbf.7 (4)
0x0c0|04 03 00 00 |.... | type: "long" (772) ((signed 64-bit) 8-byte number) 0xc0-0xc3.7 (4)
0x0c0| 1c 9e 08 00 00 00 00 00 | ........ | data: 564764 0xc4-0xcb.7 (8)
0x100| 8c 00 00 00 | .... | offset: 140 0x108-0x10b.7 (4)
| | | [2]{}: element 0xcc-0x10f.7 (68)
| | | record{}: 0xcc-0xdb.7 (16)
0x0c0| 08 00 00 00| ....| length: 8 0xcc-0xcf.7 (4)
0x0d0|04 03 00 00 |.... | type: "long" (772) ((signed 64-bit) 8-byte number) 0xd0-0xd3.7 (4)
0x0d0| 44 37 12 00 00 00 00 00 | D7...... | data: 1193796 0xd4-0xdb.7 (8)
0x100| 9c 00 00 00| ....| offset: 156 0x10c-0x10f.7 (4)
| | | [3]{}: element 0xdc-0x113.7 (56)
| | | record{}: 0xdc-0xeb.7 (16)
0x0d0| 08 00 00 00| ....| length: 8 0xdc-0xdf.7 (4)
0x0e0|04 03 00 00 |.... | type: "long" (772) ((signed 64-bit) 8-byte number) 0xe0-0xe3.7 (4)
0x0e0| 9f 15 15 00 00 00 00 00 | ........ | data: 1381791 0xe4-0xeb.7 (8)
0x110|ac 00 00 00 |.... | offset: 172 0x110-0x113.7 (4)
| | | [4]{}: element 0xec-0x117.7 (44)
| | | record{}: 0xec-0xfb.7 (16)
0x0e0| 08 00 00 00| ....| length: 8 0xec-0xef.7 (4)
0x0f0|04 03 00 00 |.... | type: "long" (772) ((signed 64-bit) 8-byte number) 0xf0-0xf3.7 (4)
0x0f0| 53 f0 f9 01 00 00 00 00 | S....... | data: 33157203 0xf4-0xfb.7 (8)
0x110| bc 00 00 00 | .... | offset: 188 0x114-0x117.7 (4)
0x0f0| 14 00 00 00| ....| length: 20 0xfc-0xff.7 (4)
0x100|01 06 00 00 |.... | type: "array" (1537) (Array of 4-byte offsets to data items) 0x100-0x103.7 (4)
0x220| 05 10 00 00 | .... | key: "target_cnid_path" (4101) (Array of CNIDs) 0x228-0x22b.7 (4)
0x220| cc 00 00 00| ....| offset_to_record: 204 0x22c-0x22f.7 (4)
0x230|00 00 00 00 |.... | unused: 0 0x230-0x233.7 (4)
| | | [2]{}: entry 0x128-0x23f.7 (280)
| | | record{}: 0x128-0x147.7 (32)
0x120| 18 00 00 00 | .... | length: 24 (valid) 0x128-0x12b.7 (4)
0x120| 01 02 00 00| ....| raw_type: "data" (513) (valid) 0x12c-0x12f.7 (4)
| | | type: "flag_data" 0x130-NA (0)
| | | property_flags{}: 0x130-0x137.7 (8)
0x130|02 |. | is_hidden: false 0x130-0x130 (0.1)
0x130|02 |. | is_user_immutable: false 0x130.1-0x130.1 (0.1)
0x130|02 |. | is_system_immutable: false 0x130.2-0x130.2 (0.1)
0x130|02 |. | is_package: false 0x130.3-0x130.3 (0.1)
0x130|02 |. | is_volume: false 0x130.4-0x130.4 (0.1)
0x130|02 |. | is_symbolic_link: false 0x130.5-0x130.5 (0.1)
0x130|02 |. | is_directory: true 0x130.6-0x130.6 (0.1)
0x130|02 |. | is_regular_file: false 0x130.7-0x130.7 (0.1)
0x130| 00 | . | is_alias_file: false 0x131-0x131 (0.1)
0x130| 00 | . | is_executable: false 0x131.1-0x131.1 (0.1)
0x130| 00 | . | is_writeable: false 0x131.2-0x131.2 (0.1)
0x130| 00 | . | is_readable: false 0x131.3-0x131.3 (0.1)
0x130| 00 | . | can_set_hidden_extension: false 0x131.4-0x131.4 (0.1)
0x130| 00 | . | is_compressed: false 0x131.5-0x131.5 (0.1)
0x130| 00 | . | is_application: false 0x131.6-0x131.6 (0.1)
0x130| 00 | . | has_hidden_extension: false 0x131.7-0x131.7 (0.1)
0x130| 00 | . | reserved_bits_0: raw bits 0x132-0x132.6 (0.7)
0x130| 00 | . | is_mount_trigger: false 0x132.7-0x132.7 (0.1)
0x130| 00 00 00 00 00 | ..... | reserved: raw bits 0x133-0x137.7 (5)
| | | enabled_property_flags{}: 0x138-0x13f.7 (8)
0x130| 1f | . | is_hidden: false 0x138-0x138 (0.1)
0x130| 1f | . | is_user_immutable: false 0x138.1-0x138.1 (0.1)
0x130| 1f | . | is_system_immutable: false 0x138.2-0x138.2 (0.1)
0x130| 1f | . | is_package: true 0x138.3-0x138.3 (0.1)
0x130| 1f | . | is_volume: true 0x138.4-0x138.4 (0.1)
0x130| 1f | . | is_symbolic_link: true 0x138.5-0x138.5 (0.1)
0x130| 1f | . | is_directory: true 0x138.6-0x138.6 (0.1)
0x130| 1f | . | is_regular_file: true 0x138.7-0x138.7 (0.1)
0x130| 02 | . | is_alias_file: false 0x139-0x139 (0.1)
0x130| 02 | . | is_executable: false 0x139.1-0x139.1 (0.1)
0x130| 02 | . | is_writeable: false 0x139.2-0x139.2 (0.1)
0x130| 02 | . | is_readable: false 0x139.3-0x139.3 (0.1)
0x130| 02 | . | can_set_hidden_extension: false 0x139.4-0x139.4 (0.1)
0x130| 02 | . | is_compressed: false 0x139.5-0x139.5 (0.1)
0x130| 02 | . | is_application: true 0x139.6-0x139.6 (0.1)
0x130| 02 | . | has_hidden_extension: false 0x139.7-0x139.7 (0.1)
0x130| 00 | . | reserved_bits_0: raw bits 0x13a-0x13a.6 (0.7)
0x130| 00 | . | is_mount_trigger: false 0x13a.7-0x13a.7 (0.1)
0x130| 00 00 00 00 00| .....| reserved: raw bits 0x13b-0x13f.7 (5)
0x140|1a 02 00 00 00 00 00 00 |........ | reserved: raw bits 0x140-0x147.7 (8)
0x230| 10 10 00 00 | .... | key: "target_flags" (4112) (flag bitfield) 0x234-0x237.7 (4)
0x230| f8 00 00 00 | .... | offset_to_record: 248 0x238-0x23b.7 (4)
0x230| 00 00 00 00| ....| unused: 0 0x23c-0x23f.7 (4)
| | | [3]{}: entry 0x118-0x24b.7 (308)
| | | record{}: 0x118-0x127.7 (16)
0x110| 08 00 00 00 | .... | length: 8 0x118-0x11b.7 (4)
0x110| 00 04 00 00| ....| type: "date" (1024) (Big-endian IEEE double precision seconds since 2001-01-01 00:00:00 UTC) 0x11c-0x11f.7 (4)
0x120|41 c3 ff 97 6f 07 8e 98 |A...o... | data: 6.710351020590391e+08 (2022-04-07T14:38:22Z) 0x120-0x127.7 (8)
0x240|40 10 00 00 |@... | key: "target_creation_date" (4160) (Date) 0x240-0x243.7 (4)
0x240| e8 00 00 00 | .... | offset_to_record: 232 0x244-0x247.7 (4)
0x240| 00 00 00 00 | .... | unused: 0 0x248-0x24b.7 (4)
| | | [4]{}: entry 0x1f4-0x257.7 (100)
| | | record{}: 0x1f4-0x1ff.7 (12)
0x1f0| 01 00 00 00 | .... | length: 1 0x1f4-0x1f7.7 (4)
0x1f0| 01 01 00 00 | .... | type: "string" (257) (UTF-8 String) 0x1f8-0x1fb.7 (4)
0x1f0| 2f | / | data: "/" 0x1fc-0x1fc.7 (1)
0x1f0| 00 00 00| ...| alignment_bytes: raw bits 0x1fd-0x1ff.7 (3)
0x240| 02 20 00 00| . ..| key: "volume_path" (8194) (Array of individual path components) 0x24c-0x24f.7 (4)
0x250|c4 01 00 00 |.... | offset_to_record: 452 0x250-0x253.7 (4)
0x250| 00 00 00 00 | .... | unused: 0 0x254-0x257.7 (4)
| | | [5]{}: entry 0x164-0x263.7 (256)
| | | record{}: 0x164-0x173.7 (16)
0x160| 08 00 00 00 | .... | length: 8 0x164-0x167.7 (4)
0x160| 01 09 00 00 | .... | type: "url" (2305) (UTF-8 string) 0x168-0x16b.7 (4)
0x160| 66 69 6c 65| file| data: "file:///" 0x16c-0x173.7 (8)
0x170|3a 2f 2f 2f |:/// |
0x250| 05 20 00 00 | . .. | key: "volume_url" (8197) (URL of volume root) 0x258-0x25b.7 (4)
0x250| 34 01 00 00| 4...| offset_to_record: 308 0x25c-0x25f.7 (4)
0x260|00 00 00 00 |.... | unused: 0 0x260-0x263.7 (4)
| | | [6]{}: entry 0x174-0x26f.7 (252)
| | | record{}: 0x174-0x18b.7 (24)
0x170| 0c 00 00 00 | .... | length: 12 0x174-0x177.7 (4)
0x170| 01 01 00 00 | .... | type: "string" (257) (UTF-8 String) 0x178-0x17b.7 (4)
0x170| 4d 61 63 69| Maci| data: "Macintosh HD" 0x17c-0x187.7 (12)
0x180|6e 74 6f 73 68 20 48 44 |ntosh HD |
0x180| 08 00 00 00 | .... | alignment_bytes: raw bits 0x188-0x18b.7 (4)
0x260| 10 20 00 00 | . .. | key: "volume_name" (8208) (String) 0x264-0x267.7 (4)
0x260| 44 01 00 00 | D... | offset_to_record: 324 0x268-0x26b.7 (4)
0x260| 00 00 00 00| ....| unused: 0 0x26c-0x26f.7 (4)
| | | [7]{}: entry 0x1a8-0x27b.7 (212)
| | | record{}: 0x1a8-0x1d7.7 (48)
0x1a0| 24 00 00 00 | $... | length: 36 0x1a8-0x1ab.7 (4)
0x1a0| 01 01 00 00| ....| type: "string" (257) (UTF-8 String) 0x1ac-0x1af.7 (4)
0x1b0|38 41 38 32 39 41 30 33 2d 39 42 32 34 2d 34 30|8A829A03-9B24-40| data: "8A829A03-9B24-4085-8051-18978712E4BE" 0x1b0-0x1d3.7 (36)
* |until 0x1d3.7 (36) | |
0x1d0| 18 00 00 00 | .... | alignment_bytes: raw bits 0x1d4-0x1d7.7 (4)
0x270|11 20 00 00 |. .. | key: "volume_uuid" (8209) (String UUID) 0x270-0x273.7 (4)
0x270| 78 01 00 00 | x... | offset_to_record: 376 0x274-0x277.7 (4)
0x270| 00 00 00 00 | .... | unused: 0 0x278-0x27b.7 (4)
| | | [8]{}: entry 0x188-0x287.7 (256)
| | | record{}: 0x188-0x197.7 (16)
0x180| 08 00 00 00 | .... | length: 8 0x188-0x18b.7 (4)
0x180| 04 03 00 00| ....| type: "long" (772) ((signed 64-bit) 8-byte number) 0x18c-0x18f.7 (4)
0x190|00 a0 20 68 74 00 00 00 |.. ht... | data: 499963174912 0x190-0x197.7 (8)
0x270| 12 20 00 00| . ..| key: "volume_size" (8210) (8-byte integer) 0x27c-0x27f.7 (4)
0x280|58 01 00 00 |X... | offset_to_record: 344 0x280-0x283.7 (4)
0x280| 00 00 00 00 | .... | unused: 0 0x284-0x287.7 (4)
| | | [9]{}: entry 0x198-0x293.7 (252)
| | | record{}: 0x198-0x1a7.7 (16)
0x190| 08 00 00 00 | .... | length: 8 0x198-0x19b.7 (4)
0x190| 00 04 00 00| ....| type: "date" (1024) (Big-endian IEEE double precision seconds since 2001-01-01 00:00:00 UTC) 0x19c-0x19f.7 (4)
0x1a0|41 c1 de 44 80 00 00 00 |A..D.... | data: 5.995584e+08 (2020-01-01T08:00:00Z) 0x1a0-0x1a7.7 (8)
0x280| 13 20 00 00 | . .. | key: "volume_creation_date" (8211) (Date) 0x288-0x28b.7 (4)
0x280| 68 01 00 00| h...| offset_to_record: 360 0x28c-0x28f.7 (4)
0x290|00 00 00 00 |.... | unused: 0 0x290-0x293.7 (4)
| | | [10]{}: entry 0x1d4-0x29f.7 (204)
| | | record{}: 0x1d4-0x1f3.7 (32)
0x1d0| 18 00 00 00 | .... | length: 24 (valid) 0x1d4-0x1d7.7 (4)
0x1d0| 01 02 00 00 | .... | raw_type: "data" (513) (valid) 0x1d8-0x1db.7 (4)
| | | type: "flag_data" 0x1dc-NA (0)
| | | property_flags{}: 0x1dc-0x1e3.7 (8)
0x1d0| 81 | . | is_internal: true 0x1dc-0x1dc (0.1)
0x1d0| 81 | . | is_removable: false 0x1dc.1-0x1dc.1 (0.1)
0x1d0| 81 | . | is_ejectable: false 0x1dc.2-0x1dc.2 (0.1)
0x1d0| 81 | . | is_quarantined: false 0x1dc.3-0x1dc.3 (0.1)
0x1d0| 81 | . | is_read_only: false 0x1dc.4-0x1dc.4 (0.1)
0x1d0| 81 | . | dont_browse: false 0x1dc.5-0x1dc.5 (0.1)
0x1d0| 81 | . | is_automount: false 0x1dc.6-0x1dc.6 (0.1)
0x1d0| 81 | . | is_local: true 0x1dc.7-0x1dc.7 (0.1)
0x1d0| 00 | . | is_dvd: false 0x1dd-0x1dd (0.1)
0x1d0| 00 | . | is_cd: false 0x1dd.1-0x1dd.1 (0.1)
0x1d0| 00 | . | is_idisk: false 0x1dd.2-0x1dd.2 (0.1)
0x1d0| 00 | . | is_ipod: false 0x1dd.3-0x1dd.3 (0.1)
0x1d0| 00 | . | is_local_idisk_mirror: false 0x1dd.4-0x1dd.4 (0.1)
0x1d0| 00 | . | is_file_vault: false 0x1dd.5-0x1dd.5 (0.1)
0x1d0| 00 | . | is_disk_image: false 0x1dd.6-0x1dd.6 (0.1)
0x1d0| 00 | . | is_external: false 0x1dd.7-0x1dd.7 (0.1)
0x1d0| 00 | . | reserved_0: raw bits 0x1de-0x1de.6 (0.7)
0x1d0| 00 | . | is_device_file_system: false 0x1de.7-0x1de.7 (0.1)
0x1d0| 00| .| reserved_1: raw bits 0x1df-0x1df.7 (1)
0x1e0|01 |. | supports_read_dir_attr: false 0x1e0-0x1e0 (0.1)
0x1e0|01 |. | supports_copy_file: false 0x1e0.1-0x1e0.1 (0.1)
0x1e0|01 |. | supports_deny_modes: false 0x1e0.2-0x1e0.2 (0.1)
0x1e0|01 |. | supports_symbolic_links: false 0x1e0.3-0x1e0.3 (0.1)
0x1e0|01 |. | reserved_2: false 0x1e0.4-0x1e0.4 (0.1)
0x1e0|01 |. | supports_exchange: false 0x1e0.5-0x1e0.5 (0.1)
0x1e0|01 |. | supports_search_fs: false 0x1e0.6-0x1e0.6 (0.1)
0x1e0|01 |. | supports_persistent_ids: true 0x1e0.7-0x1e0.7 (0.1)
0x1e0| 00 | . | supports_extended_security: false 0x1e1-0x1e1 (0.1)
0x1e0| 00 | . | has_no_root_directory_times: false 0x1e1.1-0x1e1.1 (0.1)
0x1e0| 00 | . | supports_flock: false 0x1e1.2-0x1e1.2 (0.1)
0x1e0| 00 | . | supports_case_preserved_names: false 0x1e1.3-0x1e1.3 (0.1)
0x1e0| 00 | . | supports_case_sensitive_names: false 0x1e1.4-0x1e1.4 (0.1)
0x1e0| 00 | . | supports_fast_stat_fs: false 0x1e1.5-0x1e1.5 (0.1)
0x1e0| 00 | . | supports_rename: false 0x1e1.6-0x1e1.6 (0.1)
0x1e0| 00 | . | supports_journaling: false 0x1e1.7-0x1e1.7 (0.1)
0x1e0| 00 | . | supports_zero_runs: false 0x1e2-0x1e2 (0.1)
0x1e0| 00 | . | supports_sparse_files: false 0x1e2.1-0x1e2.1 (0.1)
0x1e0| 00 | . | is_journaling: false 0x1e2.2-0x1e2.2 (0.1)
0x1e0| 00 | . | reserved_3: false 0x1e2.3-0x1e2.3 (0.1)
0x1e0| 00 | . | supports_path_from_id: false 0x1e2.4-0x1e2.4 (0.1)
0x1e0| 00 | . | supports_mandatory_byte_range_locks: false 0x1e2.5-0x1e2.5 (0.1)
0x1e0| 00 | . | supports_hard_links: false 0x1e2.6-0x1e2.6 (0.1)
0x1e0| 00 | . | supports_2_tb_file_size: false 0x1e2.7-0x1e2.7 (0.1)
0x1e0| 00 | . | reserved_4: raw bits 0x1e3-0x1e3.2 (0.3)
0x1e0| 00 | . | has64_bit_object_ids: false 0x1e3.3-0x1e3.3 (0.1)
0x1e0| 00 | . | supports_decmp_fs_compression: false 0x1e3.4-0x1e3.4 (0.1)
0x1e0| 00 | . | supports_hidden_files: false 0x1e3.5-0x1e3.5 (0.1)
0x1e0| 00 | . | supports_remote_events: false 0x1e3.6-0x1e3.6 (0.1)
0x1e0| 00 | . | supports_volume_sizes: false 0x1e3.7-0x1e3.7 (0.1)
| | | enabled_property_flags{}: 0x1e4-0x1eb.7 (8)
0x1e0| ef | . | is_internal: true 0x1e4-0x1e4 (0.1)
0x1e0| ef | . | is_removable: true 0x1e4.1-0x1e4.1 (0.1)
0x1e0| ef | . | is_ejectable: true 0x1e4.2-0x1e4.2 (0.1)
0x1e0| ef | . | is_quarantined: false 0x1e4.3-0x1e4.3 (0.1)
0x1e0| ef | . | is_read_only: true 0x1e4.4-0x1e4.4 (0.1)
0x1e0| ef | . | dont_browse: true 0x1e4.5-0x1e4.5 (0.1)
0x1e0| ef | . | is_automount: true 0x1e4.6-0x1e4.6 (0.1)
0x1e0| ef | . | is_local: true 0x1e4.7-0x1e4.7 (0.1)
0x1e0| 13 | . | is_dvd: false 0x1e5-0x1e5 (0.1)
0x1e0| 13 | . | is_cd: false 0x1e5.1-0x1e5.1 (0.1)
0x1e0| 13 | . | is_idisk: false 0x1e5.2-0x1e5.2 (0.1)
0x1e0| 13 | . | is_ipod: true 0x1e5.3-0x1e5.3 (0.1)
0x1e0| 13 | . | is_local_idisk_mirror: false 0x1e5.4-0x1e5.4 (0.1)
0x1e0| 13 | . | is_file_vault: false 0x1e5.5-0x1e5.5 (0.1)
0x1e0| 13 | . | is_disk_image: true 0x1e5.6-0x1e5.6 (0.1)
0x1e0| 13 | . | is_external: true 0x1e5.7-0x1e5.7 (0.1)
0x1e0| 00 | . | reserved_0: raw bits 0x1e6-0x1e6.6 (0.7)
0x1e0| 00 | . | is_device_file_system: false 0x1e6.7-0x1e6.7 (0.1)
0x1e0| 00 | . | reserved_1: raw bits 0x1e7-0x1e7.7 (1)
0x1e0| 01 | . | supports_read_dir_attr: false 0x1e8-0x1e8 (0.1)
0x1e0| 01 | . | supports_copy_file: false 0x1e8.1-0x1e8.1 (0.1)
0x1e0| 01 | . | supports_deny_modes: false 0x1e8.2-0x1e8.2 (0.1)
0x1e0| 01 | . | supports_symbolic_links: false 0x1e8.3-0x1e8.3 (0.1)
0x1e0| 01 | . | reserved_2: false 0x1e8.4-0x1e8.4 (0.1)
0x1e0| 01 | . | supports_exchange: false 0x1e8.5-0x1e8.5 (0.1)
0x1e0| 01 | . | supports_search_fs: false 0x1e8.6-0x1e8.6 (0.1)
0x1e0| 01 | . | supports_persistent_ids: true 0x1e8.7-0x1e8.7 (0.1)
0x1e0| 00 | . | supports_extended_security: false 0x1e9-0x1e9 (0.1)
0x1e0| 00 | . | has_no_root_directory_times: false 0x1e9.1-0x1e9.1 (0.1)
0x1e0| 00 | . | supports_flock: false 0x1e9.2-0x1e9.2 (0.1)
0x1e0| 00 | . | supports_case_preserved_names: false 0x1e9.3-0x1e9.3 (0.1)
0x1e0| 00 | . | supports_case_sensitive_names: false 0x1e9.4-0x1e9.4 (0.1)
0x1e0| 00 | . | supports_fast_stat_fs: false 0x1e9.5-0x1e9.5 (0.1)
0x1e0| 00 | . | supports_rename: false 0x1e9.6-0x1e9.6 (0.1)
0x1e0| 00 | . | supports_journaling: false 0x1e9.7-0x1e9.7 (0.1)
0x1e0| 00 | . | supports_zero_runs: false 0x1ea-0x1ea (0.1)
0x1e0| 00 | . | supports_sparse_files: false 0x1ea.1-0x1ea.1 (0.1)
0x1e0| 00 | . | is_journaling: false 0x1ea.2-0x1ea.2 (0.1)
0x1e0| 00 | . | reserved_3: false 0x1ea.3-0x1ea.3 (0.1)
0x1e0| 00 | . | supports_path_from_id: false 0x1ea.4-0x1ea.4 (0.1)
0x1e0| 00 | . | supports_mandatory_byte_range_locks: false 0x1ea.5-0x1ea.5 (0.1)
0x1e0| 00 | . | supports_hard_links: false 0x1ea.6-0x1ea.6 (0.1)
0x1e0| 00 | . | supports_2_tb_file_size: false 0x1ea.7-0x1ea.7 (0.1)
0x1e0| 00 | . | reserved_4: raw bits 0x1eb-0x1eb.2 (0.3)
0x1e0| 00 | . | has64_bit_object_ids: false 0x1eb.3-0x1eb.3 (0.1)
0x1e0| 00 | . | supports_decmp_fs_compression: false 0x1eb.4-0x1eb.4 (0.1)
0x1e0| 00 | . | supports_hidden_files: false 0x1eb.5-0x1eb.5 (0.1)
0x1e0| 00 | . | supports_remote_events: false 0x1eb.6-0x1eb.6 (0.1)
0x1e0| 00 | . | supports_volume_sizes: false 0x1eb.7-0x1eb.7 (0.1)
0x1e0| 00 00 00 00| ....| reserved: raw bits 0x1ec-0x1f3.7 (8)
0x1f0|00 00 00 00 |.... |
0x290| 20 20 00 00 | .. | key: "volume_flags" (8224) (flag bitfield) 0x294-0x297.7 (4)
0x290| a4 01 00 00 | .... | offset_to_record: 420 0x298-0x29b.7 (4)
0x290| 00 00 00 00| ....| unused: 0 0x29c-0x29f.7 (4)
| | | [11]{}: entry 0x200-0x2ab.7 (172)
| | | record{}: 0x200-0x207.7 (8)
0x200|00 00 00 00 |.... | length: 0 0x200-0x203.7 (4)
0x200| 01 05 00 00 | .... | type: "boolean_true" (1281) (True) 0x204-0x207.7 (4)
0x2a0|30 20 00 00 |0 .. | key: "volume_is_root" (8240) (True if the volume was the filesystem root) 0x2a0-0x2a3.7 (4)
0x2a0| d0 01 00 00 | .... | offset_to_record: 464 0x2a4-0x2a7.7 (4)
0x2a0| 00 00 00 00 | .... | unused: 0 0x2a8-0x2ab.7 (4)
| | | [12]{}: entry 0x148-0x2b7.7 (368)
| | | record{}: 0x148-0x157.7 (16)
0x140| 08 00 00 00 | .... | length: 8 0x148-0x14b.7 (4)
0x140| 04 03 00 00| ....| type: "long" (772) ((signed 64-bit) 8-byte number) 0x14c-0x14f.7 (4)
0x150|03 00 00 00 00 00 00 00 |........ | data: 3 0x150-0x157.7 (8)
0x2a0| 01 c0 00 00| ....| key: "containing_folder_index" (49153) (Integer index of containing folder in target path array) 0x2ac-0x2af.7 (4)
0x2b0|18 01 00 00 |.... | offset_to_record: 280 0x2b0-0x2b3.7 (4)
0x2b0| 00 00 00 00 | .... | unused: 0 0x2b4-0x2b7.7 (4)
| | | [13]{}: entry 0x50-0x2c3.7 (628)
| | | record{}: 0x50-0x67.7 (24)
0x050|0d 00 00 00 |.... | length: 13 0x50-0x53.7 (4)
0x050| 01 01 00 00 | .... | type: "string" (257) (UTF-8 String) 0x54-0x57.7 (4)
0x050| 64 61 76 69 64 6d 63 64| davidmcd| data: "davidmcdonald" 0x58-0x64.7 (13)
0x060|6f 6e 61 6c 64 |onald |
0x060| 00 00 00 | ... | alignment_bytes: raw bits 0x65-0x67.7 (3)
0x2b0| 11 c0 00 00 | .... | key: "creator_username" (49169) (Name of user that created bookmark) 0x2b8-0x2bb.7 (4)
0x2b0| 20 00 00 00| ...| offset_to_record: 32 0x2bc-0x2bf.7 (4)
0x2c0|00 00 00 00 |.... | unused: 0 0x2c0-0x2c3.7 (4)
| | | [14]{}: entry 0x158-0x2cf.7 (376)
| | | record{}: 0x158-0x163.7 (12)
0x150| 04 00 00 00 | .... | length: 4 0x158-0x15b.7 (4)
0x150| 03 03 00 00| ....| type: "int" (771) ((signed 32-bit) 4-byte number) 0x15c-0x15f.7 (4)
0x160|f5 01 00 00 |.... | data: 501 0x160-0x163.7 (4)
0x2c0| 12 c0 00 00 | .... | key: "creator_uid" (49170) (UID of user that created bookmark) 0x2c4-0x2c7.7 (4)
0x2c0| 28 01 00 00 | (... | offset_to_record: 296 0x2c8-0x2cb.7 (4)
0x2c0| 00 00 00 00| ....| unused: 0 0x2cc-0x2cf.7 (4)
| | | [15]{}: entry 0x34-0x2db.7 (680)
| | | record{}: 0x34-0x3f.7 (12)
0x030| 04 00 00 00 | .... | length: 4 0x34-0x37.7 (4)
0x030| 03 03 00 00 | .... | type: "int" (771) ((signed 32-bit) 4-byte number) 0x38-0x3b.7 (4)
0x030| 00 00 00 20| ... | data: 536870912 0x3c-0x3f.7 (4)
0x2d0|10 d0 00 00 |.... | key: "creation_options" (53264) (Integer containing flags passed to CFURLCreateBookmarkData) 0x2d0-0x2d3.7 (4)
0x2d0| 04 00 00 00 | .... | offset_to_record: 4 0x2d4-0x2d7.7 (4)
0x2d0| 00 00 00 00 | .... | unused: 0 0x2d8-0x2db.7 (4)
| | | [16]{}: entry 0x80-0x2e7.7 (616)
| | | record{}: 0x80-0x8f.7 (16)
0x080|06 00 00 00 |.... | length: 6 0x80-0x83.7 (4)
0x080| 01 01 00 00 | .... | type: "string" (257) (UTF-8 String) 0x84-0x87.7 (4)
0x080| 74 68 65 73 69 73 | thesis | data: "thesis" 0x88-0x8d.7 (6)
0x080| 00 00| ..| alignment_bytes: raw bits 0x8e-0x8f.7 (2)
0x2d0| 17 f0 00 00| ....| key: "display_name" (61463) (String) 0x2dc-0x2df.7 (4)
0x2e0|50 00 00 00 |P... | offset_to_record: 80 0x2e0-0x2e3.7 (4)
0x2e0| 00 00 00 00 | .... | unused: 0 0x2e4-0x2e7.7 (4)
| | | [17]{}: entry 0x200-0x2f3.7 (244)
| | | record{}: 0x200-0x207.7 (8)
0x200|00 00 00 00 |.... | length: 0 0x200-0x203.7 (4)
0x200| 01 05 00 00 | .... | type: "boolean_true" (1281) (True) 0x204-0x207.7 (4)
0x2e0| 0f 00 0f 00 | .... | key: 983055 0x2e8-0x2eb.7 (4)
0x2e0| d0 01 00 00| ....| offset_to_record: 464 0x2ec-0x2ef.7 (4)
0x2f0|00 00 00 00| |....| | unused: 0 0x2f0-0x2f3.7 (4)
| | | toc_headers[0:1]: 0x208-0x21b.7 (20)
| | | [0]{}: toc_header 0x208-0x21b.7 (20)
0x200| e4 00 00 00 | .... | toc_size: 228 0x208-0x20b.7 (4)
0x200| fe ff ff ff| ....| magic: 4294967294 (valid) 0x20c-0x20f.7 (4)
0x210|01 00 00 00 |.... | identifier: 1 0x210-0x213.7 (4)
0x210| 00 00 00 00 | .... | next_toc_offset: 0 0x214-0x217.7 (4)
0x210| 12 00 00 00 | .... | num_entries_in_toc: 18 0x218-0x21b.7 (4)
$ fq torepr sample4.book
{
"983055": true,
"containing_folder_index": 3,
"creation_options": 536870912,
"creator_uid": 501,
"creator_username": "davidmcdonald",
"display_name": "thesis",
"target_cnid_path": [
23307,
564764,
1193796,
1381791,
33157203
],
"target_creation_date": 671035102.0590391,
"target_flags": {
"is_application": false,
"is_directory": true,
"is_package": false,
"is_regular_file": false,
"is_symbolic_link": false,
"is_volume": false
},
"target_path": [
"Users",
"davidmcdonald",
"go",
"src",
"thesis"
],
"volume_creation_date": 599558400,
"volume_flags": {
"dont_browse": false,
"is_automount": false,
"is_disk_image": false,
"is_ejectable": false,
"is_external": false,
"is_internal": true,
"is_ipod": false,
"is_local": true,
"is_read_only": false,
"is_removable": false,
"supports_persistent_ids": true
},
"volume_is_root": true,
"volume_name": "Macintosh HD",
"volume_path": "/",
"volume_size": 499963174912,
"volume_url": "file:///",
"volume_uuid": "8A829A03-9B24-4085-8051-18978712E4BE"
}

133
format/bplist/bplist.go → format/apple/bplist/bplist.go
View File

@ -7,23 +7,25 @@ import (
"time"
"github.com/wader/fq/format"
"github.com/wader/fq/format/apple"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
"github.com/wader/fq/pkg/scalar"
)
//go:embed bplist.jq bplist.md
//go:embed bplist.jq bplist.md ns_keyed_archiver.jq
var bplistFS embed.FS
func init() {
interp.RegisterFormat(decode.Format{
Name: format.BPLIST,
ProbeOrder: format.ProbeOrderBinUnique,
Description: "Apple Binary Property List",
Groups: []string{format.PROBE},
DecodeFn: bplistDecode,
Functions: []string{"torepr"},
})
interp.RegisterFormat(
format.Bplist,
&decode.Format{
ProbeOrder: format.ProbeOrderBinUnique,
Description: "Apple Binary Property List",
Groups: []*decode.Group{format.Probe},
DecodeFn: bplistDecode,
Functions: []string{"torepr"},
})
interp.RegisterFS(bplistFS)
}
@ -47,7 +49,7 @@ const (
boolTrue = 0x09
)
var elementTypeMap = scalar.UToScalar{
var elementTypeMap = scalar.UintMap{
elementTypeNullOrBoolOrFill: {Sym: "singleton", Description: "Singleton value (null/bool)"},
elementTypeInt: {Sym: "int", Description: "Integer"},
elementTypeReal: {Sym: "real", Description: "Floating Point Number"},
@ -64,13 +66,13 @@ var elementTypeMap = scalar.UToScalar{
var cocoaTimeEpochDate = time.Date(2001, time.January, 1, 0, 0, 0, 0, time.UTC)
// decodes the number of bits required to store the following object
func decodeSize(d *decode.D, sms ...scalar.Mapper) uint64 {
func decodeSize(d *decode.D, sms ...scalar.UintMapper) uint64 {
n := d.FieldU4("size_bits")
if n != 0x0f {
return n
}
d.FieldU4("large_size_marker", d.AssertU(0b0001))
d.FieldU4("large_size_marker", d.UintAssert(0b0001))
// get the exponent value
n = d.FieldU4("exponent")
@ -79,11 +81,11 @@ func decodeSize(d *decode.D, sms ...scalar.Mapper) uint64 {
n = 1 << n
// decode that many bytes as big endian
n = d.FieldUFn(
n = d.FieldUintFn(
"size_bigint",
func(d *decode.D) uint64 {
v := d.UBigInt(int(n * 8))
d.AssertBigIntRange(big.NewInt(1), big.NewInt(math.MaxInt64))
d.BigIntAssertRange(big.NewInt(1), big.NewInt(math.MaxInt64))
return v.Uint64()
}, sms...)
@ -98,13 +100,13 @@ func decodeItem(d *decode.D, p *plist) bool {
m := d.FieldU4("type", elementTypeMap)
switch m {
case elementTypeNullOrBoolOrFill:
d.FieldU4("value", scalar.UToScalar{
null: scalar.S{Sym: nil},
boolTrue: scalar.S{Sym: true},
boolFalse: scalar.S{Sym: false},
d.FieldU4("value", scalar.UintMap{
null: scalar.Uint{Sym: nil},
boolTrue: scalar.Uint{Sym: true},
boolFalse: scalar.Uint{Sym: false},
})
case elementTypeInt:
n := d.FieldUFn("size", func(d *decode.D) uint64 {
case elementTypeInt, elementTypeUID:
n := d.FieldUintFn("size", func(d *decode.D) uint64 {
return 1 << d.U4()
})
switch n {
@ -123,33 +125,29 @@ func decodeItem(d *decode.D, p *plist) bool {
}
case elementTypeReal:
n := 1 << decodeSize(d)
d.FieldValueU("size", uint64(n))
d.FieldValueUint("size", uint64(n))
d.FieldF("value", n*8)
case elementTypeDate:
n := 1 << decodeSize(d, d.AssertU(4, 8))
d.FieldValueU("size", uint64(n))
d.FieldF("value", n*8, scalar.DescriptionTimeFn(scalar.S.TryActualF, cocoaTimeEpochDate, time.RFC3339))
n := 1 << decodeSize(d, d.UintAssert(4, 8))
d.FieldValueUint("size", uint64(n))
d.FieldF("value", n*8, scalar.FltActualDate(cocoaTimeEpochDate, time.RFC3339))
case elementTypeData:
n := decodeSize(d)
d.FieldValueU("size", n)
d.FieldValueUint("size", n)
d.FieldRawLen("value", int64(n*8))
case elementTypeASCIIString:
n := decodeSize(d)
d.FieldValueU("size", n)
d.FieldValueUint("size", n)
d.FieldUTF8("value", int(n))
return true
case elementTypeUnicodeString:
n := decodeSize(d)
d.FieldValueU("size", n)
d.FieldUTF16("value", int(n))
d.FieldValueUint("size", n)
d.FieldUTF16BE("value", int(n))
return true
case elementTypeUID:
n := decodeSize(d)
d.FieldValueU("size", n)
d.FieldUBigInt("value", int(n)).Uint64()
case elementTypeArray:
n := decodeSize(d)
d.FieldValueU("size", n)
d.FieldValueUint("size", n)
d.FieldStructNArray("entries", "entry", int64(n),
func(d *decode.D) {
idx := d.FieldU("object_index", int(p.t.objRefSize)*8)
@ -157,7 +155,7 @@ func decodeItem(d *decode.D, p *plist) bool {
})
case elementTypeSet:
n := decodeSize(d)
d.FieldValueU("size", n)
d.FieldValueUint("size", n)
d.FieldStructNArray("entries", "entry", int64(n),
func(d *decode.D) {
idx := d.FieldU("object_index", int(p.t.objRefSize)*8)
@ -165,7 +163,7 @@ func decodeItem(d *decode.D, p *plist) bool {
})
case elementTypeDict:
n := decodeSize(d)
d.FieldValueU("size", n)
d.FieldValueUint("size", n)
d.FieldStructNArray("entries", "entry", int64(n),
func(d *decode.D) {
var ki, vi uint64
@ -198,13 +196,9 @@ func (pl *plist) decodeReference(d *decode.D, idx uint64) bool {
d.Errorf("index %d out of bounds for object table size %d", idx, len(pl.o))
return false
}
pl.consumed[idx] = true
if pl.indexIsInStack(idx) {
d.Fatalf("recursion detected: object %d already decoded in stack %v", idx, pl.objectStack)
return false
}
pl.pushIndex(idx)
defer pl.pld.PushAndPop(idx, func() { d.Fatalf("infinite recursion detected") })()
itemOffset := pl.o[idx]
if itemOffset >= pl.t.offsetTableStart {
@ -216,7 +210,6 @@ func (pl *plist) decodeReference(d *decode.D, idx uint64) bool {
d.SeekAbs(int64(itemOffset*8), func(d *decode.D) {
isString = decodeItem(d, pl)
})
pl.popIndex()
return isString
}
@ -229,42 +222,27 @@ type trailer struct {
}
type plist struct {
t trailer
o []uint64
objectStack []uint64
t trailer
o []uint64
consumed map[uint64]bool
pld apple.PosLoopDetector[uint64]
}
func (pl *plist) pushIndex(idx uint64) {
pl.objectStack = append(pl.objectStack, idx)
}
func (pl *plist) popIndex() {
pl.objectStack = pl.objectStack[:len(pl.objectStack)-1]
}
func (pl *plist) indexIsInStack(idx uint64) bool {
for _, existing := range pl.objectStack {
if existing == idx {
return true
}
}
return false
}
func bplistDecode(d *decode.D, _ any) any {
func bplistDecode(d *decode.D) any {
d.FieldStruct("header", func(d *decode.D) {
d.FieldUTF8("magic", 6, d.AssertStr("bplist"))
d.FieldUTF8("version", 2, d.AssertStr("00"))
d.FieldUTF8("magic", 6, d.StrAssert("bplist"))
d.FieldUTF8("version", 2, d.StrAssert("00"))
})
p := new(plist)
p.consumed = make(map[uint64]bool)
d.SeekAbs(d.Len()-32*8, func(d *decode.D) {
d.FieldStruct("trailer", func(d *decode.D) {
d.FieldU40("unused")
d.FieldS8("sort_version")
p.t.offTblOffSize = d.FieldU8("offset_table_offset_size", d.AssertURange(1, 8))
p.t.objRefSize = d.FieldU8("object_reference_size", d.AssertURange(1, 8))
p.t.offTblOffSize = d.FieldU8("offset_table_offset_size", d.UintAssertRange(1, 8))
p.t.objRefSize = d.FieldU8("object_reference_size", d.UintAssertRange(1, 8))
p.t.nObjects = d.FieldU64("object_count")
p.t.topObjectOffset = d.FieldU64("top_object_offset")
p.t.offsetTableStart = d.FieldU64("offset_table_start")
@ -288,5 +266,26 @@ func bplistDecode(d *decode.D, _ any) any {
p.decodeReference(d, 0)
})
var lost []uint64
for i := uint64(0); i < p.t.nObjects; i++ {
if _, isUsed := p.consumed[i]; !isUsed {
lost = append(lost, i)
}
}
if len(lost) == 0 {
return nil
}
i := 0
d.FieldStructNArray("lost_and_found", "entry",
int64(len(lost)),
func(d *decode.D) {
p.decodeReference(d, lost[i])
i++
})
return nil
}

4
format/bplist/bplist.jq → format/apple/bplist/bplist.jq
View File

@ -7,7 +7,7 @@ def _bplist_torepr:
elif .type == "data" then .value | tovalue
elif .type == "ascii_string" then .value | tovalue
elif .type == "unicode_string" then .value | tovalue
elif .type == "uid" then .value | tovalue
elif .type == "uid" then {"cfuid": .value | tovalue}
elif .type == "array" then
( .entries
| map(_f)
@ -23,7 +23,7 @@ def _bplist_torepr:
)
else error("unknown type: \(.type)")
end
);
);
( .objects
| _f
);

61
format/apple/bplist/bplist.md Normal file
View File

@ -0,0 +1,61 @@
### Show full decoding
```sh
$ fq d Info.plist
```
### Timestamps
Timestamps in Apple Binary Property Lists are encoded as Cocoa Core Data
timestamps, where the raw value is the floating point number of seconds since
January 1, 2001. By default, `fq` will render the raw floating point value. In
order to get the raw value or string description, use the `todescription`
function, you can use the `tovalue` and `todescription` functions:
```sh
$ fq 'torepr.SomeTimeStamp | tovalue' Info.plist
685135328
$ fq 'torepr.SomeTimeStamp | todescription' Info.plist
"2022-09-17T19:22:08Z"
```
### Get JSON representation
`bplist` files can be converted to a JSON representation using the `torepr` filter:
```sh
$ fq torepr com.apple.UIAutomation.plist
{
"UIAutomationEnabled": true
}
```
### Decoding NSKeyedArchiver serialized objects
A common way that Swift and Objective-C libraries on macOS serialize objects
is through the NSKeyedArchiver API, which flattens objects into a list of elements
and class descriptions that are reconstructed into an object graph using CFUID
elements in the property list. `fq` includes a function, `from_ns_keyed_archiver`,
which will rebuild this object graph into a friendly representation.
If no parameters are supplied, it will assume that there is a CFUID located at
`."$top".root` that specifies the root from which decoding should occur. If this
is not present, an error will be produced, asking the user to specify a root
object in the `.$objects` list from which to decode.
The following examples show how this might be used (in this case, within the `fq` REPL):
```
# Assume $top.root is present
bplist> from_ns_keyed_archiver
# Specify optional root
bplist> from_ns_keyed_archiver(1)
```
### Authors
- David McDonald
[@dgmcdona](https://github.com/dgmcdona)
### References
- http://fileformats.archiveteam.org/wiki/Property_List/Binary
- https://medium.com/@karaiskc/understanding-apples-binary-property-list-format-281e6da00dbd
- https://opensource.apple.com/source/CF/CF-550/CFBinaryPList.c

70
format/apple/bplist/ns_keyed_archiver.jq Normal file
View File

@ -0,0 +1,70 @@
def from_ns_keyed_archiver(root):
( if _is_decode_value and format == "bplist" then _bplist_torepr end
| ."$objects" as $objects
| def _f($id; $seen_ids):
( def _r($id):
if $seen_ids | has("\($id)") then "cycle-\($id)"
else _f($id; $seen_ids | ."\($id)" = true)
end;
$objects[$id]
| type as $type
| if $type == "string" and . == "$null" then null
elif $type |
. == "number"
or . == "boolean"
or . == "null"
or . == "string" then .
elif $type == "array" then . # TODO: does this happen?
elif $type == "object" then
( ."$class" as $class
| if $class == null then # TODO: what case is this?
with_entries(
if .value | type == "object" and has("cfuid") then
.value |= _r(.cfuid)
end
)
else
( $objects[$class.cfuid]."$classname" as $cname
| if $cname == "NSDictionary"
or $cname == "NSMutableDictionary" then
# transform arrays [key_id1, key_id2,...] and [obj_id1, obj_id2,..] into {key: obj, ...}
( [."NS.keys", ."NS.objects"]
| transpose
| map({key: _r(.[0].cfuid), value: _r(.[1].cfuid)})
| from_entries
)
elif $cname == "NSArray"
or $cname == "NSMutableArray"
or $cname == "NSSet"
or $cname == "NSMutableSet" then
( ."NS.objects"
| map(_r(.cfuid))
)
elif $cname == "NSData" or $cname == "NSMutableData" then ."NS.Data"
elif $cname == "NSDate" then "NS.time"
elif $cname == "NSNull" then null
elif $cname == "NSAttributedString"
or $cname == "NSMutableAttributedString" then
_r(.NSString.cfuid)
elif $cname == "NSUUID" then ."NS.uuidbytes"
else
# replace class ID with classname, and dereference all cfuid values.
( ."$class" = $cname
| with_entries(
if .value | type == "object" and has("cfuid") then
.value |= _r(.cfuid)
end
)
)
end
)
end
)
end
);
root as $root
| _f($root; {"\($root)": true})
);
def from_ns_keyed_archiver:
from_ns_keyed_archiver(."$top"?.root?.cfuid // error("root node not found, must specify root ID"));

2378
format/apple/bplist/testdata/Info.fqtest vendored Normal file
View File

File diff suppressed because it is too large Load Diff

0
format/bplist/testdata/Info.plist → format/apple/bplist/testdata/Info.plist vendored
View File

2591
format/apple/bplist/testdata/dicttest_many_items.fqtest vendored Normal file
View File

File diff suppressed because it is too large Load Diff

0
format/bplist/testdata/dicttest_many_items.plist → format/apple/bplist/testdata/dicttest_many_items.plist vendored
View File

25
format/apple/bplist/testdata/float.fqtest vendored Normal file
View File

@ -0,0 +1,25 @@
$ fq dv float.plist
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: float.plist (bplist) 0x0-0x31.7 (50)
| | | header{}: 0x0-0x7.7 (8)
0x00|62 70 6c 69 73 74 |bplist | magic: "bplist" (valid) 0x0-0x5.7 (6)
0x00| 30 30 | 00 | version: "00" (valid) 0x6-0x7.7 (2)
| | | objects{}: 0x8-0x10.7 (9)
0x00| 23 | # | type: "real" (2) (Floating Point Number) 0x8-0x8.3 (0.4)
0x00| 23 | # | size_bits: 3 0x8.4-0x8.7 (0.4)
| | | size: 8 0x9-NA (0)
0x00| 40 09 21 f9 f0 1b 86| @.!....| value: 3.14159 0x9-0x10.7 (8)
0x10|6e |n |
| | | offset_table[0:1]: 0x11-0x11.7 (1)
0x10| 08 | . | [0]: 8 element 0x11-0x11.7 (1)
| | | trailer{}: 0x12-0x31.7 (32)
0x10| 00 00 00 00 00 | ..... | unused: 0 0x12-0x16.7 (5)
0x10| 00 | . | sort_version: 0 0x17-0x17.7 (1)
0x10| 01 | . | offset_table_offset_size: 1 (valid) 0x18-0x18.7 (1)
0x10| 01 | . | object_reference_size: 1 (valid) 0x19-0x19.7 (1)
0x10| 00 00 00 00 00 00| ......| object_count: 1 0x1a-0x21.7 (8)
0x20|00 01 |.. |
0x20| 00 00 00 00 00 00 00 00 | ........ | top_object_offset: 0 0x22-0x29.7 (8)
0x20| 00 00 00 00 00 00| ......| offset_table_start: 17 0x2a-0x31.7 (8)
0x30|00 11| |..| |
$ fq torepr float.plist
3.14159

0
format/bplist/testdata/float.plist → format/apple/bplist/testdata/float.plist vendored
View File

3430
format/apple/bplist/testdata/recentapps.fqtest vendored Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
format/apple/bplist/testdata/recentapps.sfl2 vendored Normal file
View File

Binary file not shown.

3218
format/apple/bplist/testdata/recentdocs.fqtest vendored Normal file
View File

File diff suppressed because one or more lines are too long

BIN
format/apple/bplist/testdata/recentdocs.sfl2 vendored Normal file
View File

Binary file not shown.

44
format/apple/bplist/testdata/uiautomation.fqtest vendored Normal file
View File

@ -0,0 +1,44 @@
$ fq dv uiautomation.plist
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: uiautomation.plist (bplist) 0x0-0x44.7 (69)
| | | header{}: 0x0-0x7.7 (8)
0x00|62 70 6c 69 73 74 |bplist | magic: "bplist" (valid) 0x0-0x5.7 (6)
0x00| 30 30 | 00 | version: "00" (valid) 0x6-0x7.7 (2)
| | | objects{}: 0x8-0x21.7 (26)
0x00| d1 | . | type: "dict" (13) (Dictionary) 0x8-0x8.3 (0.4)
0x00| d1 | . | size_bits: 1 0x8.4-0x8.7 (0.4)
| | | size: 1 0x9-NA (0)
| | | entries[0:1]: 0x9-0x21.7 (25)
| | | [0]{}: entry 0x9-0x21.7 (25)
0x00| 01 | . | key_index: 1 0x9-0x9.7 (1)
0x00| 02 | . | value_index: 2 0xa-0xa.7 (1)
| | | key{}: 0xb-0x20.7 (22)
0x00| 5f | _ | type: "ascii_string" (5) (ASCII encoded string) 0xb-0xb.3 (0.4)
0x00| 5f | _ | size_bits: 15 0xb.4-0xb.7 (0.4)
0x00| 10 | . | large_size_marker: 1 (valid) 0xc-0xc.3 (0.4)
0x00| 10 | . | exponent: 0 0xc.4-0xc.7 (0.4)
0x00| 13 | . | size_bigint: 19 0xd-0xd.7 (1)
| | | size: 19 0xe-NA (0)
0x00| 55 49| UI| value: "UIAutomationEnabled" 0xe-0x20.7 (19)
0x10|41 75 74 6f 6d 61 74 69 6f 6e 45 6e 61 62 6c 65|AutomationEnable|
0x20|64 |d |
| | | value{}: 0x21-0x21.7 (1)
0x20| 09 | . | type: "singleton" (0) (Singleton value (null/bool)) 0x21-0x21.3 (0.4)
0x20| 09 | . | value: true (9) 0x21.4-0x21.7 (0.4)
| | | offset_table[0:3]: 0x22-0x24.7 (3)
0x20| 08 | . | [0]: 8 element 0x22-0x22.7 (1)
0x20| 0b | . | [1]: 11 element 0x23-0x23.7 (1)
0x20| 21 | ! | [2]: 33 element 0x24-0x24.7 (1)
| | | trailer{}: 0x25-0x44.7 (32)
0x20| 00 00 00 00 00 | ..... | unused: 0 0x25-0x29.7 (5)
0x20| 00 | . | sort_version: 0 0x2a-0x2a.7 (1)
0x20| 01 | . | offset_table_offset_size: 1 (valid) 0x2b-0x2b.7 (1)
0x20| 01 | . | object_reference_size: 1 (valid) 0x2c-0x2c.7 (1)
0x20| 00 00 00| ...| object_count: 3 0x2d-0x34.7 (8)
0x30|00 00 00 00 03 |..... |
0x30| 00 00 00 00 00 00 00 00 | ........ | top_object_offset: 0 0x35-0x3c.7 (8)
0x30| 00 00 00| ...| offset_table_start: 34 0x3d-0x44.7 (8)
0x40|00 00 00 00 22| |...."| |
$ fq torepr uiautomation.plist
{
"UIAutomationEnabled": true
}

0
format/bplist/testdata/com.apple.UIAutomation.plist → format/apple/bplist/testdata/uiautomation.plist vendored
View File

450
format/apple/bplist/testdata/unicode.fqtest vendored Normal file
View File

@ -0,0 +1,450 @@
$ fq dv unicode.plist
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: unicode.plist (bplist) 0x0-0x38e.7 (911)
| | | header{}: 0x0-0x7.7 (8)
0x000|62 70 6c 69 73 74 |bplist | magic: "bplist" (valid) 0x0-0x5.7 (6)
0x000| 30 30 | 00 | version: "00" (valid) 0x6-0x7.7 (2)
| | | objects{}: 0x8-0x31a.7 (787)
0x000| df | . | type: "dict" (13) (Dictionary) 0x8-0x8.3 (0.4)
0x000| df | . | size_bits: 15 0x8.4-0x8.7 (0.4)
0x000| 10 | . | large_size_marker: 1 (valid) 0x9-0x9.3 (0.4)
0x000| 10 | . | exponent: 0 0x9.4-0x9.7 (0.4)
0x000| 15 | . | size_bigint: 21 0xa-0xa.7 (1)
| | | size: 21 0xb-NA (0)
| | | entries[0:21]: 0xb-0x31a.7 (784)
| | | [0]{}: entry 0xb-0x1dc.7 (466)
0x000| 01 | . | key_index: 1 0xb-0xb.7 (1)
0x020|16 |. | value_index: 22 0x20-0x20.7 (1)
| | | key{}: 0x35-0x41.7 (13)
0x030| 5c | \ | type: "ascii_string" (5) (ASCII encoded string) 0x35-0x35.3 (0.4)
0x030| 5c | \ | size_bits: 12 0x35.4-0x35.7 (0.4)
| | | size: 12 0x36-NA (0)
0x030| 43 46 42 75 6e 64 6c 65 4e 61| CFBundleNa| value: "CFBundleName" 0x36-0x41.7 (12)
0x040|6d 65 |me |
| | | value{}: 0x1c7-0x1dc.7 (22)
0x1c0| 5f | _ | type: "ascii_string" (5) (ASCII encoded string) 0x1c7-0x1c7.3 (0.4)
0x1c0| 5f | _ | size_bits: 15 0x1c7.4-0x1c7.7 (0.4)
0x1c0| 10 | . | large_size_marker: 1 (valid) 0x1c8-0x1c8.3 (0.4)
0x1c0| 10 | . | exponent: 0 0x1c8.4-0x1c8.7 (0.4)
0x1c0| 13 | . | size_bigint: 19 0x1c9-0x1c9.7 (1)
| | | size: 19 0x1ca-NA (0)
0x1c0| 43 6f 72 65 44 65| CoreDe| value: "CoreDeviceUtilities" 0x1ca-0x1dc.7 (19)
0x1d0|76 69 63 65 55 74 69 6c 69 74 69 65 73 |viceUtilities |
| | | [1]{}: entry 0xc-0x20f.7 (516)
0x000| 02 | . | key_index: 2 0xc-0xc.7 (1)
0x020| 17 | . | value_index: 23 0x21-0x21.7 (1)
| | | key{}: 0x42-0x5c.7 (27)
0x040| 5f | _ | type: "ascii_string" (5) (ASCII encoded string) 0x42-0x42.3 (0.4)
0x040| 5f | _ | size_bits: 15 0x42.4-0x42.7 (0.4)
0x040| 10 | . | large_size_marker: 1 (valid) 0x43-0x43.3 (0.4)
0x040| 10 | . | exponent: 0 0x43.4-0x43.7 (0.4)
0x040| 18 | . | size_bigint: 24 0x44-0x44.7 (1)
| | | size: 24 0x45-NA (0)
0x040| 4e 53 48 75 6d 61 6e 52 65 61 64| NSHumanRead| value: "NSHumanReadableCopyright" 0x45-0x5c.7 (24)
0x050|61 62 6c 65 43 6f 70 79 72 69 67 68 74 |ableCopyright |
| | | value{}: 0x1dd-0x20f.7 (51)
0x1d0| 6f | o | type: "unicode_string" (6) (Unicode string) 0x1dd-0x1dd.3 (0.4)
0x1d0| 6f | o | size_bits: 15 0x1dd.4-0x1dd.7 (0.4)
0x1d0| 10 | . | large_size_marker: 1 (valid) 0x1de-0x1de.3 (0.4)
0x1d0| 10 | . | exponent: 0 0x1de.4-0x1de.7 (0.4)
0x1d0| 30| 0| size_bigint: 48 0x1df-0x1df.7 (1)
| | | size: 48 0x1e0-NA (0)
0x1e0|00 43 00 6f 00 70 00 79 00 72 00 69 00 67 00 68|.C.o.p.y.r.i.g.h| value: "Copyright © 2021 Apple I" 0x1e0-0x20f.7 (48)
* |until 0x20f.7 (48) | |
| | | [2]{}: entry 0xd-0x244.7 (568)
0x000| 03 | . | key_index: 3 0xd-0xd.7 (1)
0x020| 18 | . | value_index: 24 0x22-0x22.7 (1)
| | | key{}: 0x5d-0x64.7 (8)
0x050| 57 | W | type: "ascii_string" (5) (ASCII encoded string) 0x5d-0x5d.3 (0.4)
0x050| 57 | W | size_bits: 7 0x5d.4-0x5d.7 (0.4)
| | | size: 7 0x5e-NA (0)
0x050| 44 54| DT| value: "DTXcode" 0x5e-0x64.7 (7)
0x060|58 63 6f 64 65 |Xcode |
| | | value{}: 0x240-0x244.7 (5)
0x240|54 |T | type: "ascii_string" (5) (ASCII encoded string) 0x240-0x240.3 (0.4)
0x240|54 |T | size_bits: 4 0x240.4-0x240.7 (0.4)
| | | size: 4 0x241-NA (0)
0x240| 31 33 32 30 | 1320 | value: "1320" 0x241-0x244.7 (4)
| | | [3]{}: entry 0xe-0x264.7 (599)
0x000| 04 | . | key_index: 4 0xe-0xe.7 (1)
0x020| 19 | . | value_index: 25 0x23-0x23.7 (1)
| | | key{}: 0x65-0x6e.7 (10)
0x060| 59 | Y | type: "ascii_string" (5) (ASCII encoded string) 0x65-0x65.3 (0.4)
0x060| 59 | Y | size_bits: 9 0x65.4-0x65.7 (0.4)
| | | size: 9 0x66-NA (0)
0x060| 44 54 53 44 4b 4e 61 6d 65 | DTSDKName | value: "DTSDKName" 0x66-0x6e.7 (9)
| | | value{}: 0x245-0x264.7 (32)
0x240| 5f | _ | type: "ascii_string" (5) (ASCII encoded string) 0x245-0x245.3 (0.4)
0x240| 5f | _ | size_bits: 15 0x245.4-0x245.7 (0.4)
0x240| 10 | . | large_size_marker: 1 (valid) 0x246-0x246.3 (0.4)
0x240| 10 | . | exponent: 0 0x246.4-0x246.7 (0.4)
0x240| 1d | . | size_bigint: 29 0x247-0x247.7 (1)
| | | size: 29 0x248-NA (0)
0x240| 61 70 70 6c 65 74 76 73| appletvs| value: "appletvsimulator15.2.internal" 0x248-0x264.7 (29)
0x250|69 6d 75 6c 61 74 6f 72 31 35 2e 32 2e 69 6e 74|imulator15.2.int|
0x260|65 72 6e 61 6c |ernal |
| | | [4]{}: entry 0xf-0x26a.7 (604)
0x000| 05| .| key_index: 5 0xf-0xf.7 (1)
0x020| 1a | . | value_index: 26 0x24-0x24.7 (1)
| | | key{}: 0x6f-0x79.7 (11)
0x060| 5a| Z| type: "ascii_string" (5) (ASCII encoded string) 0x6f-0x6f.3 (0.4)
0x060| 5a| Z| size_bits: 10 0x6f.4-0x6f.7 (0.4)
| | | size: 10 0x70-NA (0)
0x070|44 54 53 44 4b 42 75 69 6c 64 |DTSDKBuild | value: "DTSDKBuild" 0x70-0x79.7 (10)
| | | value{}: 0x265-0x26a.7 (6)
0x260| 55 | U | type: "ascii_string" (5) (ASCII encoded string) 0x265-0x265.3 (0.4)
0x260| 55 | U | size_bits: 5 0x265.4-0x265.7 (0.4)
| | | size: 5 0x266-NA (0)
0x260| 31 39 4b 32 34 | 19K24 | value: "19K24" 0x266-0x26a.7 (5)
| | | [5]{}: entry 0x10-0x272.7 (611)
0x010|06 |. | key_index: 6 0x10-0x10.7 (1)
0x020| 1b | . | value_index: 27 0x25-0x25.7 (1)
| | | key{}: 0x7a-0x95.7 (28)
0x070| 5f | _ | type: "ascii_string" (5) (ASCII encoded string) 0x7a-0x7a.3 (0.4)
0x070| 5f | _ | size_bits: 15 0x7a.4-0x7a.7 (0.4)
0x070| 10 | . | large_size_marker: 1 (valid) 0x7b-0x7b.3 (0.4)
0x070| 10 | . | exponent: 0 0x7b.4-0x7b.7 (0.4)
0x070| 19 | . | size_bigint: 25 0x7c-0x7c.7 (1)
| | | size: 25 0x7d-NA (0)
0x070| 43 46 42| CFB| value: "CFBundleDevelopmentRegion" 0x7d-0x95.7 (25)
0x080|75 6e 64 6c 65 44 65 76 65 6c 6f 70 6d 65 6e 74|undleDevelopment|
0x090|52 65 67 69 6f 6e |Region |
| | | value{}: 0x26b-0x272.7 (8)
0x260| 57 | W | type: "ascii_string" (5) (ASCII encoded string) 0x26b-0x26b.3 (0.4)
0x260| 57 | W | size_bits: 7 0x26b.4-0x26b.7 (0.4)
| | | size: 7 0x26c-NA (0)
0x260| 45 6e 67 6c| Engl| value: "English" 0x26c-0x272.7 (7)
0x270|69 73 68 |ish |
| | | [6]{}: entry 0x11-0x278.7 (616)
0x010| 07 | . | key_index: 7 0x11-0x11.7 (1)
0x020| 1c | . | value_index: 28 0x26-0x26.7 (1)
| | | key{}: 0x96-0xa7.7 (18)
0x090| 5f | _ | type: "ascii_string" (5) (ASCII encoded string) 0x96-0x96.3 (0.4)
0x090| 5f | _ | size_bits: 15 0x96.4-0x96.7 (0.4)
0x090| 10 | . | large_size_marker: 1 (valid) 0x97-0x97.3 (0.4)
0x090| 10 | . | exponent: 0 0x97.4-0x97.7 (0.4)
0x090| 0f | . | size_bigint: 15 0x98-0x98.7 (1)
| | | size: 15 0x99-NA (0)
0x090| 43 46 42 75 6e 64 6c| CFBundl| value: "CFBundleVersion" 0x99-0xa7.7 (15)
0x0a0|65 56 65 72 73 69 6f 6e |eVersion |
| | | value{}: 0x273-0x278.7 (6)
0x270| 55 | U | type: "ascii_string" (5) (ASCII encoded string) 0x273-0x273.3 (0.4)
0x270| 55 | U | size_bits: 5 0x273.4-0x273.7 (0.4)
| | | size: 5 0x274-NA (0)
0x270| 37 38 33 2e 35 | 783.5 | value: "783.5" 0x274-0x278.7 (5)
| | | [7]{}: entry 0x12-0x282.7 (625)
0x010| 08 | . | key_index: 8 0x12-0x12.7 (1)
0x020| 1d | . | value_index: 29 0x27-0x27.7 (1)
| | | key{}: 0xa8-0xbd.7 (22)
0x0a0| 5f | _ | type: "ascii_string" (5) (ASCII encoded string) 0xa8-0xa8.3 (0.4)
0x0a0| 5f | _ | size_bits: 15 0xa8.4-0xa8.7 (0.4)
0x0a0| 10 | . | large_size_marker: 1 (valid) 0xa9-0xa9.3 (0.4)
0x0a0| 10 | . | exponent: 0 0xa9.4-0xa9.7 (0.4)
0x0a0| 13 | . | size_bigint: 19 0xaa-0xaa.7 (1)
| | | size: 19 0xab-NA (0)
0x0a0| 42 75 69 6c 64| Build| value: "BuildMachineOSBuild" 0xab-0xbd.7 (19)
0x0b0|4d 61 63 68 69 6e 65 4f 53 42 75 69 6c 64 |MachineOSBuild |
| | | value{}: 0x279-0x282.7 (10)
0x270| 59 | Y | type: "ascii_string" (5) (ASCII encoded string) 0x279-0x279.3 (0.4)
0x270| 59 | Y | size_bits: 9 0x279.4-0x279.7 (0.4)
| | | size: 9 0x27a-NA (0)
0x270| 32 30 41 32 34 31| 20A241| value: "20A241108" 0x27a-0x282.7 (9)
0x280|31 30 38 |108 |
| | | [8]{}: entry 0x13-0x295.7 (643)
0x010| 09 | . | key_index: 9 0x13-0x13.7 (1)
0x020| 1e | . | value_index: 30 0x28-0x28.7 (1)
| | | key{}: 0xbe-0xcc.7 (15)
0x0b0| 5e | ^ | type: "ascii_string" (5) (ASCII encoded string) 0xbe-0xbe.3 (0.4)
0x0b0| 5e | ^ | size_bits: 14 0xbe.4-0xbe.7 (0.4)
| | | size: 14 0xbf-NA (0)
0x0b0| 44| D| value: "DTPlatformName" 0xbf-0xcc.7 (14)
0x0c0|54 50 6c 61 74 66 6f 72 6d 4e 61 6d 65 |TPlatformName |
| | | value{}: 0x283-0x295.7 (19)
0x280| 5f | _ | type: "ascii_string" (5) (ASCII encoded string) 0x283-0x283.3 (0.4)
0x280| 5f | _ | size_bits: 15 0x283.4-0x283.7 (0.4)
0x280| 10 | . | large_size_marker: 1 (valid) 0x284-0x284.3 (0.4)
0x280| 10 | . | exponent: 0 0x284.4-0x284.7 (0.4)
0x280| 10 | . | size_bigint: 16 0x285-0x285.7 (1)
| | | size: 16 0x286-NA (0)
0x280| 61 70 70 6c 65 74 76 73 69 6d| appletvsim| value: "appletvsimulator" 0x286-0x295.7 (16)
0x290|75 6c 61 74 6f 72 |ulator |
| | | [9]{}: entry 0x14-0x29a.7 (647)
0x010| 0a | . | key_index: 10 0x14-0x14.7 (1)
0x020| 1f | . | value_index: 31 0x29-0x29.7 (1)
| | | key{}: 0xcd-0xe2.7 (22)
0x0c0| 5f | _ | type: "ascii_string" (5) (ASCII encoded string) 0xcd-0xcd.3 (0.4)
0x0c0| 5f | _ | size_bits: 15 0xcd.4-0xcd.7 (0.4)
0x0c0| 10 | . | large_size_marker: 1 (valid) 0xce-0xce.3 (0.4)
0x0c0| 10 | . | exponent: 0 0xce.4-0xce.7 (0.4)
0x0c0| 13| .| size_bigint: 19 0xcf-0xcf.7 (1)
| | | size: 19 0xd0-NA (0)
0x0d0|43 46 42 75 6e 64 6c 65 50 61 63 6b 61 67 65 54|CFBundlePackageT| value: "CFBundlePackageType" 0xd0-0xe2.7 (19)
0x0e0|79 70 65 |ype |
| | | value{}: 0x296-0x29a.7 (5)
0x290| 54 | T | type: "ascii_string" (5) (ASCII encoded string) 0x296-0x296.3 (0.4)
0x290| 54 | T | size_bits: 4 0x296.4-0x296.7 (0.4)
| | | size: 4 0x297-NA (0)
0x290| 46 4d 57 4b | FMWK | value: "FMWK" 0x297-0x29a.7 (4)
| | | [10]{}: entry 0x15-0x278.7 (612)
0x010| 0b | . | key_index: 11 0x15-0x15.7 (1)
0x020| 1c | . | value_index: 28 0x2a-0x2a.7 (1)
| | | key{}: 0xe3-0xff.7 (29)
0x0e0| 5f | _ | type: "ascii_string" (5) (ASCII encoded string) 0xe3-0xe3.3 (0.4)
0x0e0| 5f | _ | size_bits: 15 0xe3.4-0xe3.7 (0.4)
0x0e0| 10 | . | large_size_marker: 1 (valid) 0xe4-0xe4.3 (0.4)
0x0e0| 10 | . | exponent: 0 0xe4.4-0xe4.7 (0.4)
0x0e0| 1a | . | size_bigint: 26 0xe5-0xe5.7 (1)
| | | size: 26 0xe6-NA (0)
0x0e0| 43 46 42 75 6e 64 6c 65 53 68| CFBundleSh| value: "CFBundleShortVersionString" 0xe6-0xff.7 (26)
0x0f0|6f 72 74 56 65 72 73 69 6f 6e 53 74 72 69 6e 67|ortVersionString|
| | | value{}: 0x273-0x278.7 (6)
0x270| 55 | U | type: "ascii_string" (5) (ASCII encoded string) 0x273-0x273.3 (0.4)
0x270| 55 | U | size_bits: 5 0x273.4-0x273.7 (0.4)
| | | size: 5 0x274-NA (0)
0x270| 37 38 33 2e 35 | 783.5 | value: "783.5" 0x274-0x278.7 (5)
| | | [11]{}: entry 0x16-0x2af.7 (666)
0x010| 0c | . | key_index: 12 0x16-0x16.7 (1)
0x020| 20 | | value_index: 32 0x2b-0x2b.7 (1)
| | | key{}: 0x100-0x11c.7 (29)
0x100|5f |_ | type: "ascii_string" (5) (ASCII encoded string) 0x100-0x100.3 (0.4)
0x100|5f |_ | size_bits: 15 0x100.4-0x100.7 (0.4)
0x100| 10 | . | large_size_marker: 1 (valid) 0x101-0x101.3 (0.4)
0x100| 10 | . | exponent: 0 0x101.4-0x101.7 (0.4)
0x100| 1a | . | size_bigint: 26 0x102-0x102.7 (1)
| | | size: 26 0x103-NA (0)
0x100| 43 46 42 75 6e 64 6c 65 53 75 70 70 6f| CFBundleSuppo| value: "CFBundleSupportedPlatforms" 0x103-0x11c.7 (26)
0x110|72 74 65 64 50 6c 61 74 66 6f 72 6d 73 |rtedPlatforms |
| | | value{}: 0x29b-0x2af.7 (21)
0x290| a1 | . | type: "array" (10) (Array) 0x29b-0x29b.3 (0.4)
0x290| a1 | . | size_bits: 1 0x29b.4-0x29b.7 (0.4)
| | | size: 1 0x29c-NA (0)
| | | entries[0:1]: 0x29c-0x2af.7 (20)
| | | [0]{}: entry 0x29c-0x2af.7 (20)
0x290| 21 | ! | object_index: 33 0x29c-0x29c.7 (1)
0x290| 5f | _ | type: "ascii_string" (5) (ASCII encoded string) 0x29d-0x29d.3 (0.4)
0x290| 5f | _ | size_bits: 15 0x29d.4-0x29d.7 (0.4)
0x290| 10 | . | large_size_marker: 1 (valid) 0x29e-0x29e.3 (0.4)
0x290| 10 | . | exponent: 0 0x29e.4-0x29e.7 (0.4)
0x290| 10| .| size_bigint: 16 0x29f-0x29f.7 (1)
| | | size: 16 0x2a0-NA (0)
0x2a0|41 70 70 6c 65 54 56 53 69 6d 75 6c 61 74 6f 72|AppleTVSimulator| value: "AppleTVSimulator" 0x2a0-0x2af.7 (16)
| | | [12]{}: entry 0x17-0x2b3.7 (669)
0x010| 0d | . | key_index: 13 0x17-0x17.7 (1)
0x020| 22 | " | value_index: 34 0x2c-0x2c.7 (1)
| | | key{}: 0x11d-0x13c.7 (32)
0x110| 5f | _ | type: "ascii_string" (5) (ASCII encoded string) 0x11d-0x11d.3 (0.4)
0x110| 5f | _ | size_bits: 15 0x11d.4-0x11d.7 (0.4)
0x110| 10 | . | large_size_marker: 1 (valid) 0x11e-0x11e.3 (0.4)
0x110| 10 | . | exponent: 0 0x11e.4-0x11e.7 (0.4)
0x110| 1d| .| size_bigint: 29 0x11f-0x11f.7 (1)
| | | size: 29 0x120-NA (0)
0x120|43 46 42 75 6e 64 6c 65 49 6e 66 6f 44 69 63 74|CFBundleInfoDict| value: "CFBundleInfoDictionaryVersion" 0x120-0x13c.7 (29)
0x130|69 6f 6e 61 72 79 56 65 72 73 69 6f 6e |ionaryVersion |
| | | value{}: 0x2b0-0x2b3.7 (4)
0x2b0|53 |S | type: "ascii_string" (5) (ASCII encoded string) 0x2b0-0x2b0.3 (0.4)
0x2b0|53 |S | size_bits: 3 0x2b0.4-0x2b0.7 (0.4)
| | | size: 3 0x2b1-NA (0)
0x2b0| 36 2e 30 | 6.0 | value: "6.0" 0x2b1-0x2b3.7 (3)
| | | [13]{}: entry 0x18-0x1dc.7 (453)
0x010| 0e | . | key_index: 14 0x18-0x18.7 (1)
0x020| 16 | . | value_index: 22 0x2d-0x2d.7 (1)
| | | key{}: 0x13d-0x151.7 (21)
0x130| 5f | _ | type: "ascii_string" (5) (ASCII encoded string) 0x13d-0x13d.3 (0.4)
0x130| 5f | _ | size_bits: 15 0x13d.4-0x13d.7 (0.4)
0x130| 10 | . | large_size_marker: 1 (valid) 0x13e-0x13e.3 (0.4)
0x130| 10 | . | exponent: 0 0x13e.4-0x13e.7 (0.4)
0x130| 12| .| size_bigint: 18 0x13f-0x13f.7 (1)
| | | size: 18 0x140-NA (0)
0x140|43 46 42 75 6e 64 6c 65 45 78 65 63 75 74 61 62|CFBundleExecutab| value: "CFBundleExecutable" 0x140-0x151.7 (18)
0x150|6c 65 |le |
| | | value{}: 0x1c7-0x1dc.7 (22)
0x1c0| 5f | _ | type: "ascii_string" (5) (ASCII encoded string) 0x1c7-0x1c7.3 (0.4)
0x1c0| 5f | _ | size_bits: 15 0x1c7.4-0x1c7.7 (0.4)
0x1c0| 10 | . | large_size_marker: 1 (valid) 0x1c8-0x1c8.3 (0.4)
0x1c0| 10 | . | exponent: 0 0x1c8.4-0x1c8.7 (0.4)
0x1c0| 13 | . | size_bigint: 19 0x1c9-0x1c9.7 (1)
| | | size: 19 0x1ca-NA (0)
0x1c0| 43 6f 72 65 44 65| CoreDe| value: "CoreDeviceUtilities" 0x1ca-0x1dc.7 (19)
0x1d0|76 69 63 65 55 74 69 6c 69 74 69 65 73 |viceUtilities |
| | | [14]{}: entry 0x19-0x2d8.7 (704)
0x010| 0f | . | key_index: 15 0x19-0x19.7 (1)
0x020| 23 | # | value_index: 35 0x2e-0x2e.7 (1)
| | | key{}: 0x152-0x15c.7 (11)
0x150| 5a | Z | type: "ascii_string" (5) (ASCII encoded string) 0x152-0x152.3 (0.4)
0x150| 5a | Z | size_bits: 10 0x152.4-0x152.7 (0.4)
| | | size: 10 0x153-NA (0)
0x150| 44 54 43 6f 6d 70 69 6c 65 72 | DTCompiler | value: "DTCompiler" 0x153-0x15c.7 (10)
| | | value{}: 0x2b4-0x2d8.7 (37)
0x2b0| 5f | _ | type: "ascii_string" (5) (ASCII encoded string) 0x2b4-0x2b4.3 (0.4)
0x2b0| 5f | _ | size_bits: 15 0x2b4.4-0x2b4.7 (0.4)
0x2b0| 10 | . | large_size_marker: 1 (valid) 0x2b5-0x2b5.3 (0.4)
0x2b0| 10 | . | exponent: 0 0x2b5.4-0x2b5.7 (0.4)
0x2b0| 22 | " | size_bigint: 34 0x2b6-0x2b6.7 (1)
| | | size: 34 0x2b7-NA (0)
0x2b0| 63 6f 6d 2e 61 70 70 6c 65| com.apple| value: "com.apple.compilers.llvm.clang.1_0" 0x2b7-0x2d8.7 (34)
0x2c0|2e 63 6f 6d 70 69 6c 65 72 73 2e 6c 6c 76 6d 2e|.compilers.llvm.|
0x2d0|63 6c 61 6e 67 2e 31 5f 30 |clang.1_0 |
| | | [15]{}: entry 0x1a-0x2dd.7 (708)
0x010| 10 | . | key_index: 16 0x1a-0x1a.7 (1)
0x020| 24| $| value_index: 36 0x2f-0x2f.7 (1)
| | | key{}: 0x15d-0x16f.7 (19)
0x150| 5f | _ | type: "ascii_string" (5) (ASCII encoded string) 0x15d-0x15d.3 (0.4)
0x150| 5f | _ | size_bits: 15 0x15d.4-0x15d.7 (0.4)
0x150| 10 | . | large_size_marker: 1 (valid) 0x15e-0x15e.3 (0.4)
0x150| 10 | . | exponent: 0 0x15e.4-0x15e.7 (0.4)
0x150| 10| .| size_bigint: 16 0x15f-0x15f.7 (1)
| | | size: 16 0x160-NA (0)
0x160|4d 69 6e 69 6d 75 6d 4f 53 56 65 72 73 69 6f 6e|MinimumOSVersion| value: "MinimumOSVersion" 0x160-0x16f.7 (16)
| | | value{}: 0x2d9-0x2dd.7 (5)
0x2d0| 54 | T | type: "ascii_string" (5) (ASCII encoded string) 0x2d9-0x2d9.3 (0.4)
0x2d0| 54 | T | size_bits: 4 0x2d9.4-0x2d9.7 (0.4)
| | | size: 4 0x2da-NA (0)
0x2d0| 31 30 2e 32 | 10.2 | value: "10.2" 0x2da-0x2dd.7 (4)
| | | [16]{}: entry 0x1b-0x30b.7 (753)
0x010| 11 | . | key_index: 17 0x1b-0x1b.7 (1)
0x030|25 |% | value_index: 37 0x30-0x30.7 (1)
| | | key{}: 0x170-0x184.7 (21)
0x170|5f |_ | type: "ascii_string" (5) (ASCII encoded string) 0x170-0x170.3 (0.4)
0x170|5f |_ | size_bits: 15 0x170.4-0x170.7 (0.4)
0x170| 10 | . | large_size_marker: 1 (valid) 0x171-0x171.3 (0.4)
0x170| 10 | . | exponent: 0 0x171.4-0x171.7 (0.4)
0x170| 12 | . | size_bigint: 18 0x172-0x172.7 (1)
| | | size: 18 0x173-NA (0)
0x170| 43 46 42 75 6e 64 6c 65 49 64 65 6e 74| CFBundleIdent| value: "CFBundleIdentifier" 0x173-0x184.7 (18)
0x180|69 66 69 65 72 |ifier |
| | | value{}: 0x2de-0x30b.7 (46)
0x2d0| 5f | _ | type: "ascii_string" (5) (ASCII encoded string) 0x2de-0x2de.3 (0.4)
0x2d0| 5f | _ | size_bits: 15 0x2de.4-0x2de.7 (0.4)
0x2d0| 10| .| large_size_marker: 1 (valid) 0x2df-0x2df.3 (0.4)
0x2d0| 10| .| exponent: 0 0x2df.4-0x2df.7 (0.4)
0x2e0|2b |+ | size_bigint: 43 0x2e0-0x2e0.7 (1)
| | | size: 43 0x2e1-NA (0)
0x2e0| 63 6f 6d 2e 61 70 70 6c 65 2e 43 6f 72 65 53| com.apple.CoreS| value: "com.apple.CoreSimulator.CoreDeviceUtilities" 0x2e1-0x30b.7 (43)
0x2f0|69 6d 75 6c 61 74 6f 72 2e 43 6f 72 65 44 65 76|imulator.CoreDev|
0x300|69 63 65 55 74 69 6c 69 74 69 65 73 |iceUtilities |
| | | [17]{}: entry 0x1c-0x30f.7 (756)
0x010| 12 | . | key_index: 18 0x1c-0x1c.7 (1)
0x030| 26 | & | value_index: 38 0x31-0x31.7 (1)
| | | key{}: 0x185-0x193.7 (15)
0x180| 5e | ^ | type: "ascii_string" (5) (ASCII encoded string) 0x185-0x185.3 (0.4)
0x180| 5e | ^ | size_bits: 14 0x185.4-0x185.7 (0.4)
| | | size: 14 0x186-NA (0)
0x180| 55 49 44 65 76 69 63 65 46 61| UIDeviceFa| value: "UIDeviceFamily" 0x186-0x193.7 (14)
0x190|6d 69 6c 79 |mily |
| | | value{}: 0x30c-0x30f.7 (4)
0x300| a1 | . | type: "array" (10) (Array) 0x30c-0x30c.3 (0.4)
0x300| a1 | . | size_bits: 1 0x30c.4-0x30c.7 (0.4)
| | | size: 1 0x30d-NA (0)
| | | entries[0:1]: 0x30d-0x30f.7 (3)
| | | [0]{}: entry 0x30d-0x30f.7 (3)
0x300| 27 | ' | object_index: 39 0x30d-0x30d.7 (1)
0x300| 10 | . | type: "int" (1) (Integer) 0x30e-0x30e.3 (0.4)
0x300| 10 | . | size: 1 0x30e.4-0x30e.7 (0.4)
0x300| 03| .| value: 3 0x30f-0x30f.7 (1)
| | | [18]{}: entry 0x1d-0x314.7 (760)
0x010| 13 | . | key_index: 19 0x1d-0x1d.7 (1)
0x030| 28 | ( | value_index: 40 0x32-0x32.7 (1)
| | | key{}: 0x194-0x1a7.7 (20)
0x190| 5f | _ | type: "ascii_string" (5) (ASCII encoded string) 0x194-0x194.3 (0.4)
0x190| 5f | _ | size_bits: 15 0x194.4-0x194.7 (0.4)
0x190| 10 | . | large_size_marker: 1 (valid) 0x195-0x195.3 (0.4)
0x190| 10 | . | exponent: 0 0x195.4-0x195.7 (0.4)
0x190| 11 | . | size_bigint: 17 0x196-0x196.7 (1)
| | | size: 17 0x197-NA (0)
0x190| 44 54 50 6c 61 74 66 6f 72| DTPlatfor| value: "DTPlatformVersion" 0x197-0x1a7.7 (17)
0x1a0|6d 56 65 72 73 69 6f 6e |mVersion |
| | | value{}: 0x310-0x314.7 (5)
0x310|54 |T | type: "ascii_string" (5) (ASCII encoded string) 0x310-0x310.3 (0.4)
0x310|54 |T | size_bits: 4 0x310.4-0x310.7 (0.4)
| | | size: 4 0x311-NA (0)
0x310| 31 35 2e 32 | 15.2 | value: "15.2" 0x311-0x314.7 (4)
| | | [19]{}: entry 0x1e-0x31a.7 (765)
0x010| 14 | . | key_index: 20 0x1e-0x1e.7 (1)
0x030| 29 | ) | value_index: 41 0x33-0x33.7 (1)
| | | key{}: 0x1a8-0x1b4.7 (13)
0x1a0| 5c | \ | type: "ascii_string" (5) (ASCII encoded string) 0x1a8-0x1a8.3 (0.4)
0x1a0| 5c | \ | size_bits: 12 0x1a8.4-0x1a8.7 (0.4)
| | | size: 12 0x1a9-NA (0)
0x1a0| 44 54 58 63 6f 64 65| DTXcode| value: "DTXcodeBuild" 0x1a9-0x1b4.7 (12)
0x1b0|42 75 69 6c 64 |Build |
| | | value{}: 0x315-0x31a.7 (6)
0x310| 55 | U | type: "ascii_string" (5) (ASCII encoded string) 0x315-0x315.3 (0.4)
0x310| 55 | U | size_bits: 5 0x315.4-0x315.7 (0.4)
| | | size: 5 0x316-NA (0)
0x310| 31 33 43 37 31 | 13C71 | value: "13C71" 0x316-0x31a.7 (5)
| | | [20]{}: entry 0x1f-0x26a.7 (588)
0x010| 15| .| key_index: 21 0x1f-0x1f.7 (1)
0x030| 1a | . | value_index: 26 0x34-0x34.7 (1)
| | | key{}: 0x1b5-0x1c6.7 (18)
0x1b0| 5f | _ | type: "ascii_string" (5) (ASCII encoded string) 0x1b5-0x1b5.3 (0.4)
0x1b0| 5f | _ | size_bits: 15 0x1b5.4-0x1b5.7 (0.4)
0x1b0| 10 | . | large_size_marker: 1 (valid) 0x1b6-0x1b6.3 (0.4)
0x1b0| 10 | . | exponent: 0 0x1b6.4-0x1b6.7 (0.4)
0x1b0| 0f | . | size_bigint: 15 0x1b7-0x1b7.7 (1)
| | | size: 15 0x1b8-NA (0)
0x1b0| 44 54 50 6c 61 74 66 6f| DTPlatfo| value: "DTPlatformBuild" 0x1b8-0x1c6.7 (15)
0x1c0|72 6d 42 75 69 6c 64 |rmBuild |
| | | value{}: 0x265-0x26a.7 (6)
0x260| 55 | U | type: "ascii_string" (5) (ASCII encoded string) 0x265-0x265.3 (0.4)
0x260| 55 | U | size_bits: 5 0x265.4-0x265.7 (0.4)
| | | size: 5 0x266-NA (0)
0x260| 31 39 4b 32 34 | 19K24 | value: "19K24" 0x266-0x26a.7 (5)
0x210|00 6e 00 63 00 2e 00 20 00 41 00 6c 00 6c 00 20|.n.c... .A.l.l. | gap0: raw bits 0x210-0x23f.7 (48)
* |until 0x23f.7 (48) | |
| | | offset_table[0:42]: 0x31b-0x36e.7 (84)
0x310| 00 08 | .. | [0]: 8 element 0x31b-0x31c.7 (2)
0x310| 00 35 | .5 | [1]: 53 element 0x31d-0x31e.7 (2)
0x310| 00| .| [2]: 66 element 0x31f-0x320.7 (2)
0x320|42 |B |
0x320| 00 5d | .] | [3]: 93 element 0x321-0x322.7 (2)
0x320| 00 65 | .e | [4]: 101 element 0x323-0x324.7 (2)
0x320| 00 6f | .o | [5]: 111 element 0x325-0x326.7 (2)
0x320| 00 7a | .z | [6]: 122 element 0x327-0x328.7 (2)
0x320| 00 96 | .. | [7]: 150 element 0x329-0x32a.7 (2)
0x320| 00 a8 | .. | [8]: 168 element 0x32b-0x32c.7 (2)
0x320| 00 be | .. | [9]: 190 element 0x32d-0x32e.7 (2)
0x320| 00| .| [10]: 205 element 0x32f-0x330.7 (2)
0x330|cd |. |
0x330| 00 e3 | .. | [11]: 227 element 0x331-0x332.7 (2)
0x330| 01 00 | .. | [12]: 256 element 0x333-0x334.7 (2)
0x330| 01 1d | .. | [13]: 285 element 0x335-0x336.7 (2)
0x330| 01 3d | .= | [14]: 317 element 0x337-0x338.7 (2)
0x330| 01 52 | .R | [15]: 338 element 0x339-0x33a.7 (2)
0x330| 01 5d | .] | [16]: 349 element 0x33b-0x33c.7 (2)
0x330| 01 70 | .p | [17]: 368 element 0x33d-0x33e.7 (2)
0x330| 01| .| [18]: 389 element 0x33f-0x340.7 (2)
0x340|85 |. |
0x340| 01 94 | .. | [19]: 404 element 0x341-0x342.7 (2)
0x340| 01 a8 | .. | [20]: 424 element 0x343-0x344.7 (2)
0x340| 01 b5 | .. | [21]: 437 element 0x345-0x346.7 (2)
0x340| 01 c7 | .. | [22]: 455 element 0x347-0x348.7 (2)
0x340| 01 dd | .. | [23]: 477 element 0x349-0x34a.7 (2)
0x340| 02 40 | .@ | [24]: 576 element 0x34b-0x34c.7 (2)
0x340| 02 45 | .E | [25]: 581 element 0x34d-0x34e.7 (2)
0x340| 02| .| [26]: 613 element 0x34f-0x350.7 (2)
0x350|65 |e |
0x350| 02 6b | .k | [27]: 619 element 0x351-0x352.7 (2)
0x350| 02 73 | .s | [28]: 627 element 0x353-0x354.7 (2)
0x350| 02 79 | .y | [29]: 633 element 0x355-0x356.7 (2)
0x350| 02 83 | .. | [30]: 643 element 0x357-0x358.7 (2)
0x350| 02 96 | .. | [31]: 662 element 0x359-0x35a.7 (2)
0x350| 02 9b | .. | [32]: 667 element 0x35b-0x35c.7 (2)
0x350| 02 9d | .. | [33]: 669 element 0x35d-0x35e.7 (2)
0x350| 02| .| [34]: 688 element 0x35f-0x360.7 (2)
0x360|b0 |. |
0x360| 02 b4 | .. | [35]: 692 element 0x361-0x362.7 (2)
0x360| 02 d9 | .. | [36]: 729 element 0x363-0x364.7 (2)
0x360| 02 de | .. | [37]: 734 element 0x365-0x366.7 (2)
0x360| 03 0c | .. | [38]: 780 element 0x367-0x368.7 (2)
0x360| 03 0e | .. | [39]: 782 element 0x369-0x36a.7 (2)
0x360| 03 10 | .. | [40]: 784 element 0x36b-0x36c.7 (2)
0x360| 03 15 | .. | [41]: 789 element 0x36d-0x36e.7 (2)
| | | trailer{}: 0x36f-0x38e.7 (32)
0x360| 00| .| unused: 0 0x36f-0x373.7 (5)
0x370|00 00 00 00 |.... |
0x370| 00 | . | sort_version: 0 0x374-0x374.7 (1)
0x370| 02 | . | offset_table_offset_size: 2 (valid) 0x375-0x375.7 (1)
0x370| 01 | . | object_reference_size: 1 (valid) 0x376-0x376.7 (1)
0x370| 00 00 00 00 00 00 00 2a | .......* | object_count: 42 0x377-0x37e.7 (8)
0x370| 00| .| top_object_offset: 0 0x37f-0x386.7 (8)
0x380|00 00 00 00 00 00 00 |....... |
0x380| 00 00 00 00 00 00 03 1b| | ........|| offset_table_start: 795 0x387-0x38e.7 (8)

BIN
format/apple/bplist/testdata/unicode.plist vendored Normal file
View File

Binary file not shown.

33
format/apple/loop_detector.go Normal file
View File

@ -0,0 +1,33 @@
package apple
import (
"golang.org/x/exp/constraints"
)
// PosLoopDetector is used for detecting loops when writing decoders, and can
// short-circuit infinite recursion that can cause stack overflows.
type PosLoopDetector[T constraints.Integer] []T
// Push adds the current offset to the stack and executes the supplied
// detection function
func (pld *PosLoopDetector[T]) Push(offset T, detect func()) {
for _, o := range *pld {
if offset == o {
detect()
}
}
*pld = append(*pld, offset)
}
// Pop removes the most recently added offset from the stack.
func (pld *PosLoopDetector[T]) Pop() {
*pld = (*pld)[:len(*pld)-1]
}
// PushAndPop adds the current offset to the stack, executes the supplied
// detection function, and returns the Pop method. A good usage of this is to
// pair this method call with a defer statement.
func (pld *PosLoopDetector[T]) PushAndPop(offset T, detect func()) func() {
pld.Push(offset, detect)
return pld.Pop
}

113
format/macho/macho.go → format/apple/macho/macho.go
View File

@ -18,12 +18,13 @@ import (
var machoFS embed.FS
func init() {
interp.RegisterFormat(decode.Format{
Name: format.MACHO,
Description: "Mach-O macOS executable",
Groups: []string{format.PROBE},
DecodeFn: machoDecode,
})
interp.RegisterFormat(
format.MachO,
&decode.Format{
Description: "Mach-O macOS executable",
Groups: []*decode.Group{format.Probe},
DecodeFn: machoDecode,
})
interp.RegisterFS(machoFS)
}
@ -40,12 +41,11 @@ func strIndexNull(idx int, s string) string {
type strTable string
func (m strTable) MapScalar(s scalar.S) (scalar.S, error) {
s.Sym = strIndexNull(int(s.ActualU()), string(m))
func (m strTable) MapUint(s scalar.Uint) (scalar.Uint, error) {
s.Sym = strIndexNull(int(s.Actual), string(m))
return s, nil
}
//nolint:revive
const (
MH_MAGIC = 0xfeed_face
MH_CIGAM = 0xcefa_edfe
@ -53,14 +53,14 @@ const (
MH_CIGAM_64 = 0xcffa_edfe
)
var magicSymMapper = scalar.UToScalar{
MH_MAGIC: scalar.S{Sym: "32le", Description: "32-bit little endian"},
MH_CIGAM: scalar.S{Sym: "32be", Description: "32-bit big endian"},
MH_MAGIC_64: scalar.S{Sym: "64le", Description: "64-bit little endian"},
MH_CIGAM_64: scalar.S{Sym: "64be", Description: "64-bit big endian"},
var magicSymMapper = scalar.UintMap{
MH_MAGIC: scalar.Uint{Sym: "32le", Description: "32-bit little endian"},
MH_CIGAM: scalar.Uint{Sym: "32be", Description: "32-bit big endian"},
MH_MAGIC_64: scalar.Uint{Sym: "64le", Description: "64-bit little endian"},
MH_CIGAM_64: scalar.Uint{Sym: "64be", Description: "64-bit big endian"},
}
var cpuTypes = scalar.UToSymStr{
var cpuTypes = scalar.UintMapSymStr{
0xff_ff_ff_ff: "any",
1: "vax",
2: "romp",
@ -89,7 +89,7 @@ func intelSubTypeHelper(f, m uint64) uint64 {
return f + (m << 4)
}
var cpuSubTypes = map[uint64]scalar.UToSymStr{
var cpuSubTypes = map[uint64]scalar.UintMapSymStr{
0xff_ff_ff_ff: {
0xff_ff_ff_ff: "multiple",
},
@ -214,7 +214,7 @@ var cpuSubTypes = map[uint64]scalar.UToSymStr{
},
}
var fileTypes = scalar.UToSymStr{
var fileTypes = scalar.UintMapSymStr{
0x1: "object",
0x2: "execute",
0x3: "fvmlib",
@ -228,7 +228,6 @@ var fileTypes = scalar.UToSymStr{
0xb: "kext_bundle",
}
//nolint:revive
const (
LC_REQ_DYLD = 0x80000000
LC_SEGMENT = 0x1
@ -284,7 +283,7 @@ const (
LC_BUILD_VERSION = 0x32
)
var loadCommands = scalar.UToSymStr{
var loadCommands = scalar.UintMapSymStr{
LC_REQ_DYLD: "req_dyld",
LC_SEGMENT: "segment",
LC_SYMTAB: "symtab",
@ -339,7 +338,7 @@ var loadCommands = scalar.UToSymStr{
LC_BUILD_VERSION: "build_version",
}
var sectionTypes = scalar.UToSymStr{
var sectionTypes = scalar.UintMapSymStr{
0x0: "regular",
0x1: "zerofill",
0x2: "cstring_literals",
@ -364,7 +363,7 @@ var sectionTypes = scalar.UToSymStr{
0x15: "thread_local_init_function_pointers",
}
func machoDecode(d *decode.D, _ any) any {
func machoDecode(d *decode.D) any {
var archBits int
var cpuType uint64
var ncmds uint64
@ -390,11 +389,11 @@ func machoDecode(d *decode.D, _ any) any {
d.SeekRel(-4 * 8)
d.FieldStruct("header", func(d *decode.D) {
d.FieldValueS("arch_bits", int64(archBits))
d.FieldU32("magic", magicSymMapper, scalar.ActualHex)
d.FieldValueU("bits", uint64(archBits))
cpuType = d.FieldU32("cputype", cpuTypes, scalar.ActualHex)
d.FieldU32("cpusubtype", cpuSubTypes[cpuType], scalar.ActualHex)
d.FieldValueSint("arch_bits", int64(archBits))
d.FieldU32("magic", magicSymMapper, scalar.UintHex)
d.FieldValueUint("bits", uint64(archBits))
cpuType = d.FieldU32("cputype", cpuTypes, scalar.UintHex)
d.FieldU32("cpusubtype", cpuSubTypes[cpuType], scalar.UintHex)
d.FieldU32("filetype", fileTypes)
ncmds = d.FieldU32("ncdms")
d.FieldU32("sizeofncdms")
@ -409,7 +408,7 @@ func machoDecode(d *decode.D, _ any) any {
d.FieldStruct("load_command", func(d *decode.D) {
d.SeekAbs(loadCommandsNext)
cmd := d.FieldU32("cmd", loadCommands, scalar.ActualHex)
cmd := d.FieldU32("cmd", loadCommands, scalar.UintHex)
cmdSize := d.FieldU32("cmdsize")
if cmdSize == 0 {
d.Fatalf("cmdSize is zero")
@ -431,17 +430,17 @@ func machoDecode(d *decode.D, _ any) any {
var nsects uint64
d.FieldStruct("segment_command", func(d *decode.D) {
d.FieldValueS("arch_bits", int64(archBits))
d.FieldValueSint("arch_bits", int64(archBits))
d.FieldUTF8NullFixedLen("segname", 16) // OPCODE_DECODER segname==__TEXT
if archBits == 32 {
vmaddr = int64(d.FieldU32("vmaddr", scalar.ActualHex))
vmaddr = int64(d.FieldU32("vmaddr", scalar.UintHex))
d.FieldU32("vmsize")
fileoff = int64(d.FieldU32("fileoff", scalar.ActualHex))
fileoff = int64(d.FieldU32("fileoff", scalar.UintHex))
d.FieldU32("tfilesize")
} else {
vmaddr = int64(d.FieldU64("vmaddr", scalar.ActualHex))
vmaddr = int64(d.FieldU64("vmaddr", scalar.UintHex))
d.FieldU64("vmsize")
fileoff = int64(d.FieldU64("fileoff", scalar.ActualHex))
fileoff = int64(d.FieldU64("fileoff", scalar.UintHex))
d.FieldU64("tfilesize")
}
d.FieldS32("initprot")
@ -457,13 +456,13 @@ func machoDecode(d *decode.D, _ any) any {
d.FieldUTF8NullFixedLen("segname", 16)
var size uint64
if archBits == 32 {
d.FieldU32("address", scalar.ActualHex)
d.FieldU32("address", scalar.UintHex)
size = d.FieldU32("size")
} else {
d.FieldU64("address", scalar.ActualHex)
d.FieldU64("address", scalar.UintHex)
size = d.FieldU64("size")
}
offset := d.FieldU32("offset", scalar.ActualHex)
offset := d.FieldU32("offset", scalar.UintHex)
d.FieldU32("align")
d.FieldU32("reloff")
d.FieldU32("nreloc")
@ -506,11 +505,11 @@ func machoDecode(d *decode.D, _ any) any {
const flagUTF16 = 0x07d0
d.FieldU("isa_vmaddr", archBits)
flag := d.FieldU("flags", archBits, scalar.ActualHex, scalar.UToSymStr{
flag := d.FieldU("flags", archBits, scalar.UintHex, scalar.UintMapSymStr{
flagUTF8: "utf8",
flagUTF16: "utf16",
})
dataPtr := int64(d.FieldU("data_ptr", archBits, scalar.ActualHex))
dataPtr := int64(d.FieldU("data_ptr", archBits, scalar.UintHex))
length := int64(d.FieldU("length", archBits))
offset := ((dataPtr - vmaddr) + fileoff) * 8
@ -533,7 +532,7 @@ func machoDecode(d *decode.D, _ any) any {
}
})
case LC_TWOLEVEL_HINTS:
d.FieldU32("offset", scalar.ActualHex)
d.FieldU32("offset", scalar.UintHex)
d.FieldU32("nhints")
case LC_LOAD_DYLIB,
LC_ID_DYLIB,
@ -542,7 +541,7 @@ func machoDecode(d *decode.D, _ any) any {
LC_LAZY_LOAD_DYLIB,
LC_REEXPORT_DYLIB:
d.FieldStruct("dylib_command", func(d *decode.D) {
offset := d.FieldU32("offset", scalar.ActualHex)
offset := d.FieldU32("offset", scalar.UintHex)
d.FieldU32("timestamp", timestampMapper)
d.FieldU32("current_version")
d.FieldU32("compatibility_version")
@ -551,10 +550,10 @@ func machoDecode(d *decode.D, _ any) any {
case LC_LOAD_DYLINKER,
LC_ID_DYLINKER,
LC_DYLD_ENVIRONMENT:
offset := d.FieldU32("offset", scalar.ActualHex)
offset := d.FieldU32("offset", scalar.UintHex)
d.FieldUTF8NullFixedLen("name", int(cmdSize)-int(offset))
case LC_RPATH:
offset := d.FieldU32("offset", scalar.ActualHex)
offset := d.FieldU32("offset", scalar.UintHex)
d.FieldUTF8NullFixedLen("name", int(cmdSize)-int(offset))
case LC_PREBOUND_DYLIB:
// https://github.com/aidansteele/osx-abi-macho-file-format-reference#prebound_dylib_command
@ -588,7 +587,7 @@ func machoDecode(d *decode.D, _ any) any {
case LC_ROUTINES,
LC_ROUTINES_64:
if archBits == 32 {
d.FieldU32("init_address", scalar.ActualHex)
d.FieldU32("init_address", scalar.UintHex)
d.FieldU32("init_module")
d.FieldU32("reserved1")
d.FieldU32("reserved2")
@ -597,7 +596,7 @@ func machoDecode(d *decode.D, _ any) any {
d.FieldU32("reserved5")
d.FieldU32("reserved6")
} else {
d.FieldU64("init_address", scalar.ActualHex)
d.FieldU64("init_address", scalar.UintHex)
d.FieldU64("init_module")
d.FieldU64("reserved1")
d.FieldU64("reserved2")
@ -610,7 +609,7 @@ func machoDecode(d *decode.D, _ any) any {
LC_SUB_LIBRARY,
LC_SUB_CLIENT,
LC_SUB_FRAMEWORK:
offset := d.FieldU32("offset", scalar.ActualHex)
offset := d.FieldU32("offset", scalar.UintHex)
d.FieldUTF8NullFixedLen("name", int(cmdSize)-int(offset))
case LC_SYMTAB:
symOff := d.FieldU32("symoff")
@ -626,7 +625,7 @@ func machoDecode(d *decode.D, _ any) any {
d.SeekAbs(int64(symOff) * 8)
d.FieldArray("symbols", func(d *decode.D) {
for i := 0; i < int(nSyms); i++ {
symbolTypeMap := scalar.UToSymStr{
symbolTypeMap := scalar.UintMapSymStr{
0x0: "undef",
0x1: "abs",
0x5: "indr",
@ -644,7 +643,7 @@ func machoDecode(d *decode.D, _ any) any {
})
d.FieldU8("sect")
d.FieldU16("desc")
d.FieldU("value", archBits, scalar.ActualHex)
d.FieldU("value", archBits, scalar.UintHex)
})
}
})
@ -700,20 +699,20 @@ func machoDecode(d *decode.D, _ any) any {
case LC_DYLD_INFO,
LC_DYLD_INFO_ONLY:
d.FieldStruct("dyld_info", func(d *decode.D) {
d.FieldU32("rebase_off", scalar.ActualHex)
d.FieldU32("rebase_off", scalar.UintHex)
d.FieldU32("rebase_size")
d.FieldU32("bind_off", scalar.ActualHex)
d.FieldU32("bind_off", scalar.UintHex)
d.FieldU32("bind_size")
d.FieldU32("weak_bind_off", scalar.ActualHex)
d.FieldU32("weak_bind_off", scalar.UintHex)
d.FieldU32("weak_bind_size")
d.FieldU32("lazy_bind_off", scalar.ActualHex)
d.FieldU32("lazy_bind_off", scalar.UintHex)
d.FieldU32("lazy_bind_size")
d.FieldU32("export_off", scalar.ActualHex)
d.FieldU32("export_off", scalar.UintHex)
d.FieldU32("export_size")
})
case LC_MAIN:
d.FieldStruct("entrypoint", func(d *decode.D) {
d.FieldU64("entryoff", scalar.ActualHex)
d.FieldU64("entryoff", scalar.UintHex)
d.FieldU64("stacksize")
})
case LC_SOURCE_VERSION:
@ -728,7 +727,7 @@ func machoDecode(d *decode.D, _ any) any {
case LC_ENCRYPTION_INFO,
LC_ENCRYPTION_INFO_64:
d.FieldStruct("encryption_info", func(d *decode.D) {
offset := d.FieldU32("offset", scalar.ActualHex)
offset := d.FieldU32("offset", scalar.UintHex)
size := d.FieldU32("size")
d.FieldU32("id")
d.RangeFn(int64(offset)*8, int64(size)*8, func(d *decode.D) {
@ -742,9 +741,9 @@ func machoDecode(d *decode.D, _ any) any {
case LC_IDFVMLIB,
LC_LOADFVMLIB:
d.FieldStruct("fvmlib", func(d *decode.D) {
offset := d.FieldU32("offset", scalar.ActualHex)
offset := d.FieldU32("offset", scalar.UintHex)
d.FieldU32("minor_version")
d.FieldU32("header_addr", scalar.ActualHex)
d.FieldU32("header_addr", scalar.UintHex)
d.FieldUTF8NullFixedLen("name", int(cmdSize)-int(offset))
})
default:
@ -817,8 +816,8 @@ func parseSectionFlags(d *decode.D) {
d.FieldBool("attr_loc_reloc")
}
var timestampMapper = scalar.Fn(func(s scalar.S) (scalar.S, error) {
s.Sym = time.UnixMilli(int64(s.ActualU())).UTC().String()
var timestampMapper = scalar.UintFn(func(s scalar.Uint) (scalar.Uint, error) {
s.Sym = time.UnixMilli(int64(s.Actual)).UTC().String()
return s, nil
})

0
format/macho/macho.md → format/apple/macho/macho.md
View File

32
format/macho/macho_fat.go → format/apple/macho/macho_fat.go
View File

@ -12,21 +12,21 @@ import (
var machoFormat decode.Group
func init() {
interp.RegisterFormat(decode.Format{
Name: format.MACHO_FAT,
Description: "Fat Mach-O macOS executable (multi-architecture)",
Groups: []string{format.PROBE},
DecodeFn: machoFatDecode,
Dependencies: []decode.Dependency{
{Names: []string{format.MACHO}, Group: &machoFormat},
},
})
interp.RegisterFormat(
format.MachO_Fat,
&decode.Format{
Description: "Fat Mach-O macOS executable (multi-architecture)",
Groups: []*decode.Group{format.Probe},
DecodeFn: machoFatDecode,
Dependencies: []decode.Dependency{
{Groups: []*decode.Group{format.MachO}, Out: &machoFormat},
},
})
}
//nolint:revive
const FAT_MAGIC = 0xcafe_babe
func machoFatDecode(d *decode.D, _ any) any {
func machoFatDecode(d *decode.D) any {
type ofile struct {
offset int64
size int64
@ -34,16 +34,16 @@ func machoFatDecode(d *decode.D, _ any) any {
var ofiles []ofile
d.FieldStruct("fat_header", func(d *decode.D) {
d.FieldU32("magic", magicSymMapper, scalar.ActualHex, d.AssertU(FAT_MAGIC))
d.FieldU32("magic", magicSymMapper, scalar.UintHex, d.UintAssert(FAT_MAGIC))
narchs := d.FieldU32("narchs")
d.FieldArray("archs", func(d *decode.D) {
for i := 0; i < int(narchs); i++ {
d.FieldStruct("arch", func(d *decode.D) {
// beware cputype and cpusubtype changes from ofile header to fat header
cpuType := d.FieldU32("cputype", cpuTypes, scalar.ActualHex)
d.FieldU32("cpusubtype", cpuSubTypes[cpuType], scalar.ActualHex)
offset := d.FieldU32("offset", scalar.ActualHex)
cpuType := d.FieldU32("cputype", cpuTypes, scalar.UintHex)
d.FieldU32("cpusubtype", cpuSubTypes[cpuType], scalar.UintHex)
offset := d.FieldU32("offset", scalar.UintHex)
size := d.FieldU32("size")
d.FieldU32("align")
@ -56,7 +56,7 @@ func machoFatDecode(d *decode.D, _ any) any {
d.FieldArray("files", func(d *decode.D) {
for _, o := range ofiles {
d.RangeFn(o.offset*8, o.size*8, func(d *decode.D) {
d.FieldFormat("file", machoFormat, nil)
d.FieldFormat("file", &machoFormat, nil)
})
}
})

0
format/macho/testdata/Makefile → format/apple/macho/testdata/Makefile vendored
View File

0
format/macho/testdata/a.c → format/apple/macho/testdata/a.c vendored
View File

0
format/macho/testdata/darwin_aarch64/a_dynamic → format/apple/macho/testdata/darwin_aarch64/a_dynamic vendored
View File

12
format/macho/testdata/darwin_aarch64/a_dynamic.fqtest → format/apple/macho/testdata/darwin_aarch64/a_dynamic.fqtest vendored
View File

@ -546,16 +546,16 @@ $ fq dv a_dynamic
| | | linkedit_data{}: 0x5a8-0x5af.7 (8)
0x05a0| 60 c1 00 00 | `... | off: 49504 0x5a8-0x5ab.7 (4)
0x05a0| 16 02 00 00| ....| size: 534 0x5ac-0x5af.7 (4)
0x05b0|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| unknown0: raw bits 0x5b0-0x3f2f.7 (14720)
0x05b0|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| gap0: raw bits 0x5b0-0x3f2f.7 (14720)
* |until 0x3f2f.7 (14720) | |
0x3fb0| 00 00 00 | ... | unknown1: raw bits 0x3fb5-0x3fb7.7 (3)
0x4000| 00 00 00 00 00 00 00 00| ........| unknown2: raw bits 0x4008-0x7fff.7 (16376)
0x3fb0| 00 00 00 | ... | gap1: raw bits 0x3fb5-0x3fb7.7 (3)
0x4000| 00 00 00 00 00 00 00 00| ........| gap2: raw bits 0x4008-0x7fff.7 (16376)
0x4010|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x7fff.7 (16376) | |
0x8010| 00 00 00 00 00 00 00 00| ........| unknown3: raw bits 0x8018-0xc07f.7 (16488)
0x8010| 00 00 00 00 00 00 00 00| ........| gap3: raw bits 0x8018-0xc07f.7 (16488)
0x8020|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0xc07f.7 (16488) | |
0xc0f0|04 00 00 00 05 00 00 00 06 00 00 00 04 00 00 00|................| unknown4: raw bits 0xc0f0-0xc107.7 (24)
0xc0f0|04 00 00 00 05 00 00 00 06 00 00 00 04 00 00 00|................| gap4: raw bits 0xc0f0-0xc107.7 (24)
0xc100|05 00 00 00 00 00 00 00 |........ |
0xc160|fa de 0c c0 00 00 02 16 00 00 00 01 00 00 00 00|................| unknown5: raw bits 0xc160-0xc375.7 (534)
0xc160|fa de 0c c0 00 00 02 16 00 00 00 01 00 00 00 00|................| gap5: raw bits 0xc160-0xc375.7 (534)
* |until 0xc375.7 (end) (534) | |

0
format/macho/testdata/darwin_aarch64/a_static → format/apple/macho/testdata/darwin_aarch64/a_static vendored
View File

12
format/macho/testdata/darwin_aarch64/a_static.fqtest → format/apple/macho/testdata/darwin_aarch64/a_static.fqtest vendored
View File

@ -537,16 +537,16 @@ $ fq dv a_static
| | | linkedit_data{}: 0x580-0x587.7 (8)
0x0580|60 c1 00 00 |`... | off: 49504 0x580-0x583.7 (4)
0x0580| 15 02 00 00 | .... | size: 533 0x584-0x587.7 (4)
0x0580| 00 00 00 00 00 00 00 00| ........| unknown0: raw bits 0x588-0x3f1f.7 (14744)
0x0580| 00 00 00 00 00 00 00 00| ........| gap0: raw bits 0x588-0x3f1f.7 (14744)
0x0590|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x3f1f.7 (14744) | |
0x3fb0| 00 00 00 | ... | unknown1: raw bits 0x3fb5-0x3fb7.7 (3)
0x4000| 00 00 00 00 00 00 00 00| ........| unknown2: raw bits 0x4008-0x7fff.7 (16376)
0x3fb0| 00 00 00 | ... | gap1: raw bits 0x3fb5-0x3fb7.7 (3)
0x4000| 00 00 00 00 00 00 00 00| ........| gap2: raw bits 0x4008-0x7fff.7 (16376)
0x4010|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x7fff.7 (16376) | |
0x8010|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| unknown3: raw bits 0x8010-0xc07f.7 (16496)
0x8010|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| gap3: raw bits 0x8010-0xc07f.7 (16496)
* |until 0xc07f.7 (16496) | |
0xc0f0|05 00 00 00 06 00 00 00 05 00 00 00 00 00 00 00|................| unknown4: raw bits 0xc0f0-0xc0ff.7 (16)
0xc150| 00 00 00 00 00 00 00 00| ........| unknown5: raw bits 0xc158-0xc374.7 (541)
0xc0f0|05 00 00 00 06 00 00 00 05 00 00 00 00 00 00 00|................| gap4: raw bits 0xc0f0-0xc0ff.7 (16)
0xc150| 00 00 00 00 00 00 00 00| ........| gap5: raw bits 0xc158-0xc374.7 (541)
0xc160|fa de 0c c0 00 00 02 15 00 00 00 01 00 00 00 00|................|
* |until 0xc374.7 (end) (541) | |

0
format/macho/testdata/darwin_aarch64/a_stripped → format/apple/macho/testdata/darwin_aarch64/a_stripped vendored
View File

12
format/macho/testdata/darwin_aarch64/a_stripped.fqtest → format/apple/macho/testdata/darwin_aarch64/a_stripped.fqtest vendored
View File

@ -526,17 +526,17 @@ $ fq dv a_stripped
| | | linkedit_data{}: 0x5a8-0x5af.7 (8)
0x05a0| 40 c1 00 00 | @... | off: 49472 0x5a8-0x5ab.7 (4)
0x05a0| 18 02 00 00| ....| size: 536 0x5ac-0x5af.7 (4)
0x05b0|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| unknown0: raw bits 0x5b0-0x3f2f.7 (14720)
0x05b0|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| gap0: raw bits 0x5b0-0x3f2f.7 (14720)
* |until 0x3f2f.7 (14720) | |
0x3fb0| 00 00 00 | ... | unknown1: raw bits 0x3fb5-0x3fb7.7 (3)
0x4000| 00 00 00 00 00 00 00 00| ........| unknown2: raw bits 0x4008-0x7fff.7 (16376)
0x3fb0| 00 00 00 | ... | gap1: raw bits 0x3fb5-0x3fb7.7 (3)
0x4000| 00 00 00 00 00 00 00 00| ........| gap2: raw bits 0x4008-0x7fff.7 (16376)
0x4010|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x7fff.7 (16376) | |
0x8010| 00 00 00 00 00 00 00 00| ........| unknown3: raw bits 0x8018-0xc07f.7 (16488)
0x8010| 00 00 00 00 00 00 00 00| ........| gap3: raw bits 0x8018-0xc07f.7 (16488)
0x8020|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0xc07f.7 (16488) | |
0xc0d0|02 00 00 00 03 00 00 00 04 00 00 00 02 00 00 00|................| unknown4: raw bits 0xc0d0-0xc0e7.7 (24)
0xc0d0|02 00 00 00 03 00 00 00 04 00 00 00 02 00 00 00|................| gap4: raw bits 0xc0d0-0xc0e7.7 (24)
0xc0e0|03 00 00 00 00 00 00 00 |........ |
0xc130| 00 00 00 00 00 00 00 00| ........| unknown5: raw bits 0xc138-0xc357.7 (544)
0xc130| 00 00 00 00 00 00 00 00| ........| gap5: raw bits 0xc138-0xc357.7 (544)
0xc140|fa de 0c c0 00 00 02 17 00 00 00 01 00 00 00 00|................|
* |until 0xc357.7 (end) (544) | |

0
format/macho/testdata/darwin_aarch64/libbbb.a → format/apple/macho/testdata/darwin_aarch64/libbbb.a vendored
View File

0
format/macho/testdata/darwin_aarch64/libbbb.so → format/apple/macho/testdata/darwin_aarch64/libbbb.so vendored
View File

10
format/macho/testdata/darwin_aarch64/libbbb.so.fqtest → format/apple/macho/testdata/darwin_aarch64/libbbb.so.fqtest vendored
View File

@ -496,14 +496,14 @@ $ fq dv libbbb.so
| | | linkedit_data{}: 0x528-0x52f.7 (8)
0x0520| e0 c0 00 00 | .... | off: 49376 0x528-0x52b.7 (4)
0x0520| 16 02 00 00| ....| size: 534 0x52c-0x52f.7 (4)
0x0530|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| unknown0: raw bits 0x530-0x3f5f.7 (14896)
0x0530|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| gap0: raw bits 0x530-0x3f5f.7 (14896)
* |until 0x3f5f.7 (14896) | |
0x4000| 00 00 00 00 00 00 00 00| ........| unknown1: raw bits 0x4008-0x7fff.7 (16376)
0x4000| 00 00 00 00 00 00 00 00| ........| gap1: raw bits 0x4008-0x7fff.7 (16376)
0x4010|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x7fff.7 (16376) | |
0x8010|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| unknown2: raw bits 0x8010-0xc04f.7 (16448)
0x8010|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| gap2: raw bits 0x8010-0xc04f.7 (16448)
* |until 0xc04f.7 (16448) | |
0xc090|02 00 00 00 03 00 00 00 02 00 00 00 00 00 00 00|................| unknown3: raw bits 0xc090-0xc09f.7 (16)
0xc0d0| 00 00 00 00 00 00 00 00| ........| unknown4: raw bits 0xc0d8-0xc2f5.7 (542)
0xc090|02 00 00 00 03 00 00 00 02 00 00 00 00 00 00 00|................| gap3: raw bits 0xc090-0xc09f.7 (16)
0xc0d0| 00 00 00 00 00 00 00 00| ........| gap4: raw bits 0xc0d8-0xc2f5.7 (542)
0xc0e0|fa de 0c c0 00 00 02 16 00 00 00 01 00 00 00 00|................|
* |until 0xc2f5.7 (end) (542) | |

0
format/macho/testdata/darwin_amd64/a_dynamic → format/apple/macho/testdata/darwin_amd64/a_dynamic vendored
View File

10
format/macho/testdata/darwin_amd64/a_dynamic.fqtest → format/apple/macho/testdata/darwin_amd64/a_dynamic.fqtest vendored
View File

@ -505,12 +505,12 @@ $ fq dv a_dynamic
| | | linkedit_data{}: 0x540-0x547.7 (8)
0x0540|80 80 00 00 |.... | off: 32896 0x540-0x543.7 (4)
0x0540| 00 00 00 00 | .... | size: 0 0x544-0x547.7 (4)
0x0540| 00 00 00 00 00 00 00 00| ........| unknown0: raw bits 0x548-0x3f3f.7 (14840)
0x0540| 00 00 00 00 00 00 00 00| ........| gap0: raw bits 0x548-0x3f3f.7 (14840)
0x0550|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x3f3f.7 (14840) | |
0x3fa0| 00 00 00 | ... | unknown1: raw bits 0x3fa9-0x3fab.7 (3)
0x3ff0| 00 00 00 00 00 00 00 00 00 00 00 00| ............| unknown2: raw bits 0x3ff4-0x3fff.7 (12)
0x4020|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| unknown3: raw bits 0x4020-0x807f.7 (16480)
0x3fa0| 00 00 00 | ... | gap1: raw bits 0x3fa9-0x3fab.7 (3)
0x3ff0| 00 00 00 00 00 00 00 00 00 00 00 00| ............| gap2: raw bits 0x3ff4-0x3fff.7 (12)
0x4020|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| gap3: raw bits 0x4020-0x807f.7 (16480)
* |until 0x807f.7 (16480) | |
0x80e0|03 00 00 00 04 00 00 00 00 00 00 40 05 00 00 00|...........@....| unknown4: raw bits 0x80e0-0x80f7.7 (24)
0x80e0|03 00 00 00 04 00 00 00 00 00 00 40 05 00 00 00|...........@....| gap4: raw bits 0x80e0-0x80f7.7 (24)
0x80f0|03 00 00 00 04 00 00 00 |........ |

0
format/macho/testdata/darwin_amd64/a_static → format/apple/macho/testdata/darwin_amd64/a_static vendored
View File

10
format/macho/testdata/darwin_amd64/a_static.fqtest → format/apple/macho/testdata/darwin_amd64/a_static.fqtest vendored
View File

@ -497,11 +497,11 @@ $ fq dv a_static
| | | linkedit_data{}: 0x518-0x51f.7 (8)
0x0510| 80 80 00 00 | .... | off: 32896 0x518-0x51b.7 (4)
0x0510| 00 00 00 00| ....| size: 0 0x51c-0x51f.7 (4)
0x0520|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| unknown0: raw bits 0x520-0x3f2f.7 (14864)
0x0520|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| gap0: raw bits 0x520-0x3f2f.7 (14864)
* |until 0x3f2f.7 (14864) | |
0x3f80| 00 00 | .. | unknown1: raw bits 0x3f8a-0x3f8b.7 (2)
0x3fb0| 00 | . | unknown2: raw bits 0x3fb7-0x3fb7.7 (1)
0x4010| 00 00 00 00 00 00 00 00| ........| unknown3: raw bits 0x4018-0x807f.7 (16488)
0x3f80| 00 00 | .. | gap1: raw bits 0x3f8a-0x3f8b.7 (2)
0x3fb0| 00 | . | gap2: raw bits 0x3fb7-0x3fb7.7 (1)
0x4010| 00 00 00 00 00 00 00 00| ........| gap3: raw bits 0x4018-0x807f.7 (16488)
0x4020|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x807f.7 (16488) | |
0x80e0|04 00 00 00 00 00 00 40 05 00 00 00 04 00 00 00|.......@........| unknown4: raw bits 0x80e0-0x80ef.7 (16)
0x80e0|04 00 00 00 00 00 00 40 05 00 00 00 04 00 00 00|.......@........| gap4: raw bits 0x80e0-0x80ef.7 (16)

0
format/macho/testdata/darwin_amd64/a_stripped → format/apple/macho/testdata/darwin_amd64/a_stripped vendored
View File

10
format/macho/testdata/darwin_amd64/a_stripped.fqtest → format/apple/macho/testdata/darwin_amd64/a_stripped.fqtest vendored
View File

@ -495,12 +495,12 @@ $ fq dv a_stripped
| | | linkedit_data{}: 0x540-0x547.7 (8)
0x0540|80 80 00 00 |.... | off: 32896 0x540-0x543.7 (4)
0x0540| 00 00 00 00 | .... | size: 0 0x544-0x547.7 (4)
0x0540| 00 00 00 00 00 00 00 00| ........| unknown0: raw bits 0x548-0x3f3f.7 (14840)
0x0540| 00 00 00 00 00 00 00 00| ........| gap0: raw bits 0x548-0x3f3f.7 (14840)
0x0550|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x3f3f.7 (14840) | |
0x3fa0| 00 00 00 | ... | unknown1: raw bits 0x3fa9-0x3fab.7 (3)
0x3ff0| 00 00 00 00 00 00 00 00 00 00 00 00| ............| unknown2: raw bits 0x3ff4-0x3fff.7 (12)
0x4020|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| unknown3: raw bits 0x4020-0x807f.7 (16480)
0x3fa0| 00 00 00 | ... | gap1: raw bits 0x3fa9-0x3fab.7 (3)
0x3ff0| 00 00 00 00 00 00 00 00 00 00 00 00| ............| gap2: raw bits 0x3ff4-0x3fff.7 (12)
0x4020|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| gap3: raw bits 0x4020-0x807f.7 (16480)
* |until 0x807f.7 (16480) | |
0x80d0|02 00 00 00 03 00 00 00 00 00 00 40 04 00 00 00|...........@....| unknown4: raw bits 0x80d0-0x80e7.7 (24)
0x80d0|02 00 00 00 03 00 00 00 00 00 00 40 04 00 00 00|...........@....| gap4: raw bits 0x80d0-0x80e7.7 (24)
0x80e0|02 00 00 00 03 00 00 00 |........ |

0
format/macho/testdata/darwin_amd64/libbbb.a → format/apple/macho/testdata/darwin_amd64/libbbb.a vendored
View File

0
format/macho/testdata/darwin_amd64/libbbb.so → format/apple/macho/testdata/darwin_amd64/libbbb.so vendored
View File

12
format/macho/testdata/darwin_amd64/libbbb.so.fqtest → format/apple/macho/testdata/darwin_amd64/libbbb.so.fqtest vendored
View File

@ -448,13 +448,13 @@ $ fq dv libbbb.so
| | | linkedit_data{}: 0x4c0-0x4c7.7 (8)
0x04c0|50 80 00 00 |P... | off: 32848 0x4c0-0x4c3.7 (4)
0x04c0| 00 00 00 00 | .... | size: 0 0x4c4-0x4c7.7 (4)
0x04c0| 00 00 00 00 00 00 00 00| ........| unknown0: raw bits 0x4c8-0x3f6f.7 (15016)
0x04c0| 00 00 00 00 00 00 00 00| ........| gap0: raw bits 0x4c8-0x3f6f.7 (15016)
0x04d0|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x3f6f.7 (15016) | |
0x3f80| 00 00 | .. | unknown1: raw bits 0x3f8a-0x3f8b.7 (2)
0x3fb0| 00 00 | .. | unknown2: raw bits 0x3fb2-0x3fb3.7 (2)
0x3ff0| 00 00 00 00| ....| unknown3: raw bits 0x3ffc-0x3fff.7 (4)
0x4010| 00 00 00 00 00 00 00 00| ........| unknown4: raw bits 0x4018-0x804f.7 (16440)
0x3f80| 00 00 | .. | gap1: raw bits 0x3f8a-0x3f8b.7 (2)
0x3fb0| 00 00 | .. | gap2: raw bits 0x3fb2-0x3fb3.7 (2)
0x3ff0| 00 00 00 00| ....| gap3: raw bits 0x3ffc-0x3fff.7 (4)
0x4010| 00 00 00 00 00 00 00 00| ........| gap4: raw bits 0x4018-0x804f.7 (16440)
0x4020|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x804f.7 (16440) | |
0x8080|01 00 00 00 00 00 00 40 02 00 00 00 01 00 00 00|.......@........| unknown5: raw bits 0x8080-0x808f.7 (16)
0x8080|01 00 00 00 00 00 00 40 02 00 00 00 01 00 00 00|.......@........| gap5: raw bits 0x8080-0x808f.7 (16)

0
format/macho/testdata/darwin_fat/a_dynamic → format/apple/macho/testdata/darwin_fat/a_dynamic vendored
View File

26
format/macho/testdata/darwin_fat/a_dynamic.fqtest → format/apple/macho/testdata/darwin_fat/a_dynamic.fqtest vendored
View File

@ -16,7 +16,7 @@ $ fq dv a_dynamic
0x00020| 00 01 00 00 | .... | offset: 0x10000 0x24-0x27.7 (4)
0x00020| 00 00 c3 76 | ...v | size: 50038 0x28-0x2b.7 (4)
0x00020| 00 00 00 0e| ....| align: 14 0x2c-0x2f.7 (4)
0x00030|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| unknown0: raw bits 0x30-0x3fff.7 (16336)
0x00030|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| gap0: raw bits 0x30-0x3fff.7 (16336)
* |until 0x3fff.7 (16336) | |
| | | files[0:2]: 0x4000-0x1c15f.7 (98656)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef| [0]{}: file (macho) 0x4000-0xc13f.7 (33088)
@ -1072,27 +1072,27 @@ $ fq dv a_dynamic
| | | linkedit_data{}: 0x105a8-0x105af.7 (8)
0x105a0| 60 c1 00 00 | `... | off: 49504 0x105a8-0x105ab.7 (4)
0x105a0| 16 02 00 00| ....| size: 534 0x105ac-0x105af.7 (4)
0x04540| 00 00 00 00 00 00 00 00| ........| unknown1: raw bits 0x4548-0x7f3f.7 (14840)
0x04540| 00 00 00 00 00 00 00 00| ........| gap1: raw bits 0x4548-0x7f3f.7 (14840)
0x04550|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x7f3f.7 (14840) | |
0x07fa0| 00 00 00 | ... | unknown2: raw bits 0x7fa9-0x7fab.7 (3)
0x07ff0| 00 00 00 00 00 00 00 00 00 00 00 00| ............| unknown3: raw bits 0x7ff4-0x7fff.7 (12)
0x08020|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| unknown4: raw bits 0x8020-0xc07f.7 (16480)
0x07fa0| 00 00 00 | ... | gap2: raw bits 0x7fa9-0x7fab.7 (3)
0x07ff0| 00 00 00 00 00 00 00 00 00 00 00 00| ............| gap3: raw bits 0x7ff4-0x7fff.7 (12)
0x08020|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| gap4: raw bits 0x8020-0xc07f.7 (16480)
* |until 0xc07f.7 (16480) | |
0x0c0e0|03 00 00 00 04 00 00 00 00 00 00 40 05 00 00 00|...........@....| unknown5: raw bits 0xc0e0-0xc0f7.7 (24)
0x0c0e0|03 00 00 00 04 00 00 00 00 00 00 40 05 00 00 00|...........@....| gap5: raw bits 0xc0e0-0xc0f7.7 (24)
0x0c0f0|03 00 00 00 04 00 00 00 |........ |
0x0c140|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| unknown6: raw bits 0xc140-0xffff.7 (16064)
0x0c140|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| gap6: raw bits 0xc140-0xffff.7 (16064)
* |until 0xffff.7 (16064) | |
0x105b0|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| unknown7: raw bits 0x105b0-0x13f2f.7 (14720)
0x105b0|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| gap7: raw bits 0x105b0-0x13f2f.7 (14720)
* |until 0x13f2f.7 (14720) | |
0x13fb0| 00 00 00 | ... | unknown8: raw bits 0x13fb5-0x13fb7.7 (3)
0x14000| 00 00 00 00 00 00 00 00| ........| unknown9: raw bits 0x14008-0x17fff.7 (16376)
0x13fb0| 00 00 00 | ... | gap8: raw bits 0x13fb5-0x13fb7.7 (3)
0x14000| 00 00 00 00 00 00 00 00| ........| gap9: raw bits 0x14008-0x17fff.7 (16376)
0x14010|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x17fff.7 (16376) | |
0x18010| 00 00 00 00 00 00 00 00| ........| unknown10: raw bits 0x18018-0x1c07f.7 (16488)
0x18010| 00 00 00 00 00 00 00 00| ........| gap10: raw bits 0x18018-0x1c07f.7 (16488)
0x18020|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x1c07f.7 (16488) | |
0x1c0f0|04 00 00 00 05 00 00 00 06 00 00 00 04 00 00 00|................| unknown11: raw bits 0x1c0f0-0x1c107.7 (24)
0x1c0f0|04 00 00 00 05 00 00 00 06 00 00 00 04 00 00 00|................| gap11: raw bits 0x1c0f0-0x1c107.7 (24)
0x1c100|05 00 00 00 00 00 00 00 |........ |
0x1c160|fa de 0c c0 00 00 02 16 00 00 00 01 00 00 00 00|................| unknown12: raw bits 0x1c160-0x1c375.7 (534)
0x1c160|fa de 0c c0 00 00 02 16 00 00 00 01 00 00 00 00|................| gap12: raw bits 0x1c160-0x1c375.7 (534)
* |until 0x1c375.7 (end) (534) | |

0
format/macho/testdata/darwin_fat/a_static → format/apple/macho/testdata/darwin_fat/a_static vendored
View File

26
format/macho/testdata/darwin_fat/a_static.fqtest → format/apple/macho/testdata/darwin_fat/a_static.fqtest vendored
View File

@ -16,7 +16,7 @@ $ fq dv a_static
0x00020| 00 01 00 00 | .... | offset: 0x10000 0x24-0x27.7 (4)
0x00020| 00 00 c3 75 | ...u | size: 50037 0x28-0x2b.7 (4)
0x00020| 00 00 00 0e| ....| align: 14 0x2c-0x2f.7 (4)
0x00030|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| unknown0: raw bits 0x30-0x3fff.7 (16336)
0x00030|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| gap0: raw bits 0x30-0x3fff.7 (16336)
* |until 0x3fff.7 (16336) | |
| | | files[0:2]: 0x4000-0x1c157.7 (98648)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef| [0]{}: file (macho) 0x4000-0xc137.7 (33080)
@ -1055,27 +1055,27 @@ $ fq dv a_static
| | | linkedit_data{}: 0x10580-0x10587.7 (8)
0x10580|60 c1 00 00 |`... | off: 49504 0x10580-0x10583.7 (4)
0x10580| 15 02 00 00 | .... | size: 533 0x10584-0x10587.7 (4)
0x04520|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| unknown1: raw bits 0x4520-0x7f2f.7 (14864)
0x04520|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| gap1: raw bits 0x4520-0x7f2f.7 (14864)
* |until 0x7f2f.7 (14864) | |
0x07f80| 00 00 | .. | unknown2: raw bits 0x7f8a-0x7f8b.7 (2)
0x07fb0| 00 | . | unknown3: raw bits 0x7fb7-0x7fb7.7 (1)
0x08010| 00 00 00 00 00 00 00 00| ........| unknown4: raw bits 0x8018-0xc07f.7 (16488)
0x07f80| 00 00 | .. | gap2: raw bits 0x7f8a-0x7f8b.7 (2)
0x07fb0| 00 | . | gap3: raw bits 0x7fb7-0x7fb7.7 (1)
0x08010| 00 00 00 00 00 00 00 00| ........| gap4: raw bits 0x8018-0xc07f.7 (16488)
0x08020|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0xc07f.7 (16488) | |
0x0c0e0|04 00 00 00 00 00 00 40 05 00 00 00 04 00 00 00|.......@........| unknown5: raw bits 0xc0e0-0xc0ef.7 (16)
0x0c130| 00 00 00 00 00 00 00 00| ........| unknown6: raw bits 0xc138-0xffff.7 (16072)
0x0c0e0|04 00 00 00 00 00 00 40 05 00 00 00 04 00 00 00|.......@........| gap5: raw bits 0xc0e0-0xc0ef.7 (16)
0x0c130| 00 00 00 00 00 00 00 00| ........| gap6: raw bits 0xc138-0xffff.7 (16072)
0x0c140|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0xffff.7 (16072) | |
0x10580| 00 00 00 00 00 00 00 00| ........| unknown7: raw bits 0x10588-0x13f1f.7 (14744)
0x10580| 00 00 00 00 00 00 00 00| ........| gap7: raw bits 0x10588-0x13f1f.7 (14744)
0x10590|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x13f1f.7 (14744) | |
0x13fb0| 00 00 00 | ... | unknown8: raw bits 0x13fb5-0x13fb7.7 (3)
0x14000| 00 00 00 00 00 00 00 00| ........| unknown9: raw bits 0x14008-0x17fff.7 (16376)
0x13fb0| 00 00 00 | ... | gap8: raw bits 0x13fb5-0x13fb7.7 (3)
0x14000| 00 00 00 00 00 00 00 00| ........| gap9: raw bits 0x14008-0x17fff.7 (16376)
0x14010|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x17fff.7 (16376) | |
0x18010|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| unknown10: raw bits 0x18010-0x1c07f.7 (16496)
0x18010|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| gap10: raw bits 0x18010-0x1c07f.7 (16496)
* |until 0x1c07f.7 (16496) | |
0x1c0f0|05 00 00 00 06 00 00 00 05 00 00 00 00 00 00 00|................| unknown11: raw bits 0x1c0f0-0x1c0ff.7 (16)
0x1c150| 00 00 00 00 00 00 00 00| ........| unknown12: raw bits 0x1c158-0x1c374.7 (541)
0x1c0f0|05 00 00 00 06 00 00 00 05 00 00 00 00 00 00 00|................| gap11: raw bits 0x1c0f0-0x1c0ff.7 (16)
0x1c150| 00 00 00 00 00 00 00 00| ........| gap12: raw bits 0x1c158-0x1c374.7 (541)
0x1c160|fa de 0c c0 00 00 02 15 00 00 00 01 00 00 00 00|................|
* |until 0x1c374.7 (end) (541) | |

0
format/macho/testdata/darwin_fat/a_stripped → format/apple/macho/testdata/darwin_fat/a_stripped vendored
View File

26
format/macho/testdata/darwin_fat/a_stripped.fqtest → format/apple/macho/testdata/darwin_fat/a_stripped.fqtest vendored
View File

@ -16,7 +16,7 @@ $ fq dv a_stripped
0x00020| 00 01 00 00 | .... | offset: 0x10000 0x24-0x27.7 (4)
0x00020| 00 00 c3 58 | ...X | size: 50008 0x28-0x2b.7 (4)
0x00020| 00 00 00 0e| ....| align: 14 0x2c-0x2f.7 (4)
0x00030|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| unknown0: raw bits 0x30-0x3fff.7 (16336)
0x00030|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| gap0: raw bits 0x30-0x3fff.7 (16336)
* |until 0x3fff.7 (16336) | |
| | | files[0:2]: 0x4000-0x1c137.7 (98616)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef| [0]{}: file (macho) 0x4000-0xc137.7 (33080)
@ -1042,29 +1042,29 @@ $ fq dv a_stripped
| | | linkedit_data{}: 0x105a8-0x105af.7 (8)
0x105a0| 40 c1 00 00 | @... | off: 49472 0x105a8-0x105ab.7 (4)
0x105a0| 18 02 00 00| ....| size: 536 0x105ac-0x105af.7 (4)
0x04540| 00 00 00 00 00 00 00 00| ........| unknown1: raw bits 0x4548-0x7f3f.7 (14840)
0x04540| 00 00 00 00 00 00 00 00| ........| gap1: raw bits 0x4548-0x7f3f.7 (14840)
0x04550|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x7f3f.7 (14840) | |
0x07fa0| 00 00 00 | ... | unknown2: raw bits 0x7fa9-0x7fab.7 (3)
0x07ff0| 00 00 00 00 00 00 00 00 00 00 00 00| ............| unknown3: raw bits 0x7ff4-0x7fff.7 (12)
0x08020|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| unknown4: raw bits 0x8020-0xc07f.7 (16480)
0x07fa0| 00 00 00 | ... | gap2: raw bits 0x7fa9-0x7fab.7 (3)
0x07ff0| 00 00 00 00 00 00 00 00 00 00 00 00| ............| gap3: raw bits 0x7ff4-0x7fff.7 (12)
0x08020|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| gap4: raw bits 0x8020-0xc07f.7 (16480)
* |until 0xc07f.7 (16480) | |
0x0c0d0|02 00 00 00 03 00 00 00 00 00 00 40 04 00 00 00|...........@....| unknown5: raw bits 0xc0d0-0xc0e7.7 (24)
0x0c0d0|02 00 00 00 03 00 00 00 00 00 00 40 04 00 00 00|...........@....| gap5: raw bits 0xc0d0-0xc0e7.7 (24)
0x0c0e0|02 00 00 00 03 00 00 00 |........ |
0x0c130| 00 00 00 00 00 00 00 00| ........| unknown6: raw bits 0xc138-0xffff.7 (16072)
0x0c130| 00 00 00 00 00 00 00 00| ........| gap6: raw bits 0xc138-0xffff.7 (16072)
0x0c140|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0xffff.7 (16072) | |
0x105b0|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| unknown7: raw bits 0x105b0-0x13f2f.7 (14720)
0x105b0|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| gap7: raw bits 0x105b0-0x13f2f.7 (14720)
* |until 0x13f2f.7 (14720) | |
0x13fb0| 00 00 00 | ... | unknown8: raw bits 0x13fb5-0x13fb7.7 (3)
0x14000| 00 00 00 00 00 00 00 00| ........| unknown9: raw bits 0x14008-0x17fff.7 (16376)
0x13fb0| 00 00 00 | ... | gap8: raw bits 0x13fb5-0x13fb7.7 (3)
0x14000| 00 00 00 00 00 00 00 00| ........| gap9: raw bits 0x14008-0x17fff.7 (16376)
0x14010|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x17fff.7 (16376) | |
0x18010| 00 00 00 00 00 00 00 00| ........| unknown10: raw bits 0x18018-0x1c07f.7 (16488)
0x18010| 00 00 00 00 00 00 00 00| ........| gap10: raw bits 0x18018-0x1c07f.7 (16488)
0x18020|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x1c07f.7 (16488) | |
0x1c0d0|02 00 00 00 03 00 00 00 04 00 00 00 02 00 00 00|................| unknown11: raw bits 0x1c0d0-0x1c0e7.7 (24)
0x1c0d0|02 00 00 00 03 00 00 00 04 00 00 00 02 00 00 00|................| gap11: raw bits 0x1c0d0-0x1c0e7.7 (24)
0x1c0e0|03 00 00 00 00 00 00 00 |........ |
0x1c130| 00 00 00 00 00 00 00 00| ........| unknown12: raw bits 0x1c138-0x1c357.7 (544)
0x1c130| 00 00 00 00 00 00 00 00| ........| gap12: raw bits 0x1c138-0x1c357.7 (544)
0x1c140|fa de 0c c0 00 00 02 17 00 00 00 01 00 00 00 00|................|
* |until 0x1c357.7 (end) (544) | |

0
format/macho/testdata/darwin_fat/libbbb.so → format/apple/macho/testdata/darwin_fat/libbbb.so vendored
View File

26
format/macho/testdata/darwin_fat/libbbb.so.fqtest → format/apple/macho/testdata/darwin_fat/libbbb.so.fqtest vendored
View File

@ -16,7 +16,7 @@ $ fq dv libbbb.so
0x00020| 00 01 00 00 | .... | offset: 0x10000 0x24-0x27.7 (4)
0x00020| 00 00 c2 f6 | .... | size: 49910 0x28-0x2b.7 (4)
0x00020| 00 00 00 0e| ....| align: 14 0x2c-0x2f.7 (4)
0x00030|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| unknown0: raw bits 0x30-0x3fff.7 (16336)
0x00030|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| gap0: raw bits 0x30-0x3fff.7 (16336)
* |until 0x3fff.7 (16336) | |
| | | files[0:2]: 0x4000-0x1c0d7.7 (98520)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef| [0]{}: file (macho) 0x4000-0xc0b7.7 (32952)
@ -965,27 +965,27 @@ $ fq dv libbbb.so
| | | linkedit_data{}: 0x10528-0x1052f.7 (8)
0x10520| e0 c0 00 00 | .... | off: 49376 0x10528-0x1052b.7 (4)
0x10520| 16 02 00 00| ....| size: 534 0x1052c-0x1052f.7 (4)
0x044c0| 00 00 00 00 00 00 00 00| ........| unknown1: raw bits 0x44c8-0x7f6f.7 (15016)
0x044c0| 00 00 00 00 00 00 00 00| ........| gap1: raw bits 0x44c8-0x7f6f.7 (15016)
0x044d0|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x7f6f.7 (15016) | |
0x07f80| 00 00 | .. | unknown2: raw bits 0x7f8a-0x7f8b.7 (2)
0x07fb0| 00 00 | .. | unknown3: raw bits 0x7fb2-0x7fb3.7 (2)
0x07ff0| 00 00 00 00| ....| unknown4: raw bits 0x7ffc-0x7fff.7 (4)
0x08010| 00 00 00 00 00 00 00 00| ........| unknown5: raw bits 0x8018-0xc04f.7 (16440)
0x07f80| 00 00 | .. | gap2: raw bits 0x7f8a-0x7f8b.7 (2)
0x07fb0| 00 00 | .. | gap3: raw bits 0x7fb2-0x7fb3.7 (2)
0x07ff0| 00 00 00 00| ....| gap4: raw bits 0x7ffc-0x7fff.7 (4)
0x08010| 00 00 00 00 00 00 00 00| ........| gap5: raw bits 0x8018-0xc04f.7 (16440)
0x08020|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0xc04f.7 (16440) | |
0x0c080|01 00 00 00 00 00 00 40 02 00 00 00 01 00 00 00|.......@........| unknown6: raw bits 0xc080-0xc08f.7 (16)
0x0c0b0| 00 00 00 00 00 00 00 00| ........| unknown7: raw bits 0xc0b8-0xffff.7 (16200)
0x0c080|01 00 00 00 00 00 00 40 02 00 00 00 01 00 00 00|.......@........| gap6: raw bits 0xc080-0xc08f.7 (16)
0x0c0b0| 00 00 00 00 00 00 00 00| ........| gap7: raw bits 0xc0b8-0xffff.7 (16200)
0x0c0c0|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0xffff.7 (16200) | |
0x10530|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| unknown8: raw bits 0x10530-0x13f5f.7 (14896)
0x10530|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| gap8: raw bits 0x10530-0x13f5f.7 (14896)
* |until 0x13f5f.7 (14896) | |
0x14000| 00 00 00 00 00 00 00 00| ........| unknown9: raw bits 0x14008-0x17fff.7 (16376)
0x14000| 00 00 00 00 00 00 00 00| ........| gap9: raw bits 0x14008-0x17fff.7 (16376)
0x14010|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................|
* |until 0x17fff.7 (16376) | |
0x18010|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| unknown10: raw bits 0x18010-0x1c04f.7 (16448)
0x18010|00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00|................| gap10: raw bits 0x18010-0x1c04f.7 (16448)
* |until 0x1c04f.7 (16448) | |
0x1c090|02 00 00 00 03 00 00 00 02 00 00 00 00 00 00 00|................| unknown11: raw bits 0x1c090-0x1c09f.7 (16)
0x1c0d0| 00 00 00 00 00 00 00 00| ........| unknown12: raw bits 0x1c0d8-0x1c2f5.7 (542)
0x1c090|02 00 00 00 03 00 00 00 02 00 00 00 00 00 00 00|................| gap11: raw bits 0x1c090-0x1c09f.7 (16)
0x1c0d0| 00 00 00 00 00 00 00 00| ........| gap12: raw bits 0x1c0d8-0x1c2f5.7 (542)
0x1c0e0|fa de 0c c0 00 00 02 16 00 00 00 01 00 00 00 00|................|
* |until 0x1c2f5.7 (end) (542) | |

5
format/macho/testdata/help_macho.fqtest → format/apple/macho/testdata/help_macho.fqtest vendored
View File

@ -15,12 +15,13 @@ Select 64bit load segments
==========================
$ fq '.load_commands[] | select(.cmd=="segment_64")' file
References
==========
- https://github.com/aidansteele/osx-abi-macho-file-format-reference
Authors
=======
- Sıddık AÇIL acils@itu.edu.tr @Akaame (https://github.com/Akaame)
- Sıddık AÇIL acils@itu.edu.tr @Akaame (https://github.com/Akaame)

0
format/macho/testdata/libbbb.c → format/apple/macho/testdata/libbbb.c vendored
View File

0
format/macho/testdata/libbbb.h → format/apple/macho/testdata/libbbb.h vendored
View File

42
format/ar/ar.go
View File

@ -7,38 +7,38 @@ import (
"github.com/wader/fq/pkg/scalar"
)
var probeFormat decode.Group
var probeGroup decode.Group
func init() {
interp.RegisterFormat(decode.Format{
Name: format.AR,
Description: "Unix archive",
Groups: []string{format.PROBE},
DecodeFn: decodeAr,
Dependencies: []decode.Dependency{
{Names: []string{format.PROBE}, Group: &probeFormat},
},
})
interp.RegisterFormat(
format.AR,
&decode.Format{
Description: "Unix archive",
Groups: []*decode.Group{format.Probe},
DecodeFn: decodeAr,
Dependencies: []decode.Dependency{
{Groups: []*decode.Group{format.Probe}, Out: &probeGroup},
},
})
}
func decodeAr(d *decode.D, _ any) any {
d.FieldUTF8("signature", 8, d.AssertStr("!<arch>\n"))
func decodeAr(d *decode.D) any {
d.FieldUTF8("signature", 8, d.StrAssert("!<arch>\n"))
d.FieldArray("files", func(d *decode.D) {
for !d.End() {
d.FieldStruct("file", func(d *decode.D) {
d.FieldUTF8("identifier", 16, scalar.ActualTrimSpace)
// TODO: try scalar.DescriptionSymUUnixTime
d.FieldUTF8("modification_timestamp", 12, scalar.ActualTrimSpace, scalar.TrySymUParseUint(10))
d.FieldUTF8("owner_id", 6, scalar.ActualTrimSpace, scalar.TrySymUParseUint(10))
d.FieldUTF8("group_id", 6, scalar.ActualTrimSpace, scalar.TrySymUParseUint(10))
d.FieldUTF8("file_mode", 8, scalar.ActualTrimSpace, scalar.TrySymUParseUint(8)) // Octal
sizeS := d.FieldScalarUTF8("file_size", 10, scalar.ActualTrimSpace, scalar.TrySymUParseUint(10))
if sizeS.Sym == nil {
d.FieldUTF8("modification_timestamp", 12, scalar.ActualTrimSpace, scalar.TryStrSymParseUint(10))
d.FieldUTF8("owner_id", 6, scalar.ActualTrimSpace, scalar.TryStrSymParseUint(10))
d.FieldUTF8("group_id", 6, scalar.ActualTrimSpace, scalar.TryStrSymParseUint(10))
d.FieldUTF8("file_mode", 8, scalar.ActualTrimSpace, scalar.TryStrSymParseUint(8)) // Octal
sizeStr := d.FieldScalarUTF8("file_size", 10, scalar.ActualTrimSpace, scalar.TryStrSymParseUint(10))
if sizeStr.Sym == nil {
d.Fatalf("could not decode file_size")
}
size := int64(sizeS.SymU()) * 8
size := int64(sizeStr.SymUint()) * 8
d.FieldUTF8("ending_characters", 2)
d.FieldFormatOrRawLen("data", size, probeFormat, nil)
d.FieldFormatOrRawLen("data", size, &probeGroup, nil)
padding := d.AlignBits(16)
if padding > 0 {
d.FieldRawLen("padding", int64(padding))

73
format/asn1/asn1_ber.go
View File

@ -33,12 +33,13 @@ import (
var asn1FS embed.FS
func init() {
interp.RegisterFormat(decode.Format{
Name: format.ASN1_BER,
Description: "ASN1 BER (basic encoding rules, also CER and DER)",
DecodeFn: decodeASN1BER,
Functions: []string{"torepr"},
})
interp.RegisterFormat(
format.ASN1_BER,
&decode.Format{
Description: "ASN1 BER (basic encoding rules, also CER and DER)",
DecodeFn: decodeASN1BER,
Functions: []string{"torepr"},
})
interp.RegisterFS(asn1FS)
}
@ -49,7 +50,7 @@ const (
classPrivate = 0b11
)
var tagClassMap = scalar.UToSymStr{
var tagClassMap = scalar.UintMapSymStr{
classUniversal: "universal",
classApplication: "application",
classContext: "context",
@ -61,7 +62,7 @@ const (
formConstructed = 1
)
var constructedPrimitiveMap = scalar.UToSymStr{
var constructedPrimitiveMap = scalar.UintMapSymStr{
formConstructed: "constructed",
formPrimitive: "primitive",
}
@ -95,7 +96,7 @@ const (
universalTypeUniversalString = 0x1c // not encoded?
)
var universalTypeMap = scalar.UToSymStr{
var universalTypeMap = scalar.UintMapSymStr{
universalTypeEndOfContent: "end_of_content",
universalTypeBoolean: "boolean",
universalTypeInteger: "integer",
@ -136,7 +137,7 @@ const (
decimalMinusZero = 0b00_00_11
)
var lengthMap = scalar.UToSymStr{
var lengthMap = scalar.UintMapSymStr{
0: "indefinite",
}
@ -179,12 +180,12 @@ func decodeASN1BERValue(d *decode.D, bib *bitio.Buffer, sb *strings.Builder, par
var tag uint64
switch class {
case classUniversal:
tag = d.FieldUFn("tag", decodeTagNumber, universalTypeMap, scalar.ActualHex)
tag = d.FieldUintFn("tag", decodeTagNumber, universalTypeMap, scalar.UintHex)
default:
tag = d.FieldUFn("tag", decodeTagNumber)
tag = d.FieldUintFn("tag", decodeTagNumber)
}
length := d.FieldUFn("length", decodeLength, lengthMap)
length := d.FieldUintFn("length", decodeLength, lengthMap)
var l int64
switch length {
case lengthIndefinite:
@ -202,7 +203,7 @@ func decodeASN1BERValue(d *decode.D, bib *bitio.Buffer, sb *strings.Builder, par
case form == formConstructed || tag == universalTypeSequence || tag == universalTypeSet:
d.FieldArray("constructed", func(d *decode.D) {
for !d.End() {
if length == lengthIndefinite && d.PeekBits(16) == lengthEndMarker {
if length == lengthIndefinite && d.PeekUintBits(16) == lengthEndMarker {
break
}
@ -261,9 +262,9 @@ func decodeASN1BERValue(d *decode.D, bib *bitio.Buffer, sb *strings.Builder, par
case class == classUniversal && tag == universalTypeEndOfContent:
// nop
case class == classUniversal && tag == universalTypeBoolean:
d.FieldU8("value", scalar.URangeToScalar{
{Range: [2]uint64{0, 0}, S: scalar.S{Sym: false}},
{Range: [2]uint64{0x01, 0xff1}, S: scalar.S{Sym: true}},
d.FieldU8("value", scalar.UintRangeToScalar{
{Range: [2]uint64{0, 0}, S: scalar.Uint{Sym: false}},
{Range: [2]uint64{0x01, 0xff1}, S: scalar.Uint{Sym: true}},
})
case class == classUniversal && tag == universalTypeInteger:
if length > 8 {
@ -295,15 +296,15 @@ func decodeASN1BERValue(d *decode.D, bib *bitio.Buffer, sb *strings.Builder, par
}
}
case class == classUniversal && tag == universalTypeNull:
d.FieldValueNil("value")
d.FieldValueAny("value", nil)
case class == classUniversal && tag == universalTypeObjectIdentifier:
d.FieldArray("value", func(d *decode.D) {
// first byte is = oid0*40 + oid1
d.FieldUFn("oid", func(d *decode.D) uint64 { return d.U8() / 40 })
d.FieldUintFn("oid", func(d *decode.D) uint64 { return d.U8() / 40 })
d.SeekRel(-8)
d.FieldUFn("oid", func(d *decode.D) uint64 { return d.U8() % 40 })
d.FieldUintFn("oid", func(d *decode.D) uint64 { return d.U8() % 40 })
for !d.End() {
d.FieldUFn("oid", func(d *decode.D) uint64 {
d.FieldUintFn("oid", func(d *decode.D) uint64 {
more := true
var n uint64
for more {
@ -322,20 +323,20 @@ func decodeASN1BERValue(d *decode.D, bib *bitio.Buffer, sb *strings.Builder, par
case class == classUniversal && tag == universalTypeReal:
switch {
case length == 0:
d.FieldValueU("value", 0)
d.FieldValueUint("value", 0)
default:
switch d.FieldBool("binary_encoding") {
case true:
s := d.FieldScalarBool("sign", scalar.BoolToSymS{
s := d.FieldScalarBool("sign", scalar.BoolMapSymSint{
true: -1,
false: 1,
}).SymS()
base := d.FieldScalarU2("base", scalar.UToSymU{
}).SymSint()
base := d.FieldScalarU2("base", scalar.UintMapSymUint{
0b00: 2,
0b01: 8,
0b10: 16,
0b11: 0,
}).SymU()
}).SymUint()
scale := d.FieldU2("scale")
format := d.FieldU2("format")
@ -354,15 +355,13 @@ func decodeASN1BERValue(d *decode.D, bib *bitio.Buffer, sb *strings.Builder, par
}
n := d.FieldU("n", int(d.BitsLeft()))
m := float64(s) * float64(n) * math.Pow(float64(base), float64(exp)) * float64(int(1)<<scale)
d.FieldValueFloat("value", m)
d.FieldValueFlt("value", m)
case false:
switch d.FieldBool("decimal_encoding") {
case true:
n := d.FieldU6("special", scalar.UToSymStr{
n := d.FieldU6("special", scalar.UintMapSymStr{
decimalPlusInfinity: "plus_infinity",
decimalMinusInfinity: "minus_infinity",
decimalNan: "nan",
@ -371,21 +370,21 @@ func decodeASN1BERValue(d *decode.D, bib *bitio.Buffer, sb *strings.Builder, par
switch n {
case decimalPlusInfinity:
d.FieldValueFloat("value", math.Inf(1))
d.FieldValueFlt("value", math.Inf(1))
case decimalMinusInfinity:
d.FieldValueFloat("value", math.Inf(-1))
d.FieldValueFlt("value", math.Inf(-1))
case decimalNan:
d.FieldValueFloat("value", math.NaN())
d.FieldValueFlt("value", math.NaN())
case decimalMinusZero:
d.FieldValueFloat("value", -0)
d.FieldValueFlt("value", -0)
}
case false:
d.FieldU6("representation", scalar.UToSymStr{
d.FieldU6("representation", scalar.UintMapSymStr{
0b00_00_01: "nr1",
0b00_00_10: "nr2",
0b00_00_11: "nr3",
})
d.FieldFFn("value", func(d *decode.D) float64 {
d.FieldFltFn("value", func(d *decode.D) float64 {
// TODO: can ParseFloat do all ISO-6093 nr?
n, _ := strconv.ParseFloat(d.UTF8(int(d.BitsLeft()/8)), 64)
return n
@ -415,7 +414,7 @@ func decodeASN1BERValue(d *decode.D, bib *bitio.Buffer, sb *strings.Builder, par
})
}
func decodeASN1BER(d *decode.D, _ any) any {
func decodeASN1BER(d *decode.D) any {
decodeASN1BERValue(d, nil, nil, formConstructed, universalTypeSequence)
return nil
}

Some files were not shown because too many files have changed in this diff Show More