mirror of
https://github.com/wader/fq.git
synced 2024-12-22 12:51:38 +03:00
decode,interp: Support for format specific options
interp: Refactor format help and also include options
interp: Add -o name=@path to load file content as value (not documented yet, might change)
interp,decode: Expose decode out value as _out (might change)
interp: Refactor foramts.jq into format_{decode,func,include}.jq
interp: Refactor torepr into _format_func for generic format function overloading
interp: Refactor -o options parsing to be more generic and collect unknowns options to be used as format options
decode of decode alises
func for format overloaded functions
include for format specific jq functions (also _help, torepr etc)
flac_frame: Add bits_per_sample option
mp3: Add max_unique_header_config and max_sync_seek options
mp4: Add decode_samples and allow_truncate options
avc_au: Has length_size option
hevc_au: Has length_size option
aac_frame: Has object_typee option
doc: Rewrite format doc generation, less hack more jq
This commit is contained in:
Mattias Wadman
parent
411c970093
commit
f4480c6fe5
15
README.md
15
README.md
@ -32,9 +32,9 @@ I pronounce jq /‘dʒei’kju:/ so I usually pronounce fq /‘ef’kju:/.
|
||||
|
||||
### Supported formats
|
||||
|
||||
[./formats_list.sh]: sh-start
|
||||
[fq -rn -L doc 'include "formats"; formats_list']: sh-start
|
||||
|
||||
aac_frame,
|
||||
[aac_frame](doc/formats.md#aac_frame),
|
||||
adts,
|
||||
adts_frame,
|
||||
amf0,
|
||||
@ -45,14 +45,14 @@ av1_ccr,
|
||||
av1_frame,
|
||||
av1_obu,
|
||||
avc_annexb,
|
||||
avc_au,
|
||||
[avc_au](doc/formats.md#avc_au),
|
||||
avc_dcr,
|
||||
avc_nalu,
|
||||
avc_pps,
|
||||
avc_sei,
|
||||
avc_sps,
|
||||
[avro_ocf](doc/formats.md#avro_ocf),
|
||||
bencode,
|
||||
[bencode](doc/formats.md#bencode),
|
||||
bsd_loopback_frame,
|
||||
[bson](doc/formats.md#bson),
|
||||
bzip2,
|
||||
@ -63,7 +63,7 @@ elf,
|
||||
ether8023_frame,
|
||||
exif,
|
||||
flac,
|
||||
flac_frame,
|
||||
[flac_frame](doc/formats.md#flac_frame),
|
||||
flac_metadatablock,
|
||||
flac_metadatablocks,
|
||||
flac_picture,
|
||||
@ -71,7 +71,7 @@ flac_streaminfo,
|
||||
gif,
|
||||
gzip,
|
||||
hevc_annexb,
|
||||
hevc_au,
|
||||
[hevc_au](doc/formats.md#hevc_au),
|
||||
hevc_dcr,
|
||||
hevc_nalu,
|
||||
hevc_pps,
|
||||
@ -89,7 +89,7 @@ jpeg,
|
||||
json,
|
||||
[macho](doc/formats.md#macho),
|
||||
[matroska](doc/formats.md#matroska),
|
||||
mp3,
|
||||
[mp3](doc/formats.md#mp3),
|
||||
mp3_frame,
|
||||
[mp4](doc/formats.md#mp4),
|
||||
mpeg_asc,
|
||||
@ -261,6 +261,7 @@ Licenses of direct dependencies:
|
||||
- Forked version of readline https://github.com/chzyer/readline/blob/master/LICENSE (MIT)
|
||||
- gopacket https://github.com/google/gopacket/blob/master/LICENSE (BSD)
|
||||
- mapstructure https://github.com/mitchellh/mapstructure/blob/master/LICENSE (MIT)
|
||||
- copystructure https://github.com/mitchellh/copystructure/blob/master/LICENSE (MIT)
|
||||
- go-difflib https://github.com/pmezard/go-difflib/blob/master/LICENSE (BSD)
|
||||
- golang/x/text https://github.com/golang/text/blob/master/LICENSE (BSD)
|
||||
- golang/snappy https://github.com/golang/snappy/blob/master/LICENSE (BSD)
|
||||
|
||||
149
doc/formats.jq
Executable file
149
doc/formats.jq
Executable file
@ -0,0 +1,149 @@
|
||||
#!/usr/bin/env fq -rnf
|
||||
|
||||
def code: "`\(.)`";
|
||||
def nbsp: gsub(" "; " ");
|
||||
def has_section($f; $fhelp): $fhelp.notes or $fhelp.examples or $fhelp.links or $f.decode_in_arg;
|
||||
|
||||
def formats_list:
|
||||
[ formats
|
||||
| to_entries[] as {$key, $value}
|
||||
| (_format_func($key; "_help")? // {}) as $fhelp
|
||||
| if has_section($value; $fhelp) then "[\($key)](doc/formats.md#\($key))"
|
||||
else $key
|
||||
end
|
||||
] | join(",\n");
|
||||
|
||||
def formats_table:
|
||||
( [ {
|
||||
name: "Name",
|
||||
desc: "Description",
|
||||
uses: "Dependencies"
|
||||
},
|
||||
{
|
||||
name: "-",
|
||||
desc: "-",
|
||||
uses: "-"
|
||||
},
|
||||
( formats
|
||||
| to_entries[]
|
||||
| (_format_func(.key; "_help")? // {}) as $fhelp
|
||||
| {
|
||||
name:
|
||||
( ( .key as $format
|
||||
| if has_section(.value; $fhelp) then "[\($format | code)](#\($format))"
|
||||
else $format | code
|
||||
end
|
||||
)
|
||||
+ " "
|
||||
),
|
||||
desc: (.value.description | nbsp),
|
||||
uses: "<sub>\((((.value.dependencies | flatten | map(code)) | join(" "))? // ""))</sub>"
|
||||
}
|
||||
),
|
||||
( [ formats
|
||||
| to_entries[]
|
||||
| . as $e
|
||||
| select(.value.groups)
|
||||
| .value.groups[] | {key: ., value: $e.key}
|
||||
]
|
||||
| reduce .[] as $e ({}; .[$e.key] += [$e.value])
|
||||
| to_entries[]
|
||||
| {
|
||||
name: ((.key | code) + " "),
|
||||
desc: "Group",
|
||||
uses: "<sub>\(((.value | map(code)) | join(" ")))</sub>"
|
||||
}
|
||||
)
|
||||
]
|
||||
| table(
|
||||
[ .name
|
||||
, .desc
|
||||
, .uses
|
||||
];
|
||||
[ ""
|
||||
, (.[0] | . as $rc | $rc.string | rpad(" "; $rc.maxwidth))
|
||||
, (.[1] | . as $rc | $rc.string | rpad(" "; $rc.maxwidth))
|
||||
, .[2].string
|
||||
, ""
|
||||
] | join("|")
|
||||
)
|
||||
);
|
||||
|
||||
def formats_sections:
|
||||
( formats[] as $f
|
||||
| (_format_func($f.name; "_help")? // {} | _help_format_enrich("fq"; $f; false)) as $fhelp
|
||||
| select(has_section($f; $fhelp))
|
||||
| "### \($f.name)"
|
||||
, ""
|
||||
, ($fhelp.notes | if . then ., "" else empty end)
|
||||
, if $f.decode_in_arg then
|
||||
( "#### Options"
|
||||
, ""
|
||||
, ( [ { name: "Name"
|
||||
, default: "Default"
|
||||
, desc: "Description"
|
||||
}
|
||||
, { name: "-"
|
||||
, default: "-"
|
||||
, desc: "-"
|
||||
}
|
||||
, ( $f.decode_in_arg
|
||||
| to_entries[] as {$key,$value}
|
||||
| { name: ($key | code)
|
||||
, default: ($value | tostring)
|
||||
, desc: $f.decode_in_arg_doc[$key]
|
||||
}
|
||||
)
|
||||
]
|
||||
| table(
|
||||
[ .name
|
||||
, .default
|
||||
, .desc
|
||||
];
|
||||
[ ""
|
||||
, (.[0] | . as $rc | $rc.string | rpad(" "; $rc.maxwidth))
|
||||
, (.[1] | . as $rc | $rc.string | rpad(" "; $rc.maxwidth))
|
||||
, .[2].string
|
||||
, ""
|
||||
] | join("|")
|
||||
)
|
||||
)
|
||||
, ""
|
||||
)
|
||||
else empty
|
||||
end
|
||||
, if $fhelp.examples then
|
||||
( "#### Examples"
|
||||
, ""
|
||||
, ( $fhelp.examples[]
|
||||
| "\(.comment)"
|
||||
, if .shell then
|
||||
( "```"
|
||||
, "$ \(.shell)"
|
||||
, "```"
|
||||
)
|
||||
elif .expr then
|
||||
( "```"
|
||||
, "... | \(.expr)"
|
||||
, "```"
|
||||
)
|
||||
else empty
|
||||
end
|
||||
, ""
|
||||
)
|
||||
)
|
||||
else empty
|
||||
end
|
||||
, if $fhelp.links then
|
||||
( "#### References and links"
|
||||
, ""
|
||||
, ( $fhelp.links[]
|
||||
| if .title then "- [\(.title)](\(.url))"
|
||||
else "- \(.url)"
|
||||
end
|
||||
)
|
||||
, ""
|
||||
)
|
||||
else empty
|
||||
end
|
||||
);
|
||||
514
doc/formats.md
514
doc/formats.md
@ -1,114 +1,114 @@
|
||||
## Supported formats
|
||||
|
||||
[./formats_table.sh]: sh-start
|
||||
[fq -rn -L . 'include "formats"; formats_table']: sh-start
|
||||
|
||||
|Name |Description |Dependencies|
|
||||
|- |- |-|
|
||||
|`aac_frame` |Advanced Audio Coding frame |<sub></sub>|
|
||||
|`adts` |Audio Data Transport Stream |<sub>`adts_frame`</sub>|
|
||||
|`adts_frame` |Audio Data Transport Stream frame |<sub>`aac_frame`</sub>|
|
||||
|`amf0` |Action Message Format 0 |<sub></sub>|
|
||||
|`apev2` |APEv2 metadata tag |<sub>`image`</sub>|
|
||||
|`ar` |Unix archive |<sub>`probe`</sub>|
|
||||
|[`asn1_ber`](#asn1_ber) |ASN1 Basic Encoding Rules (also CER and DER) |<sub></sub>|
|
||||
|`av1_ccr` |AV1 Codec Configuration Record |<sub></sub>|
|
||||
|`av1_frame` |AV1 frame |<sub>`av1_obu`</sub>|
|
||||
|`av1_obu` |AV1 Open Bitstream Unit |<sub></sub>|
|
||||
|`avc_annexb` |H.264/AVC Annex B |<sub>`avc_nalu`</sub>|
|
||||
|`avc_au` |H.264/AVC Access Unit |<sub>`avc_nalu`</sub>|
|
||||
|`avc_dcr` |H.264/AVC Decoder Configuration Record |<sub>`avc_nalu`</sub>|
|
||||
|`avc_nalu` |H.264/AVC Network Access Layer Unit |<sub>`avc_sps` `avc_pps` `avc_sei`</sub>|
|
||||
|`avc_pps` |H.264/AVC Picture Parameter Set |<sub></sub>|
|
||||
|`avc_sei` |H.264/AVC Supplemental Enhancement Information |<sub></sub>|
|
||||
|`avc_sps` |H.264/AVC Sequence Parameter Set |<sub></sub>|
|
||||
|[`avro_ocf`](#avro_ocf) |Avro object container file |<sub></sub>|
|
||||
|`bencode` |BitTorrent bencoding |<sub></sub>|
|
||||
|`bsd_loopback_frame` |BSD loopback frame |<sub>`inet_packet`</sub>|
|
||||
|[`bson`](#bson) |Binary JSON |<sub></sub>|
|
||||
|`bzip2` |bzip2 compression |<sub>`probe`</sub>|
|
||||
|[`cbor`](#cbor) |Concise Binary Object Representation |<sub></sub>|
|
||||
|`dns` |DNS packet |<sub></sub>|
|
||||
|`dns_tcp` |DNS packet (TCP) |<sub></sub>|
|
||||
|`elf` |Executable and Linkable Format |<sub></sub>|
|
||||
|`ether8023_frame` |Ethernet 802.3 frame |<sub>`inet_packet`</sub>|
|
||||
|`exif` |Exchangeable Image File Format |<sub></sub>|
|
||||
|`flac` |Free Lossless Audio Codec file |<sub>`flac_metadatablocks` `flac_frame`</sub>|
|
||||
|`flac_frame` |FLAC frame |<sub></sub>|
|
||||
|`flac_metadatablock` |FLAC metadatablock |<sub>`flac_streaminfo` `flac_picture` `vorbis_comment`</sub>|
|
||||
|`flac_metadatablocks` |FLAC metadatablocks |<sub>`flac_metadatablock`</sub>|
|
||||
|`flac_picture` |FLAC metadatablock picture |<sub>`image`</sub>|
|
||||
|`flac_streaminfo` |FLAC streaminfo |<sub></sub>|
|
||||
|`gif` |Graphics Interchange Format |<sub></sub>|
|
||||
|`gzip` |gzip compression |<sub>`probe`</sub>|
|
||||
|`hevc_annexb` |H.265/HEVC Annex B |<sub>`hevc_nalu`</sub>|
|
||||
|`hevc_au` |H.265/HEVC Access Unit |<sub>`hevc_nalu`</sub>|
|
||||
|`hevc_dcr` |H.265/HEVC Decoder Configuration Record |<sub>`hevc_nalu`</sub>|
|
||||
|`hevc_nalu` |H.265/HEVC Network Access Layer Unit |<sub>`hevc_vps` `hevc_pps` `hevc_sps`</sub>|
|
||||
|`hevc_pps` |H.265/HEVC Picture Parameter Set |<sub></sub>|
|
||||
|`hevc_sps` |H.265/HEVC Sequence Parameter Set |<sub></sub>|
|
||||
|`hevc_vps` |H.265/HEVC Video Parameter Set |<sub></sub>|
|
||||
|`icc_profile` |International Color Consortium profile |<sub></sub>|
|
||||
|`icmp` |Internet Control Message Protocol |<sub></sub>|
|
||||
|`icmpv6` |Internet Control Message Protocol v6 |<sub></sub>|
|
||||
|`id3v1` |ID3v1 metadata |<sub></sub>|
|
||||
|`id3v11` |ID3v1.1 metadata |<sub></sub>|
|
||||
|`id3v2` |ID3v2 metadata |<sub>`image`</sub>|
|
||||
|`ipv4_packet` |Internet protocol v4 packet |<sub>`ip_packet`</sub>|
|
||||
|`ipv6_packet` |Internet protocol v6 packet |<sub>`ip_packet`</sub>|
|
||||
|`jpeg` |Joint Photographic Experts Group file |<sub>`exif` `icc_profile`</sub>|
|
||||
|`json` |JSON |<sub></sub>|
|
||||
|[`macho`](#macho) |Mach-O macOS executable |<sub></sub>|
|
||||
|[`matroska`](#matroska) |Matroska file |<sub>`aac_frame` `av1_ccr` `av1_frame` `avc_au` `avc_dcr` `flac_frame` `flac_metadatablocks` `hevc_au` `hevc_dcr` `image` `mp3_frame` `mpeg_asc` `mpeg_pes_packet` `mpeg_spu` `opus_packet` `vorbis_packet` `vp8_frame` `vp9_cfm` `vp9_frame`</sub>|
|
||||
|`mp3` |MP3 file |<sub>`id3v2` `id3v1` `id3v11` `apev2` `mp3_frame`</sub>|
|
||||
|`mp3_frame` |MPEG audio layer 3 frame |<sub>`xing`</sub>|
|
||||
|[`mp4`](#mp4) |ISOBMFF MPEG-4 part 12 and similar |<sub>`aac_frame` `av1_ccr` `av1_frame` `flac_frame` `flac_metadatablocks` `id3v2` `image` `jpeg` `mp3_frame` `avc_au` `avc_dcr` `mpeg_es` `hevc_au` `hevc_dcr` `mpeg_pes_packet` `opus_packet` `protobuf_widevine` `pssh_playready` `vorbis_packet` `vp9_frame` `vpx_ccr` `icc_profile`</sub>|
|
||||
|`mpeg_asc` |MPEG-4 Audio Specific Config |<sub></sub>|
|
||||
|`mpeg_es` |MPEG Elementary Stream |<sub>`mpeg_asc` `vorbis_packet`</sub>|
|
||||
|`mpeg_pes` |MPEG Packetized elementary stream |<sub>`mpeg_pes_packet` `mpeg_spu`</sub>|
|
||||
|`mpeg_pes_packet` |MPEG Packetized elementary stream packet |<sub></sub>|
|
||||
|`mpeg_spu` |Sub Picture Unit (DVD subtitle) |<sub></sub>|
|
||||
|`mpeg_ts` |MPEG Transport Stream |<sub></sub>|
|
||||
|[`msgpack`](#msgpack) |MessagePack |<sub></sub>|
|
||||
|`ogg` |OGG file |<sub>`ogg_page` `vorbis_packet` `opus_packet` `flac_metadatablock` `flac_frame`</sub>|
|
||||
|`ogg_page` |OGG page |<sub></sub>|
|
||||
|`opus_packet` |Opus packet |<sub>`vorbis_comment`</sub>|
|
||||
|`pcap` |PCAP packet capture |<sub>`link_frame` `tcp_stream` `ipv4_packet`</sub>|
|
||||
|`pcapng` |PCAPNG packet capture |<sub>`link_frame` `tcp_stream` `ipv4_packet`</sub>|
|
||||
|`png` |Portable Network Graphics file |<sub>`icc_profile` `exif`</sub>|
|
||||
|[`protobuf`](#protobuf) |Protobuf |<sub></sub>|
|
||||
|`protobuf_widevine` |Widevine protobuf |<sub>`protobuf`</sub>|
|
||||
|`pssh_playready` |PlayReady PSSH |<sub></sub>|
|
||||
|`raw` |Raw bits |<sub></sub>|
|
||||
|[`rtmp`](#rtmp) |Real-Time Messaging Protocol |<sub>`amf0` `mpeg_asc`</sub>|
|
||||
|`sll2_packet` |Linux cooked capture encapsulation v2 |<sub>`inet_packet`</sub>|
|
||||
|`sll_packet` |Linux cooked capture encapsulation |<sub>`inet_packet`</sub>|
|
||||
|`tar` |Tar archive |<sub>`probe`</sub>|
|
||||
|`tcp_segment` |Transmission control protocol segment |<sub></sub>|
|
||||
|`tiff` |Tag Image File Format |<sub>`icc_profile`</sub>|
|
||||
|`udp_datagram` |User datagram protocol |<sub>`udp_payload`</sub>|
|
||||
|`vorbis_comment` |Vorbis comment |<sub>`flac_picture`</sub>|
|
||||
|`vorbis_packet` |Vorbis packet |<sub>`vorbis_comment`</sub>|
|
||||
|`vp8_frame` |VP8 frame |<sub></sub>|
|
||||
|`vp9_cfm` |VP9 Codec Feature Metadata |<sub></sub>|
|
||||
|`vp9_frame` |VP9 frame |<sub></sub>|
|
||||
|`vpx_ccr` |VPX Codec Configuration Record |<sub></sub>|
|
||||
|`wav` |WAV file |<sub>`id3v2` `id3v1` `id3v11`</sub>|
|
||||
|`webp` |WebP image |<sub>`vp8_frame`</sub>|
|
||||
|`xing` |Xing header |<sub></sub>|
|
||||
|`zip` |ZIP archive |<sub>`probe`</sub>|
|
||||
|`image` |Group |<sub>`gif` `jpeg` `mp4` `png` `tiff` `webp`</sub>|
|
||||
|`inet_packet` |Group |<sub>`ipv4_packet` `ipv6_packet`</sub>|
|
||||
|`ip_packet` |Group |<sub>`icmp` `icmpv6` `tcp_segment` `udp_datagram`</sub>|
|
||||
|`link_frame` |Group |<sub>`bsd_loopback_frame` `ether8023_frame` `sll2_packet` `sll_packet`</sub>|
|
||||
|`probe` |Group |<sub>`adts` `ar` `avro_ocf` `bzip2` `elf` `flac` `gif` `gzip` `jpeg` `json` `macho` `matroska` `mp3` `mp4` `mpeg_ts` `ogg` `pcap` `pcapng` `png` `tar` `tiff` `wav` `webp` `zip`</sub>|
|
||||
|`tcp_stream` |Group |<sub>`dns` `rtmp`</sub>|
|
||||
|`udp_payload` |Group |<sub>`dns`</sub>|
|
||||
|Name |Description |Dependencies|
|
||||
|- |- |-|
|
||||
|[`aac_frame`](#aac_frame) |Advanced Audio Coding frame |<sub></sub>|
|
||||
|`adts` |Audio Data Transport Stream |<sub>`adts_frame`</sub>|
|
||||
|`adts_frame` |Audio Data Transport Stream frame |<sub>`aac_frame`</sub>|
|
||||
|`amf0` |Action Message Format 0 |<sub></sub>|
|
||||
|`apev2` |APEv2 metadata tag |<sub>`image`</sub>|
|
||||
|`ar` |Unix archive |<sub>`probe`</sub>|
|
||||
|[`asn1_ber`](#asn1_ber) |ASN1 BER (basic encoding rules, also CER and DER)|<sub></sub>|
|
||||
|`av1_ccr` |AV1 Codec Configuration Record |<sub></sub>|
|
||||
|`av1_frame` |AV1 frame |<sub>`av1_obu`</sub>|
|
||||
|`av1_obu` |AV1 Open Bitstream Unit |<sub></sub>|
|
||||
|`avc_annexb` |H.264/AVC Annex B |<sub>`avc_nalu`</sub>|
|
||||
|[`avc_au`](#avc_au) |H.264/AVC Access Unit |<sub>`avc_nalu`</sub>|
|
||||
|`avc_dcr` |H.264/AVC Decoder Configuration Record |<sub>`avc_nalu`</sub>|
|
||||
|`avc_nalu` |H.264/AVC Network Access Layer Unit |<sub>`avc_sps` `avc_pps` `avc_sei`</sub>|
|
||||
|`avc_pps` |H.264/AVC Picture Parameter Set |<sub></sub>|
|
||||
|`avc_sei` |H.264/AVC Supplemental Enhancement Information |<sub></sub>|
|
||||
|`avc_sps` |H.264/AVC Sequence Parameter Set |<sub></sub>|
|
||||
|[`avro_ocf`](#avro_ocf) |Avro object container file |<sub></sub>|
|
||||
|[`bencode`](#bencode) |BitTorrent bencoding |<sub></sub>|
|
||||
|`bsd_loopback_frame` |BSD loopback frame |<sub>`inet_packet`</sub>|
|
||||
|[`bson`](#bson) |Binary JSON |<sub></sub>|
|
||||
|`bzip2` |bzip2 compression |<sub>`probe`</sub>|
|
||||
|[`cbor`](#cbor) |Concise Binary Object Representation |<sub></sub>|
|
||||
|`dns` |DNS packet |<sub></sub>|
|
||||
|`dns_tcp` |DNS packet (TCP) |<sub></sub>|
|
||||
|`elf` |Executable and Linkable Format |<sub></sub>|
|
||||
|`ether8023_frame` |Ethernet 802.3 frame |<sub>`inet_packet`</sub>|
|
||||
|`exif` |Exchangeable Image File Format |<sub></sub>|
|
||||
|`flac` |Free Lossless Audio Codec file |<sub>`flac_metadatablocks` `flac_frame`</sub>|
|
||||
|[`flac_frame`](#flac_frame) |FLAC frame |<sub></sub>|
|
||||
|`flac_metadatablock` |FLAC metadatablock |<sub>`flac_streaminfo` `flac_picture` `vorbis_comment`</sub>|
|
||||
|`flac_metadatablocks` |FLAC metadatablocks |<sub>`flac_metadatablock`</sub>|
|
||||
|`flac_picture` |FLAC metadatablock picture |<sub>`image`</sub>|
|
||||
|`flac_streaminfo` |FLAC streaminfo |<sub></sub>|
|
||||
|`gif` |Graphics Interchange Format |<sub></sub>|
|
||||
|`gzip` |gzip compression |<sub>`probe`</sub>|
|
||||
|`hevc_annexb` |H.265/HEVC Annex B |<sub>`hevc_nalu`</sub>|
|
||||
|[`hevc_au`](#hevc_au) |H.265/HEVC Access Unit |<sub>`hevc_nalu`</sub>|
|
||||
|`hevc_dcr` |H.265/HEVC Decoder Configuration Record |<sub>`hevc_nalu`</sub>|
|
||||
|`hevc_nalu` |H.265/HEVC Network Access Layer Unit |<sub>`hevc_vps` `hevc_pps` `hevc_sps`</sub>|
|
||||
|`hevc_pps` |H.265/HEVC Picture Parameter Set |<sub></sub>|
|
||||
|`hevc_sps` |H.265/HEVC Sequence Parameter Set |<sub></sub>|
|
||||
|`hevc_vps` |H.265/HEVC Video Parameter Set |<sub></sub>|
|
||||
|`icc_profile` |International Color Consortium profile |<sub></sub>|
|
||||
|`icmp` |Internet Control Message Protocol |<sub></sub>|
|
||||
|`icmpv6` |Internet Control Message Protocol v6 |<sub></sub>|
|
||||
|`id3v1` |ID3v1 metadata |<sub></sub>|
|
||||
|`id3v11` |ID3v1.1 metadata |<sub></sub>|
|
||||
|`id3v2` |ID3v2 metadata |<sub>`image`</sub>|
|
||||
|`ipv4_packet` |Internet protocol v4 packet |<sub>`ip_packet`</sub>|
|
||||
|`ipv6_packet` |Internet protocol v6 packet |<sub>`ip_packet`</sub>|
|
||||
|`jpeg` |Joint Photographic Experts Group file |<sub>`exif` `icc_profile`</sub>|
|
||||
|`json` |JSON |<sub></sub>|
|
||||
|[`macho`](#macho) |Mach-O macOS executable |<sub></sub>|
|
||||
|[`matroska`](#matroska) |Matroska file |<sub>`aac_frame` `av1_ccr` `av1_frame` `avc_au` `avc_dcr` `flac_frame` `flac_metadatablocks` `hevc_au` `hevc_dcr` `image` `mp3_frame` `mpeg_asc` `mpeg_pes_packet` `mpeg_spu` `opus_packet` `vorbis_packet` `vp8_frame` `vp9_cfm` `vp9_frame`</sub>|
|
||||
|[`mp3`](#mp3) |MP3 file |<sub>`id3v2` `id3v1` `id3v11` `apev2` `mp3_frame`</sub>|
|
||||
|`mp3_frame` |MPEG audio layer 3 frame |<sub>`xing`</sub>|
|
||||
|[`mp4`](#mp4) |ISOBMFF MPEG-4 part 12 and similar |<sub>`aac_frame` `av1_ccr` `av1_frame` `flac_frame` `flac_metadatablocks` `id3v2` `image` `jpeg` `mp3_frame` `avc_au` `avc_dcr` `mpeg_es` `hevc_au` `hevc_dcr` `mpeg_pes_packet` `opus_packet` `protobuf_widevine` `pssh_playready` `vorbis_packet` `vp9_frame` `vpx_ccr` `icc_profile`</sub>|
|
||||
|`mpeg_asc` |MPEG-4 Audio Specific Config |<sub></sub>|
|
||||
|`mpeg_es` |MPEG Elementary Stream |<sub>`mpeg_asc` `vorbis_packet`</sub>|
|
||||
|`mpeg_pes` |MPEG Packetized elementary stream |<sub>`mpeg_pes_packet` `mpeg_spu`</sub>|
|
||||
|`mpeg_pes_packet` |MPEG Packetized elementary stream packet |<sub></sub>|
|
||||
|`mpeg_spu` |Sub Picture Unit (DVD subtitle) |<sub></sub>|
|
||||
|`mpeg_ts` |MPEG Transport Stream |<sub></sub>|
|
||||
|[`msgpack`](#msgpack) |MessagePack |<sub></sub>|
|
||||
|`ogg` |OGG file |<sub>`ogg_page` `vorbis_packet` `opus_packet` `flac_metadatablock` `flac_frame`</sub>|
|
||||
|`ogg_page` |OGG page |<sub></sub>|
|
||||
|`opus_packet` |Opus packet |<sub>`vorbis_comment`</sub>|
|
||||
|`pcap` |PCAP packet capture |<sub>`link_frame` `tcp_stream` `ipv4_packet`</sub>|
|
||||
|`pcapng` |PCAPNG packet capture |<sub>`link_frame` `tcp_stream` `ipv4_packet`</sub>|
|
||||
|`png` |Portable Network Graphics file |<sub>`icc_profile` `exif`</sub>|
|
||||
|[`protobuf`](#protobuf) |Protobuf |<sub></sub>|
|
||||
|`protobuf_widevine` |Widevine protobuf |<sub>`protobuf`</sub>|
|
||||
|`pssh_playready` |PlayReady PSSH |<sub></sub>|
|
||||
|`raw` |Raw bits |<sub></sub>|
|
||||
|[`rtmp`](#rtmp) |Real-Time Messaging Protocol |<sub>`amf0` `mpeg_asc`</sub>|
|
||||
|`sll2_packet` |Linux cooked capture encapsulation v2 |<sub>`inet_packet`</sub>|
|
||||
|`sll_packet` |Linux cooked capture encapsulation |<sub>`inet_packet`</sub>|
|
||||
|`tar` |Tar archive |<sub>`probe`</sub>|
|
||||
|`tcp_segment` |Transmission control protocol segment |<sub></sub>|
|
||||
|`tiff` |Tag Image File Format |<sub>`icc_profile`</sub>|
|
||||
|`udp_datagram` |User datagram protocol |<sub>`udp_payload`</sub>|
|
||||
|`vorbis_comment` |Vorbis comment |<sub>`flac_picture`</sub>|
|
||||
|`vorbis_packet` |Vorbis packet |<sub>`vorbis_comment`</sub>|
|
||||
|`vp8_frame` |VP8 frame |<sub></sub>|
|
||||
|`vp9_cfm` |VP9 Codec Feature Metadata |<sub></sub>|
|
||||
|`vp9_frame` |VP9 frame |<sub></sub>|
|
||||
|`vpx_ccr` |VPX Codec Configuration Record |<sub></sub>|
|
||||
|`wav` |WAV file |<sub>`id3v2` `id3v1` `id3v11`</sub>|
|
||||
|`webp` |WebP image |<sub>`vp8_frame`</sub>|
|
||||
|`xing` |Xing header |<sub></sub>|
|
||||
|`zip` |ZIP archive |<sub>`probe`</sub>|
|
||||
|`image` |Group |<sub>`gif` `jpeg` `mp4` `png` `tiff` `webp`</sub>|
|
||||
|`inet_packet` |Group |<sub>`ipv4_packet` `ipv6_packet`</sub>|
|
||||
|`ip_packet` |Group |<sub>`icmp` `icmpv6` `tcp_segment` `udp_datagram`</sub>|
|
||||
|`link_frame` |Group |<sub>`bsd_loopback_frame` `ether8023_frame` `sll2_packet` `sll_packet`</sub>|
|
||||
|`probe` |Group |<sub>`adts` `ar` `avro_ocf` `bzip2` `elf` `flac` `gif` `gzip` `jpeg` `json` `macho` `matroska` `mp3` `mp4` `mpeg_ts` `ogg` `pcap` `pcapng` `png` `tar` `tiff` `wav` `webp` `zip`</sub>|
|
||||
|`tcp_stream` |Group |<sub>`dns` `rtmp`</sub>|
|
||||
|`udp_payload` |Group |<sub>`dns`</sub>|
|
||||
|
||||
[#]: sh-end
|
||||
|
||||
## Format options
|
||||
## Global format options
|
||||
|
||||
Currently the only option is `force` and is used to ignore some format assertion errors. It can be used as a decode option or as a CLI `-o` option:
|
||||
Currently the only global option is `force` and is used to ignore some format assertion errors. It can be used as a decode option or as a CLI `-o` option:
|
||||
|
||||
```
|
||||
fq -d mp4 -o force=true file.mp4
|
||||
@ -117,78 +117,207 @@ fq -d raw 'mp4({force: true})' file.mp4
|
||||
|
||||
## Format details
|
||||
|
||||
[./formats_collect.sh]: sh-start
|
||||
[fq -rn -L . 'include "formats"; formats_sections']: sh-start
|
||||
|
||||
### aac_frame
|
||||
|
||||
#### Options
|
||||
|
||||
|Name |Default|Description|
|
||||
|- |- |-|
|
||||
|`object_type`|1 |Audio object type|
|
||||
|
||||
#### Examples
|
||||
|
||||
Decode file using options
|
||||
```
|
||||
$ fq -d aac_frame -o object_type=1 file
|
||||
```
|
||||
|
||||
Decode value as aac_frame
|
||||
```
|
||||
... | aac_frame({object_type: 1})
|
||||
```
|
||||
|
||||
### asn1_ber
|
||||
|
||||
Supports decoding BER, CER and DER ([X.690]([X.690_1297.pdf)).
|
||||
Supports decoding BER, CER and DER (X.690).
|
||||
|
||||
- Currently no extra validation is done for CER and DER.
|
||||
- Does not support specifying a schema.
|
||||
- Supports `torepr` but without schema all sequences and sets will be arrays.
|
||||
|
||||
#### Examples
|
||||
|
||||
`frompem` and `topem` can be used to work with PEM format
|
||||
```
|
||||
fq -d asn1_ber torepr file.ber
|
||||
$ fq -d raw 'frompem | asn1_ber | d' cert.pem
|
||||
```
|
||||
|
||||
Functions `frompem` and `topem` can help working with PEM format:
|
||||
|
||||
Can be used to decode nested parts
|
||||
```
|
||||
fq -d raw 'frompem | asn1_ber | d' cert.pem
|
||||
$ fq -d asn1_ber '.constructed[1].value | asn1_ber' file.ber
|
||||
```
|
||||
|
||||
If the schema is known and not that complicated it can be reproduced:
|
||||
|
||||
If schema is known and not complicated it can be reproduced
|
||||
```
|
||||
fq -d asn1_ber 'torepr as $r | ["version", "modulus", "private_exponent", "private_exponen", "prime1", "prime2", "exponent1", "exponent2", "coefficient"] | with_entries({key: .value, value: $r[.key]})' pkcs1.der
|
||||
$ fq -d asn1_ber 'torepr as $r | ["version", "modulus", "private_exponent", "private_exponen", "prime1", "prime2", "exponent1", "exponent2", "coefficient"] | with_entries({key: .value, value: $r[.key]})' pkcs1.der
|
||||
```
|
||||
|
||||
Can be used to decode nested parts:
|
||||
|
||||
Supports `torepr`
|
||||
```
|
||||
fq -d asn1_ber '.constructed[1].value | asn1_ber' file.ber
|
||||
$ fq -d asn1_ber torepr file
|
||||
```
|
||||
|
||||
References and tools:
|
||||
Supports `torepr`
|
||||
```
|
||||
... | asn1_ber | torepr
|
||||
```
|
||||
|
||||
#### References and links
|
||||
|
||||
- https://www.itu.int/ITU-T/studygroups/com10/languages/X.690_1297.pdf
|
||||
- https://en.wikipedia.org/wiki/X.690
|
||||
- https://letsencrypt.org/docs/a-warm-welcome-to-asn1-and-der/
|
||||
- https://lapo.it/asn1js/
|
||||
|
||||
### avc_au
|
||||
|
||||
#### Options
|
||||
|
||||
|Name |Default|Description|
|
||||
|- |- |-|
|
||||
|`length_size`|4 |Length value size|
|
||||
|
||||
#### Examples
|
||||
|
||||
Decode file using options
|
||||
```
|
||||
$ fq -d avc_au -o length_size=4 file
|
||||
```
|
||||
|
||||
Decode value as avc_au
|
||||
```
|
||||
... | avc_au({length_size: 4})
|
||||
```
|
||||
|
||||
### avro_ocf
|
||||
|
||||
Supports reading Avro Object Container Format (OCF) files based on the [1.11.0 specification](https://avro.apache.org/docs/current/spec.html#Object+Container+Files).
|
||||
Supports reading Avro Object Container Format (OCF) files based on the 1.11.0 specification.
|
||||
|
||||
Capable of handling null, deflate, and snappy codecs for data compression.
|
||||
|
||||
Limitations:
|
||||
- Schema does not support self-referential types, only built-in types.
|
||||
- Decimal logical types are not supported for decoding, will just be treated as their primitive type
|
||||
### becode
|
||||
|
||||
Supports `torepr`:
|
||||
#### References and links
|
||||
|
||||
- https://avro.apache.org/docs/current/spec.html#Object+Container+Files
|
||||
|
||||
### bencode
|
||||
|
||||
#### Examples
|
||||
|
||||
bencode as JSON
|
||||
```
|
||||
fq -d bencode torepr file.torrent
|
||||
$ fq -d bencode torepr file
|
||||
```
|
||||
|
||||
Supports `torepr`
|
||||
```
|
||||
$ fq -d bencode torepr file
|
||||
```
|
||||
|
||||
Supports `torepr`
|
||||
```
|
||||
... | bencode | torepr
|
||||
```
|
||||
|
||||
#### References and links
|
||||
|
||||
- https://bsonspec.org/spec.html
|
||||
|
||||
### bson
|
||||
|
||||
Supports `torepr`:
|
||||
#### Examples
|
||||
|
||||
BSON as JSON
|
||||
```
|
||||
fq -d bson torepr file.bson
|
||||
$ fq -d bson torepr file
|
||||
```
|
||||
|
||||
Supports `torepr`
|
||||
```
|
||||
$ fq -d bson torepr file
|
||||
```
|
||||
|
||||
Supports `torepr`
|
||||
```
|
||||
... | bson | torepr
|
||||
```
|
||||
|
||||
#### References and links
|
||||
|
||||
- https://wiki.theory.org/BitTorrentSpecification#Bencoding
|
||||
|
||||
### cbor
|
||||
|
||||
Supports `torepr`:
|
||||
#### Examples
|
||||
|
||||
Supports `torepr`
|
||||
```
|
||||
fq -d cbor torepr file.cbor
|
||||
fq -d cbor 'torepr.field' file.cbor
|
||||
fq -d cbor 'torepr | .field' file.cbor
|
||||
fq -d cbor 'torepr | grep("abc")' file.cbor
|
||||
$ fq -d cbor torepr file
|
||||
```
|
||||
|
||||
Supports `torepr`
|
||||
```
|
||||
... | cbor | torepr
|
||||
```
|
||||
|
||||
#### References and links
|
||||
|
||||
- https://en.wikipedia.org/wiki/CBOR
|
||||
- https://www.rfc-editor.org/rfc/rfc8949.html
|
||||
|
||||
### flac_frame
|
||||
|
||||
#### Options
|
||||
|
||||
|Name |Default|Description|
|
||||
|- |- |-|
|
||||
|`bits_per_sample`|16 |Bits per sample|
|
||||
|
||||
#### Examples
|
||||
|
||||
Decode file using options
|
||||
```
|
||||
$ fq -d flac_frame -o bits_per_sample=16 file
|
||||
```
|
||||
|
||||
Decode value as flac_frame
|
||||
```
|
||||
... | flac_frame({bits_per_sample: 16})
|
||||
```
|
||||
|
||||
### hevc_au
|
||||
|
||||
#### Options
|
||||
|
||||
|Name |Default|Description|
|
||||
|- |- |-|
|
||||
|`length_size`|4 |Length value size|
|
||||
|
||||
#### Examples
|
||||
|
||||
Decode file using options
|
||||
```
|
||||
$ fq -d hevc_au -o length_size=4 file
|
||||
```
|
||||
|
||||
Decode value as hevc_au
|
||||
```
|
||||
... | hevc_au({length_size: 4})
|
||||
```
|
||||
|
||||
### macho
|
||||
@ -197,85 +326,138 @@ Supports decoding vanilla and FAT Mach-O binaries.
|
||||
|
||||
#### Examples
|
||||
|
||||
To decode the macOS build of `fq`:
|
||||
|
||||
Select 64bit load segments
|
||||
```
|
||||
fq . /path/to/fq
|
||||
$ fq '.load_commands[] | select(.cmd=="segment_64")' file
|
||||
```
|
||||
|
||||
```
|
||||
fq '.load_commands[] | select(.cmd=="segment_64")' /path/to/fq
|
||||
```
|
||||
#### References and links
|
||||
|
||||
Note you can use `-d macho` to decode a broken Mach-O binary.
|
||||
|
||||
#### References:
|
||||
- https://github.com/aidansteele/osx-abi-macho-file-format-reference
|
||||
|
||||
### matroska
|
||||
|
||||
Supports `matroska_path`:
|
||||
#### Examples
|
||||
|
||||
Lookup element decode value using `matroska_path`
|
||||
```
|
||||
$ fq 'matroska_path(".Segment.Tracks[0]")' file.mkv
|
||||
│00 01 02 03 04 05 06 07 08 09│0123456789│.elements[1].elements[3]{}:
|
||||
0x122│ 16 54 ae 6b │ .T.k │ id: "Tracks" (0x1654ae6b) (A Top-Level Element of information with many tracks described.)
|
||||
│ │ │ type: "master" (7)
|
||||
0x122│ 4d bf │ M. │ size: 3519
|
||||
0x122│ bf│ .│ elements[0:3]:
|
||||
0x12c│84 cf 8b db a0 ae 01 00 00 00│..........│
|
||||
0x136│00 00 00 78 d7 81 01 73 c5 88│...x...s..│
|
||||
* │until 0xee9.7 (3519) │ │
|
||||
... | matroska_path(".Segment.Tracks[0)"
|
||||
```
|
||||
|
||||
Return `matroska_path` string for a box decode value
|
||||
```
|
||||
$ fq 'first(grep_by(.id == "Tracks")) | matroska_path' test.mkv
|
||||
".Segment.Tracks"
|
||||
... | grep_by(.id == "Tracks") | matroska_path
|
||||
```
|
||||
|
||||
#### References and links
|
||||
|
||||
- https://tools.ietf.org/html/draft-ietf-cellar-ebml-00
|
||||
- https://matroska.org/technical/specs/index.html
|
||||
- https://www.matroska.org/technical/basics.html
|
||||
- https://www.matroska.org/technical/codec_specs.html
|
||||
- https://wiki.xiph.org/MatroskaOpus
|
||||
|
||||
### mp3
|
||||
|
||||
#### Options
|
||||
|
||||
|Name |Default|Description|
|
||||
|- |- |-|
|
||||
|`max_sync_seek` |32768 |Max byte distance to next sync|
|
||||
|`max_unique_header_configs`|5 |Max number of unique frame header configs allowed|
|
||||
|
||||
#### Examples
|
||||
|
||||
Decode file using options
|
||||
```
|
||||
$ fq -d mp3 -o max_sync_seek=32768 -o max_unique_header_configs=5 file
|
||||
```
|
||||
|
||||
Decode value as mp3
|
||||
```
|
||||
... | mp3({max_sync_seek: 32768, max_unique_header_configs: 5})
|
||||
```
|
||||
|
||||
### mp4
|
||||
|
||||
Supports `mp4_path`:
|
||||
Support `mp4_path`
|
||||
|
||||
#### Options
|
||||
|
||||
|Name |Default|Description|
|
||||
|- |- |-|
|
||||
|`allow_truncated`|false |Allow box to be truncated|
|
||||
|`decode_samples` |true |Decode supported media samples|
|
||||
|
||||
#### Examples
|
||||
|
||||
Lookup box decode value using `mp4_path`
|
||||
```
|
||||
$ fq 'mp4_path(".moov.trak[1]")' file.mp4
|
||||
│00 01 02 03 04 05 06 07 08 09│0123456789│.boxes[3].boxes[1]{}:
|
||||
0x4f6│ 00 00 02│ ...│ size: 573
|
||||
0x500│3d │= │
|
||||
0x500│ 74 72 61 6b │ trak │ type: "trak" (Container for an individual track or stream)
|
||||
0x500│ 00 00 00 5c 74│ ...\t│ boxes[0:3]:
|
||||
0x50a│6b 68 64 00 00 00 03 00 00 00│khd.......│
|
||||
0x514│00 00 00 00 00 00 00 00 01 00│..........│
|
||||
* │until 0x739.7 (565) │ │
|
||||
... | mp4_path(".moov.trak[1]")
|
||||
```
|
||||
|
||||
Return `mp4_path` string for a box decode value
|
||||
```
|
||||
$ fq 'first(grep_by(.type == "trak")) | mp4_path' file.mp4
|
||||
".moov.trak"
|
||||
... | grep_by(.type == "trak") | mp4_path
|
||||
```
|
||||
|
||||
Decode file using options
|
||||
```
|
||||
$ fq -d mp4 -o allow_truncated=false -o decode_samples=true file
|
||||
```
|
||||
|
||||
Decode value as mp4
|
||||
```
|
||||
... | mp4({allow_truncated: false, decode_samples: true})
|
||||
```
|
||||
|
||||
#### References and links
|
||||
|
||||
- [ISO/IEC base media file format (MPEG-4 Part 12)](https://en.wikipedia.org/wiki/ISO/IEC_base_media_file_format)
|
||||
- [Quicktime file format](https://developer.apple.com/standards/qtff-2001.pdf)
|
||||
|
||||
### msgpack
|
||||
|
||||
Supports `torepr`:
|
||||
#### Examples
|
||||
|
||||
Supports `torepr`
|
||||
```
|
||||
fq -d msgpack torepr file.msgpack
|
||||
$ fq -d msgpack torepr file
|
||||
```
|
||||
|
||||
Supports `torepr`
|
||||
```
|
||||
... | msgpack | torepr
|
||||
```
|
||||
|
||||
#### References and links
|
||||
|
||||
- https://github.com/msgpack/msgpack/blob/master/spec.md
|
||||
|
||||
### protobuf
|
||||
|
||||
`protobuf` decoder can be used to decode sub messages:
|
||||
#### Examples
|
||||
|
||||
Can be used to decode sub messages
|
||||
```
|
||||
fq -d protobuf '.fields[6].wire_value | protobuf | d'
|
||||
$ fq -d protobuf '.fields[6].wire_value | protobuf | d'
|
||||
```
|
||||
|
||||
#### References and links
|
||||
|
||||
- https://developers.google.com/protocol-buffers/docs/encoding
|
||||
|
||||
### rtmp
|
||||
|
||||
Current only supports plain RTMP (not RTMPT or encrypted variants etc) with AMF0 (not AMF3).
|
||||
|
||||
[#]: sh-end
|
||||
#### References and links
|
||||
|
||||
- https://rtmp.veriskope.com/docs/spec/
|
||||
- https://rtmp.veriskope.com/pdf/video_file_format_spec_v10.pdf
|
||||
|
||||
|
||||
[#]: sh-end
|
||||
|
||||
## Dependency graph
|
||||
|
||||
|
||||
@ -1,9 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
for i in $(cd "$REPODIR" && ls -1 format/*/*.md | sort -t / -k 3); do
|
||||
FORMAT=$(echo "$i" | sed 's#format/.*/\(.*\).md#\1#')
|
||||
echo "### $FORMAT"
|
||||
echo
|
||||
cat "$REPODIR/$i"
|
||||
echo
|
||||
done
|
||||
@ -1,7 +0,0 @@
|
||||
#!/usr/bin/env fq -rnf
|
||||
|
||||
[ (formats | keys[]) as $format
|
||||
| if ($doc_formats | indices($format)) != [] then "[\($format)](doc/formats.md#\($format))"
|
||||
else $format
|
||||
end
|
||||
] | join(",\n")
|
||||
@ -1,4 +0,0 @@
|
||||
#!/bin/sh
|
||||
# what formats has a .md file
|
||||
DOC_FORMATS=$(echo $(ls -1 $REPODIR/format/*/*.md | sed "s#$REPODIR/format/.*/\(.*\).md#\1#"))
|
||||
./formats_list.jq --arg doc_formats "$DOC_FORMATS"
|
||||
@ -1,59 +0,0 @@
|
||||
#!/usr/bin/env fq -rnf
|
||||
|
||||
def code: "`\(.)`";
|
||||
def nbsp: gsub(" "; " ");
|
||||
|
||||
def format_table:
|
||||
( ($doc_formats | split(" ")) as $doc_formats
|
||||
| [ {
|
||||
name: "Name",
|
||||
desc: "Description",
|
||||
uses: "Dependencies"
|
||||
},
|
||||
{
|
||||
name: "-",
|
||||
desc: "-",
|
||||
uses: "-"
|
||||
},
|
||||
( formats
|
||||
| to_entries[]
|
||||
| {
|
||||
name:
|
||||
( ( .key as $format
|
||||
| if ($doc_formats | indices($format)) != [] then "[\($format | code)](#\($format))"
|
||||
else $format | code
|
||||
end
|
||||
)
|
||||
+ " "
|
||||
),
|
||||
desc: ((.value.description | nbsp) + " "),
|
||||
uses: "<sub>\((((.value.dependencies | flatten | map(code)) | join(" "))? // ""))</sub>"
|
||||
}
|
||||
),
|
||||
( [ formats
|
||||
| to_entries[]
|
||||
| . as $e
|
||||
| select(.value.groups)
|
||||
| .value.groups[] | {key: ., value: $e.key}
|
||||
]
|
||||
| reduce .[] as $e ({}; .[$e.key] += [$e.value])
|
||||
| to_entries[]
|
||||
| {
|
||||
name: ((.key | code) + " "),
|
||||
desc: "Group",
|
||||
uses: "<sub>\(((.value | map(code)) | join(" ")))</sub>"
|
||||
}
|
||||
)
|
||||
]
|
||||
| table(
|
||||
[.name, .desc, .uses];
|
||||
[ ""
|
||||
, (.[0] | . as $rc | $rc.string | rpad(" "; $rc.maxwidth))
|
||||
, (.[1] | . as $rc | $rc.string | rpad(" "; $rc.maxwidth))
|
||||
, .[2].string
|
||||
, ""
|
||||
] | join("|")
|
||||
)
|
||||
);
|
||||
|
||||
format_table
|
||||
@ -1,4 +0,0 @@
|
||||
#!/bin/sh
|
||||
# what formats has a .md file
|
||||
DOC_FORMATS=$(echo $(ls -1 $REPODIR/format/*/*.md | sed "s#$REPODIR/format/.*/\(.*\).md#\1#"))
|
||||
./formats_table.jq --arg doc_formats "$DOC_FORMATS"
|
||||
772
format/all/help.fqtest
Normal file
772
format/all/help.fqtest
Normal file
@ -0,0 +1,772 @@
|
||||
$ fq -n '_registry.formats | keys[] | "help(\(.))", ([_help("fq"; .)] | join("\n") | split("\n")[] | ("out \(.)" | println))'
|
||||
"help(aac_frame)"
|
||||
out aac_frame: Advanced Audio Coding frame decoder
|
||||
out Options:
|
||||
out object_type=1 Audio object type
|
||||
out Examples:
|
||||
out # Decode file as aac_frame
|
||||
out $ fq -d aac_frame file
|
||||
out # Decode value as aac_frame
|
||||
out ... | aac_frame
|
||||
out # Decode file using options
|
||||
out $ fq -d aac_frame -o object_type=1 file
|
||||
out # Decode value as aac_frame
|
||||
out ... | aac_frame({object_type: 1})
|
||||
"help(adts)"
|
||||
out adts: Audio Data Transport Stream decoder
|
||||
out Examples:
|
||||
out # Decode file as adts
|
||||
out $ fq -d adts file
|
||||
out # Decode value as adts
|
||||
out ... | adts
|
||||
"help(adts_frame)"
|
||||
out adts_frame: Audio Data Transport Stream frame decoder
|
||||
out Examples:
|
||||
out # Decode file as adts_frame
|
||||
out $ fq -d adts_frame file
|
||||
out # Decode value as adts_frame
|
||||
out ... | adts_frame
|
||||
"help(amf0)"
|
||||
out amf0: Action Message Format 0 decoder
|
||||
out Examples:
|
||||
out # Decode file as amf0
|
||||
out $ fq -d amf0 file
|
||||
out # Decode value as amf0
|
||||
out ... | amf0
|
||||
"help(apev2)"
|
||||
out apev2: APEv2 metadata tag decoder
|
||||
out Examples:
|
||||
out # Decode file as apev2
|
||||
out $ fq -d apev2 file
|
||||
out # Decode value as apev2
|
||||
out ... | apev2
|
||||
"help(ar)"
|
||||
out ar: Unix archive decoder
|
||||
out Examples:
|
||||
out # Decode file as ar
|
||||
out $ fq -d ar file
|
||||
out # Decode value as ar
|
||||
out ... | ar
|
||||
"help(asn1_ber)"
|
||||
out asn1_ber: ASN1 BER (basic encoding rules, also CER and DER) decoder
|
||||
out Supports decoding BER, CER and DER (X.690).
|
||||
out
|
||||
out - Currently no extra validation is done for CER and DER.
|
||||
out - Does not support specifying a schema.
|
||||
out - Supports torepr but without schema all sequences and sets will be arrays.
|
||||
out Examples:
|
||||
out # frompem` and `topem can be used to work with PEM format
|
||||
out $ fq -d raw 'frompem | asn1_ber | d' cert.pem
|
||||
out # Can be used to decode nested parts
|
||||
out $ fq -d asn1_ber '.constructed[1].value | asn1_ber' file.ber
|
||||
out # If schema is known and not complicated it can be reproduced
|
||||
out $ fq -d asn1_ber 'torepr as $r | ["version", "modulus", "private_exponent", "private_exponen", "prime1", "prime2", "exponent1", "exponent2", "coefficient"] | with_entries({key: .value, value: $r[.key]})' pkcs1.der
|
||||
out # Decode file as asn1_ber
|
||||
out $ fq -d asn1_ber file
|
||||
out # Decode value as asn1_ber
|
||||
out ... | asn1_ber
|
||||
out # Supports torepr
|
||||
out $ fq -d asn1_ber torepr file
|
||||
out # Supports torepr
|
||||
out ... | asn1_ber | torepr
|
||||
out References and links
|
||||
out https://www.itu.int/ITU-T/studygroups/com10/languages/X.690_1297.pdf
|
||||
out https://en.wikipedia.org/wiki/X.690
|
||||
out https://letsencrypt.org/docs/a-warm-welcome-to-asn1-and-der/
|
||||
out https://lapo.it/asn1js/
|
||||
"help(av1_ccr)"
|
||||
out av1_ccr: AV1 Codec Configuration Record decoder
|
||||
out Examples:
|
||||
out # Decode file as av1_ccr
|
||||
out $ fq -d av1_ccr file
|
||||
out # Decode value as av1_ccr
|
||||
out ... | av1_ccr
|
||||
"help(av1_frame)"
|
||||
out av1_frame: AV1 frame decoder
|
||||
out Examples:
|
||||
out # Decode file as av1_frame
|
||||
out $ fq -d av1_frame file
|
||||
out # Decode value as av1_frame
|
||||
out ... | av1_frame
|
||||
"help(av1_obu)"
|
||||
out av1_obu: AV1 Open Bitstream Unit decoder
|
||||
out Examples:
|
||||
out # Decode file as av1_obu
|
||||
out $ fq -d av1_obu file
|
||||
out # Decode value as av1_obu
|
||||
out ... | av1_obu
|
||||
"help(avc_annexb)"
|
||||
out avc_annexb: H.264/AVC Annex B decoder
|
||||
out Examples:
|
||||
out # Decode file as avc_annexb
|
||||
out $ fq -d avc_annexb file
|
||||
out # Decode value as avc_annexb
|
||||
out ... | avc_annexb
|
||||
"help(avc_au)"
|
||||
out avc_au: H.264/AVC Access Unit decoder
|
||||
out Options:
|
||||
out length_size=4 Length value size
|
||||
out Examples:
|
||||
out # Decode file as avc_au
|
||||
out $ fq -d avc_au file
|
||||
out # Decode value as avc_au
|
||||
out ... | avc_au
|
||||
out # Decode file using options
|
||||
out $ fq -d avc_au -o length_size=4 file
|
||||
out # Decode value as avc_au
|
||||
out ... | avc_au({length_size: 4})
|
||||
"help(avc_dcr)"
|
||||
out avc_dcr: H.264/AVC Decoder Configuration Record decoder
|
||||
out Examples:
|
||||
out # Decode file as avc_dcr
|
||||
out $ fq -d avc_dcr file
|
||||
out # Decode value as avc_dcr
|
||||
out ... | avc_dcr
|
||||
"help(avc_nalu)"
|
||||
out avc_nalu: H.264/AVC Network Access Layer Unit decoder
|
||||
out Examples:
|
||||
out # Decode file as avc_nalu
|
||||
out $ fq -d avc_nalu file
|
||||
out # Decode value as avc_nalu
|
||||
out ... | avc_nalu
|
||||
"help(avc_pps)"
|
||||
out avc_pps: H.264/AVC Picture Parameter Set decoder
|
||||
out Examples:
|
||||
out # Decode file as avc_pps
|
||||
out $ fq -d avc_pps file
|
||||
out # Decode value as avc_pps
|
||||
out ... | avc_pps
|
||||
"help(avc_sei)"
|
||||
out avc_sei: H.264/AVC Supplemental Enhancement Information decoder
|
||||
out Examples:
|
||||
out # Decode file as avc_sei
|
||||
out $ fq -d avc_sei file
|
||||
out # Decode value as avc_sei
|
||||
out ... | avc_sei
|
||||
"help(avc_sps)"
|
||||
out avc_sps: H.264/AVC Sequence Parameter Set decoder
|
||||
out Examples:
|
||||
out # Decode file as avc_sps
|
||||
out $ fq -d avc_sps file
|
||||
out # Decode value as avc_sps
|
||||
out ... | avc_sps
|
||||
"help(avro_ocf)"
|
||||
out avro_ocf: Avro object container file decoder
|
||||
out Supports reading Avro Object Container Format (OCF) files based on the 1.11.0 specification.
|
||||
out
|
||||
out Capable of handling null, deflate, and snappy codecs for data compression.
|
||||
out
|
||||
out Limitations:
|
||||
out - Schema does not support self-referential types, only built-in types.
|
||||
out - Decimal logical types are not supported for decoding, will just be treated as their primitive type
|
||||
out Examples:
|
||||
out # Decode file as avro_ocf
|
||||
out $ fq -d avro_ocf file
|
||||
out # Decode value as avro_ocf
|
||||
out ... | avro_ocf
|
||||
out References and links
|
||||
out https://avro.apache.org/docs/current/spec.html#Object+Container+Files
|
||||
"help(bencode)"
|
||||
out bencode: BitTorrent bencoding decoder
|
||||
out Examples:
|
||||
out # bencode as JSON
|
||||
out $ fq -d bencode torepr file
|
||||
out # Decode file as bencode
|
||||
out $ fq -d bencode file
|
||||
out # Decode value as bencode
|
||||
out ... | bencode
|
||||
out # Supports torepr
|
||||
out $ fq -d bencode torepr file
|
||||
out # Supports torepr
|
||||
out ... | bencode | torepr
|
||||
out References and links
|
||||
out https://bsonspec.org/spec.html
|
||||
"help(bsd_loopback_frame)"
|
||||
out bsd_loopback_frame: BSD loopback frame decoder
|
||||
out Examples:
|
||||
out # Decode file as bsd_loopback_frame
|
||||
out $ fq -d bsd_loopback_frame file
|
||||
out # Decode value as bsd_loopback_frame
|
||||
out ... | bsd_loopback_frame
|
||||
"help(bson)"
|
||||
out bson: Binary JSON decoder
|
||||
out Examples:
|
||||
out # BSON as JSON
|
||||
out $ fq -d bson torepr file
|
||||
out # Decode file as bson
|
||||
out $ fq -d bson file
|
||||
out # Decode value as bson
|
||||
out ... | bson
|
||||
out # Supports torepr
|
||||
out $ fq -d bson torepr file
|
||||
out # Supports torepr
|
||||
out ... | bson | torepr
|
||||
out References and links
|
||||
out https://wiki.theory.org/BitTorrentSpecification#Bencoding
|
||||
"help(bzip2)"
|
||||
out bzip2: bzip2 compression decoder
|
||||
out Examples:
|
||||
out # Decode file as bzip2
|
||||
out $ fq -d bzip2 file
|
||||
out # Decode value as bzip2
|
||||
out ... | bzip2
|
||||
"help(cbor)"
|
||||
out cbor: Concise Binary Object Representation decoder
|
||||
out Examples:
|
||||
out # Decode file as cbor
|
||||
out $ fq -d cbor file
|
||||
out # Decode value as cbor
|
||||
out ... | cbor
|
||||
out # Supports torepr
|
||||
out $ fq -d cbor torepr file
|
||||
out # Supports torepr
|
||||
out ... | cbor | torepr
|
||||
out References and links
|
||||
out https://en.wikipedia.org/wiki/CBOR
|
||||
out https://www.rfc-editor.org/rfc/rfc8949.html
|
||||
"help(dns)"
|
||||
out dns: DNS packet decoder
|
||||
out Examples:
|
||||
out # Decode file as dns
|
||||
out $ fq -d dns file
|
||||
out # Decode value as dns
|
||||
out ... | dns
|
||||
"help(dns_tcp)"
|
||||
out dns_tcp: DNS packet (TCP) decoder
|
||||
out Examples:
|
||||
out # Decode file as dns_tcp
|
||||
out $ fq -d dns_tcp file
|
||||
out # Decode value as dns_tcp
|
||||
out ... | dns_tcp
|
||||
"help(elf)"
|
||||
out elf: Executable and Linkable Format decoder
|
||||
out Examples:
|
||||
out # Decode file as elf
|
||||
out $ fq -d elf file
|
||||
out # Decode value as elf
|
||||
out ... | elf
|
||||
"help(ether8023_frame)"
|
||||
out ether8023_frame: Ethernet 802.3 frame decoder
|
||||
out Examples:
|
||||
out # Decode file as ether8023_frame
|
||||
out $ fq -d ether8023_frame file
|
||||
out # Decode value as ether8023_frame
|
||||
out ... | ether8023_frame
|
||||
"help(exif)"
|
||||
out exif: Exchangeable Image File Format decoder
|
||||
out Examples:
|
||||
out # Decode file as exif
|
||||
out $ fq -d exif file
|
||||
out # Decode value as exif
|
||||
out ... | exif
|
||||
"help(flac)"
|
||||
out flac: Free Lossless Audio Codec file decoder
|
||||
out Examples:
|
||||
out # Decode file as flac
|
||||
out $ fq -d flac file
|
||||
out # Decode value as flac
|
||||
out ... | flac
|
||||
"help(flac_frame)"
|
||||
out flac_frame: FLAC frame decoder
|
||||
out Options:
|
||||
out bits_per_sample=16 Bits per sample
|
||||
out Examples:
|
||||
out # Decode file as flac_frame
|
||||
out $ fq -d flac_frame file
|
||||
out # Decode value as flac_frame
|
||||
out ... | flac_frame
|
||||
out # Decode file using options
|
||||
out $ fq -d flac_frame -o bits_per_sample=16 file
|
||||
out # Decode value as flac_frame
|
||||
out ... | flac_frame({bits_per_sample: 16})
|
||||
"help(flac_metadatablock)"
|
||||
out flac_metadatablock: FLAC metadatablock decoder
|
||||
out Examples:
|
||||
out # Decode file as flac_metadatablock
|
||||
out $ fq -d flac_metadatablock file
|
||||
out # Decode value as flac_metadatablock
|
||||
out ... | flac_metadatablock
|
||||
"help(flac_metadatablocks)"
|
||||
out flac_metadatablocks: FLAC metadatablocks decoder
|
||||
out Examples:
|
||||
out # Decode file as flac_metadatablocks
|
||||
out $ fq -d flac_metadatablocks file
|
||||
out # Decode value as flac_metadatablocks
|
||||
out ... | flac_metadatablocks
|
||||
"help(flac_picture)"
|
||||
out flac_picture: FLAC metadatablock picture decoder
|
||||
out Examples:
|
||||
out # Decode file as flac_picture
|
||||
out $ fq -d flac_picture file
|
||||
out # Decode value as flac_picture
|
||||
out ... | flac_picture
|
||||
"help(flac_streaminfo)"
|
||||
out flac_streaminfo: FLAC streaminfo decoder
|
||||
out Examples:
|
||||
out # Decode file as flac_streaminfo
|
||||
out $ fq -d flac_streaminfo file
|
||||
out # Decode value as flac_streaminfo
|
||||
out ... | flac_streaminfo
|
||||
"help(gif)"
|
||||
out gif: Graphics Interchange Format decoder
|
||||
out Examples:
|
||||
out # Decode file as gif
|
||||
out $ fq -d gif file
|
||||
out # Decode value as gif
|
||||
out ... | gif
|
||||
"help(gzip)"
|
||||
out gzip: gzip compression decoder
|
||||
out Examples:
|
||||
out # Decode file as gzip
|
||||
out $ fq -d gzip file
|
||||
out # Decode value as gzip
|
||||
out ... | gzip
|
||||
"help(hevc_annexb)"
|
||||
out hevc_annexb: H.265/HEVC Annex B decoder
|
||||
out Examples:
|
||||
out # Decode file as hevc_annexb
|
||||
out $ fq -d hevc_annexb file
|
||||
out # Decode value as hevc_annexb
|
||||
out ... | hevc_annexb
|
||||
"help(hevc_au)"
|
||||
out hevc_au: H.265/HEVC Access Unit decoder
|
||||
out Options:
|
||||
out length_size=4 Length value size
|
||||
out Examples:
|
||||
out # Decode file as hevc_au
|
||||
out $ fq -d hevc_au file
|
||||
out # Decode value as hevc_au
|
||||
out ... | hevc_au
|
||||
out # Decode file using options
|
||||
out $ fq -d hevc_au -o length_size=4 file
|
||||
out # Decode value as hevc_au
|
||||
out ... | hevc_au({length_size: 4})
|
||||
"help(hevc_dcr)"
|
||||
out hevc_dcr: H.265/HEVC Decoder Configuration Record decoder
|
||||
out Examples:
|
||||
out # Decode file as hevc_dcr
|
||||
out $ fq -d hevc_dcr file
|
||||
out # Decode value as hevc_dcr
|
||||
out ... | hevc_dcr
|
||||
"help(hevc_nalu)"
|
||||
out hevc_nalu: H.265/HEVC Network Access Layer Unit decoder
|
||||
out Examples:
|
||||
out # Decode file as hevc_nalu
|
||||
out $ fq -d hevc_nalu file
|
||||
out # Decode value as hevc_nalu
|
||||
out ... | hevc_nalu
|
||||
"help(hevc_pps)"
|
||||
out hevc_pps: H.265/HEVC Picture Parameter Set decoder
|
||||
out Examples:
|
||||
out # Decode file as hevc_pps
|
||||
out $ fq -d hevc_pps file
|
||||
out # Decode value as hevc_pps
|
||||
out ... | hevc_pps
|
||||
"help(hevc_sps)"
|
||||
out hevc_sps: H.265/HEVC Sequence Parameter Set decoder
|
||||
out Examples:
|
||||
out # Decode file as hevc_sps
|
||||
out $ fq -d hevc_sps file
|
||||
out # Decode value as hevc_sps
|
||||
out ... | hevc_sps
|
||||
"help(hevc_vps)"
|
||||
out hevc_vps: H.265/HEVC Video Parameter Set decoder
|
||||
out Examples:
|
||||
out # Decode file as hevc_vps
|
||||
out $ fq -d hevc_vps file
|
||||
out # Decode value as hevc_vps
|
||||
out ... | hevc_vps
|
||||
"help(icc_profile)"
|
||||
out icc_profile: International Color Consortium profile decoder
|
||||
out Examples:
|
||||
out # Decode file as icc_profile
|
||||
out $ fq -d icc_profile file
|
||||
out # Decode value as icc_profile
|
||||
out ... | icc_profile
|
||||
"help(icmp)"
|
||||
out icmp: Internet Control Message Protocol decoder
|
||||
out Examples:
|
||||
out # Decode file as icmp
|
||||
out $ fq -d icmp file
|
||||
out # Decode value as icmp
|
||||
out ... | icmp
|
||||
"help(icmpv6)"
|
||||
out icmpv6: Internet Control Message Protocol v6 decoder
|
||||
out Examples:
|
||||
out # Decode file as icmpv6
|
||||
out $ fq -d icmpv6 file
|
||||
out # Decode value as icmpv6
|
||||
out ... | icmpv6
|
||||
"help(id3v1)"
|
||||
out id3v1: ID3v1 metadata decoder
|
||||
out Examples:
|
||||
out # Decode file as id3v1
|
||||
out $ fq -d id3v1 file
|
||||
out # Decode value as id3v1
|
||||
out ... | id3v1
|
||||
"help(id3v11)"
|
||||
out id3v11: ID3v1.1 metadata decoder
|
||||
out Examples:
|
||||
out # Decode file as id3v11
|
||||
out $ fq -d id3v11 file
|
||||
out # Decode value as id3v11
|
||||
out ... | id3v11
|
||||
"help(id3v2)"
|
||||
out id3v2: ID3v2 metadata decoder
|
||||
out Examples:
|
||||
out # Decode file as id3v2
|
||||
out $ fq -d id3v2 file
|
||||
out # Decode value as id3v2
|
||||
out ... | id3v2
|
||||
"help(ipv4_packet)"
|
||||
out ipv4_packet: Internet protocol v4 packet decoder
|
||||
out Examples:
|
||||
out # Decode file as ipv4_packet
|
||||
out $ fq -d ipv4_packet file
|
||||
out # Decode value as ipv4_packet
|
||||
out ... | ipv4_packet
|
||||
"help(ipv6_packet)"
|
||||
out ipv6_packet: Internet protocol v6 packet decoder
|
||||
out Examples:
|
||||
out # Decode file as ipv6_packet
|
||||
out $ fq -d ipv6_packet file
|
||||
out # Decode value as ipv6_packet
|
||||
out ... | ipv6_packet
|
||||
"help(jpeg)"
|
||||
out jpeg: Joint Photographic Experts Group file decoder
|
||||
out Examples:
|
||||
out # Decode file as jpeg
|
||||
out $ fq -d jpeg file
|
||||
out # Decode value as jpeg
|
||||
out ... | jpeg
|
||||
"help(json)"
|
||||
out json: JSON decoder
|
||||
out Examples:
|
||||
out # Decode file as json
|
||||
out $ fq -d json file
|
||||
out # Decode value as json
|
||||
out ... | json
|
||||
"help(macho)"
|
||||
out macho: Mach-O macOS executable decoder
|
||||
out Supports decoding vanilla and FAT Mach-O binaries.
|
||||
out Examples:
|
||||
out # Select 64bit load segments
|
||||
out $ fq '.load_commands[] | select(.cmd=="segment_64")' file
|
||||
out # Decode file as macho
|
||||
out $ fq -d macho file
|
||||
out # Decode value as macho
|
||||
out ... | macho
|
||||
out References and links
|
||||
out https://github.com/aidansteele/osx-abi-macho-file-format-reference
|
||||
"help(matroska)"
|
||||
out matroska: Matroska file decoder
|
||||
out Examples:
|
||||
out # Lookup element decode value using matroska_path
|
||||
out ... | matroska_path(".Segment.Tracks[0)"
|
||||
out # Return matroska_path string for a box decode value
|
||||
out ... | grep_by(.id == "Tracks") | matroska_path
|
||||
out # Decode file as matroska
|
||||
out $ fq -d matroska file
|
||||
out # Decode value as matroska
|
||||
out ... | matroska
|
||||
out References and links
|
||||
out https://tools.ietf.org/html/draft-ietf-cellar-ebml-00
|
||||
out https://matroska.org/technical/specs/index.html
|
||||
out https://www.matroska.org/technical/basics.html
|
||||
out https://www.matroska.org/technical/codec_specs.html
|
||||
out https://wiki.xiph.org/MatroskaOpus
|
||||
"help(mp3)"
|
||||
out mp3: MP3 file decoder
|
||||
out Options:
|
||||
out max_sync_seek=32768 Max byte distance to next sync
|
||||
out max_unique_header_configs=5 Max number of unique frame header configs allowed
|
||||
out Examples:
|
||||
out # Decode file as mp3
|
||||
out $ fq -d mp3 file
|
||||
out # Decode value as mp3
|
||||
out ... | mp3
|
||||
out # Decode file using options
|
||||
out $ fq -d mp3 -o max_sync_seek=32768 -o max_unique_header_configs=5 file
|
||||
out # Decode value as mp3
|
||||
out ... | mp3({max_sync_seek: 32768, max_unique_header_configs: 5})
|
||||
"help(mp3_frame)"
|
||||
out mp3_frame: MPEG audio layer 3 frame decoder
|
||||
out Examples:
|
||||
out # Decode file as mp3_frame
|
||||
out $ fq -d mp3_frame file
|
||||
out # Decode value as mp3_frame
|
||||
out ... | mp3_frame
|
||||
"help(mp4)"
|
||||
out mp4: ISOBMFF MPEG-4 part 12 and similar decoder
|
||||
out Support mp4_path
|
||||
out Options:
|
||||
out allow_truncated=false Allow box to be truncated
|
||||
out decode_samples=true Decode supported media samples
|
||||
out Examples:
|
||||
out # Lookup box decode value using mp4_path
|
||||
out ... | mp4_path(".moov.trak[1]")
|
||||
out # Return mp4_path string for a box decode value
|
||||
out ... | grep_by(.type == "trak") | mp4_path
|
||||
out # Decode file as mp4
|
||||
out $ fq -d mp4 file
|
||||
out # Decode value as mp4
|
||||
out ... | mp4
|
||||
out # Decode file using options
|
||||
out $ fq -d mp4 -o allow_truncated=false -o decode_samples=true file
|
||||
out # Decode value as mp4
|
||||
out ... | mp4({allow_truncated: false, decode_samples: true})
|
||||
out References and links
|
||||
out ISO/IEC base media file format (MPEG-4 Part 12) https://en.wikipedia.org/wiki/ISO/IEC_base_media_file_format
|
||||
out Quicktime file format https://developer.apple.com/standards/qtff-2001.pdf
|
||||
"help(mpeg_asc)"
|
||||
out mpeg_asc: MPEG-4 Audio Specific Config decoder
|
||||
out Examples:
|
||||
out # Decode file as mpeg_asc
|
||||
out $ fq -d mpeg_asc file
|
||||
out # Decode value as mpeg_asc
|
||||
out ... | mpeg_asc
|
||||
"help(mpeg_es)"
|
||||
out mpeg_es: MPEG Elementary Stream decoder
|
||||
out Examples:
|
||||
out # Decode file as mpeg_es
|
||||
out $ fq -d mpeg_es file
|
||||
out # Decode value as mpeg_es
|
||||
out ... | mpeg_es
|
||||
"help(mpeg_pes)"
|
||||
out mpeg_pes: MPEG Packetized elementary stream decoder
|
||||
out Examples:
|
||||
out # Decode file as mpeg_pes
|
||||
out $ fq -d mpeg_pes file
|
||||
out # Decode value as mpeg_pes
|
||||
out ... | mpeg_pes
|
||||
"help(mpeg_pes_packet)"
|
||||
out mpeg_pes_packet: MPEG Packetized elementary stream packet decoder
|
||||
out Examples:
|
||||
out # Decode file as mpeg_pes_packet
|
||||
out $ fq -d mpeg_pes_packet file
|
||||
out # Decode value as mpeg_pes_packet
|
||||
out ... | mpeg_pes_packet
|
||||
"help(mpeg_spu)"
|
||||
out mpeg_spu: Sub Picture Unit (DVD subtitle) decoder
|
||||
out Examples:
|
||||
out # Decode file as mpeg_spu
|
||||
out $ fq -d mpeg_spu file
|
||||
out # Decode value as mpeg_spu
|
||||
out ... | mpeg_spu
|
||||
"help(mpeg_ts)"
|
||||
out mpeg_ts: MPEG Transport Stream decoder
|
||||
out Examples:
|
||||
out # Decode file as mpeg_ts
|
||||
out $ fq -d mpeg_ts file
|
||||
out # Decode value as mpeg_ts
|
||||
out ... | mpeg_ts
|
||||
"help(msgpack)"
|
||||
out msgpack: MessagePack decoder
|
||||
out Examples:
|
||||
out # Decode file as msgpack
|
||||
out $ fq -d msgpack file
|
||||
out # Decode value as msgpack
|
||||
out ... | msgpack
|
||||
out # Supports torepr
|
||||
out $ fq -d msgpack torepr file
|
||||
out # Supports torepr
|
||||
out ... | msgpack | torepr
|
||||
out References and links
|
||||
out https://github.com/msgpack/msgpack/blob/master/spec.md
|
||||
"help(ogg)"
|
||||
out ogg: OGG file decoder
|
||||
out Examples:
|
||||
out # Decode file as ogg
|
||||
out $ fq -d ogg file
|
||||
out # Decode value as ogg
|
||||
out ... | ogg
|
||||
"help(ogg_page)"
|
||||
out ogg_page: OGG page decoder
|
||||
out Examples:
|
||||
out # Decode file as ogg_page
|
||||
out $ fq -d ogg_page file
|
||||
out # Decode value as ogg_page
|
||||
out ... | ogg_page
|
||||
"help(opus_packet)"
|
||||
out opus_packet: Opus packet decoder
|
||||
out Examples:
|
||||
out # Decode file as opus_packet
|
||||
out $ fq -d opus_packet file
|
||||
out # Decode value as opus_packet
|
||||
out ... | opus_packet
|
||||
"help(pcap)"
|
||||
out pcap: PCAP packet capture decoder
|
||||
out Examples:
|
||||
out # Decode file as pcap
|
||||
out $ fq -d pcap file
|
||||
out # Decode value as pcap
|
||||
out ... | pcap
|
||||
"help(pcapng)"
|
||||
out pcapng: PCAPNG packet capture decoder
|
||||
out Examples:
|
||||
out # Decode file as pcapng
|
||||
out $ fq -d pcapng file
|
||||
out # Decode value as pcapng
|
||||
out ... | pcapng
|
||||
"help(png)"
|
||||
out png: Portable Network Graphics file decoder
|
||||
out Examples:
|
||||
out # Decode file as png
|
||||
out $ fq -d png file
|
||||
out # Decode value as png
|
||||
out ... | png
|
||||
"help(protobuf)"
|
||||
out protobuf: Protobuf decoder
|
||||
out Examples:
|
||||
out # Can be used to decode sub messages
|
||||
out $ fq -d protobuf '.fields[6].wire_value | protobuf | d'
|
||||
out # Decode file as protobuf
|
||||
out $ fq -d protobuf file
|
||||
out # Decode value as protobuf
|
||||
out ... | protobuf
|
||||
out References and links
|
||||
out https://developers.google.com/protocol-buffers/docs/encoding
|
||||
"help(protobuf_widevine)"
|
||||
out protobuf_widevine: Widevine protobuf decoder
|
||||
out Examples:
|
||||
out # Decode file as protobuf_widevine
|
||||
out $ fq -d protobuf_widevine file
|
||||
out # Decode value as protobuf_widevine
|
||||
out ... | protobuf_widevine
|
||||
"help(pssh_playready)"
|
||||
out pssh_playready: PlayReady PSSH decoder
|
||||
out Examples:
|
||||
out # Decode file as pssh_playready
|
||||
out $ fq -d pssh_playready file
|
||||
out # Decode value as pssh_playready
|
||||
out ... | pssh_playready
|
||||
"help(raw)"
|
||||
out raw: Raw bits decoder
|
||||
out Examples:
|
||||
out # Decode file as raw
|
||||
out $ fq -d raw file
|
||||
out # Decode value as raw
|
||||
out ... | raw
|
||||
"help(rtmp)"
|
||||
out rtmp: Real-Time Messaging Protocol decoder
|
||||
out Current only supports plain RTMP (not RTMPT or encrypted variants etc) with AMF0 (not AMF3).
|
||||
out Examples:
|
||||
out # Decode file as rtmp
|
||||
out $ fq -d rtmp file
|
||||
out # Decode value as rtmp
|
||||
out ... | rtmp
|
||||
out References and links
|
||||
out https://rtmp.veriskope.com/docs/spec/
|
||||
out https://rtmp.veriskope.com/pdf/video_file_format_spec_v10.pdf
|
||||
"help(sll2_packet)"
|
||||
out sll2_packet: Linux cooked capture encapsulation v2 decoder
|
||||
out Examples:
|
||||
out # Decode file as sll2_packet
|
||||
out $ fq -d sll2_packet file
|
||||
out # Decode value as sll2_packet
|
||||
out ... | sll2_packet
|
||||
"help(sll_packet)"
|
||||
out sll_packet: Linux cooked capture encapsulation decoder
|
||||
out Examples:
|
||||
out # Decode file as sll_packet
|
||||
out $ fq -d sll_packet file
|
||||
out # Decode value as sll_packet
|
||||
out ... | sll_packet
|
||||
"help(tar)"
|
||||
out tar: Tar archive decoder
|
||||
out Examples:
|
||||
out # Decode file as tar
|
||||
out $ fq -d tar file
|
||||
out # Decode value as tar
|
||||
out ... | tar
|
||||
"help(tcp_segment)"
|
||||
out tcp_segment: Transmission control protocol segment decoder
|
||||
out Examples:
|
||||
out # Decode file as tcp_segment
|
||||
out $ fq -d tcp_segment file
|
||||
out # Decode value as tcp_segment
|
||||
out ... | tcp_segment
|
||||
"help(tiff)"
|
||||
out tiff: Tag Image File Format decoder
|
||||
out Examples:
|
||||
out # Decode file as tiff
|
||||
out $ fq -d tiff file
|
||||
out # Decode value as tiff
|
||||
out ... | tiff
|
||||
"help(udp_datagram)"
|
||||
out udp_datagram: User datagram protocol decoder
|
||||
out Examples:
|
||||
out # Decode file as udp_datagram
|
||||
out $ fq -d udp_datagram file
|
||||
out # Decode value as udp_datagram
|
||||
out ... | udp_datagram
|
||||
"help(vorbis_comment)"
|
||||
out vorbis_comment: Vorbis comment decoder
|
||||
out Examples:
|
||||
out # Decode file as vorbis_comment
|
||||
out $ fq -d vorbis_comment file
|
||||
out # Decode value as vorbis_comment
|
||||
out ... | vorbis_comment
|
||||
"help(vorbis_packet)"
|
||||
out vorbis_packet: Vorbis packet decoder
|
||||
out Examples:
|
||||
out # Decode file as vorbis_packet
|
||||
out $ fq -d vorbis_packet file
|
||||
out # Decode value as vorbis_packet
|
||||
out ... | vorbis_packet
|
||||
"help(vp8_frame)"
|
||||
out vp8_frame: VP8 frame decoder
|
||||
out Examples:
|
||||
out # Decode file as vp8_frame
|
||||
out $ fq -d vp8_frame file
|
||||
out # Decode value as vp8_frame
|
||||
out ... | vp8_frame
|
||||
"help(vp9_cfm)"
|
||||
out vp9_cfm: VP9 Codec Feature Metadata decoder
|
||||
out Examples:
|
||||
out # Decode file as vp9_cfm
|
||||
out $ fq -d vp9_cfm file
|
||||
out # Decode value as vp9_cfm
|
||||
out ... | vp9_cfm
|
||||
"help(vp9_frame)"
|
||||
out vp9_frame: VP9 frame decoder
|
||||
out Examples:
|
||||
out # Decode file as vp9_frame
|
||||
out $ fq -d vp9_frame file
|
||||
out # Decode value as vp9_frame
|
||||
out ... | vp9_frame
|
||||
"help(vpx_ccr)"
|
||||
out vpx_ccr: VPX Codec Configuration Record decoder
|
||||
out Examples:
|
||||
out # Decode file as vpx_ccr
|
||||
out $ fq -d vpx_ccr file
|
||||
out # Decode value as vpx_ccr
|
||||
out ... | vpx_ccr
|
||||
"help(wav)"
|
||||
out wav: WAV file decoder
|
||||
out Examples:
|
||||
out # Decode file as wav
|
||||
out $ fq -d wav file
|
||||
out # Decode value as wav
|
||||
out ... | wav
|
||||
"help(webp)"
|
||||
out webp: WebP image decoder
|
||||
out Examples:
|
||||
out # Decode file as webp
|
||||
out $ fq -d webp file
|
||||
out # Decode value as webp
|
||||
out ... | webp
|
||||
"help(xing)"
|
||||
out xing: Xing header decoder
|
||||
out Examples:
|
||||
out # Decode file as xing
|
||||
out $ fq -d xing file
|
||||
out # Decode value as xing
|
||||
out ... | xing
|
||||
"help(zip)"
|
||||
out zip: ZIP archive decoder
|
||||
out Examples:
|
||||
out # Decode file as zip
|
||||
out $ fq -d zip file
|
||||
out # Decode value as zip
|
||||
out ... | zip
|
||||
@ -28,16 +28,16 @@ import (
|
||||
"github.com/wader/fq/pkg/scalar"
|
||||
)
|
||||
|
||||
//go:embed *.jq
|
||||
//go:embed asn1_ber.jq
|
||||
var asn1FS embed.FS
|
||||
|
||||
func init() {
|
||||
registry.MustRegister(decode.Format{
|
||||
Name: format.ASN1_BER,
|
||||
Description: "ASN1 Basic Encoding Rules (also CER and DER)",
|
||||
Description: "ASN1 BER (basic encoding rules, also CER and DER)",
|
||||
DecodeFn: decodeASN1BER,
|
||||
Files: asn1FS,
|
||||
ToRepr: "_asn1_ber_torepr",
|
||||
Functions: []string{"torepr", "_help"},
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@ -6,3 +6,24 @@ def _asn1_ber_torepr:
|
||||
end
|
||||
else .constructed | map(_asn1_ber_torepr)
|
||||
end;
|
||||
|
||||
def _asn1_ber__help:
|
||||
{ notes: "Supports decoding BER, CER and DER (X.690).
|
||||
|
||||
- Currently no extra validation is done for CER and DER.
|
||||
- Does not support specifying a schema.
|
||||
- Supports `torepr` but without schema all sequences and sets will be arrays.",
|
||||
examples: [
|
||||
{comment: "`frompem` and `topem` can be used to work with PEM format", shell: "fq -d raw 'frompem | asn1_ber | d' cert.pem"},
|
||||
{comment: "Can be used to decode nested parts", shell: "fq -d asn1_ber '.constructed[1].value | asn1_ber' file.ber"},
|
||||
{ comment: "If schema is known and not complicated it can be reproduced",
|
||||
shell: "fq -d asn1_ber 'torepr as $r | [\"version\", \"modulus\", \"private_exponent\", \"private_exponen\", \"prime1\", \"prime2\", \"exponent1\", \"exponent2\", \"coefficient\"] | with_entries({key: .value, value: $r[.key]})' pkcs1.der"
|
||||
}
|
||||
],
|
||||
links: [
|
||||
{url: "https://www.itu.int/ITU-T/studygroups/com10/languages/X.690_1297.pdf"},
|
||||
{url: "https://en.wikipedia.org/wiki/X.690"},
|
||||
{url: "https://letsencrypt.org/docs/a-warm-welcome-to-asn1-and-der/"},
|
||||
{url: "https://lapo.it/asn1js/"}
|
||||
]
|
||||
};
|
||||
|
||||
@ -1,33 +0,0 @@
|
||||
Supports decoding BER, CER and DER ([X.690]([X.690_1297.pdf)).
|
||||
|
||||
- Currently no extra validation is done for CER and DER.
|
||||
- Does not support specifying a schema.
|
||||
- Supports `torepr` but without schema all sequences and sets will be arrays.
|
||||
|
||||
```
|
||||
fq -d asn1_ber torepr file.ber
|
||||
```
|
||||
|
||||
Functions `frompem` and `topem` can help working with PEM format:
|
||||
|
||||
```
|
||||
fq -d raw 'frompem | asn1_ber | d' cert.pem
|
||||
```
|
||||
|
||||
If the schema is known and not that complicated it can be reproduced:
|
||||
|
||||
```
|
||||
fq -d asn1_ber 'torepr as $r | ["version", "modulus", "private_exponent", "private_exponen", "prime1", "prime2", "exponent1", "exponent2", "coefficient"] | with_entries({key: .value, value: $r[.key]})' pkcs1.der
|
||||
```
|
||||
|
||||
Can be used to decode nested parts:
|
||||
|
||||
```
|
||||
fq -d asn1_ber '.constructed[1].value | asn1_ber' file.ber
|
||||
```
|
||||
|
||||
References and tools:
|
||||
- https://www.itu.int/ITU-T/studygroups/com10/languages/X.690_1297.pdf
|
||||
- https://en.wikipedia.org/wiki/X.690
|
||||
- https://letsencrypt.org/docs/a-warm-welcome-to-asn1-and-der/
|
||||
- https://lapo.it/asn1js/
|
||||
@ -3,6 +3,7 @@ package avro
|
||||
import (
|
||||
"bytes"
|
||||
"compress/flate"
|
||||
"embed"
|
||||
"hash/crc32"
|
||||
|
||||
"github.com/golang/snappy"
|
||||
@ -15,12 +16,17 @@ import (
|
||||
"github.com/wader/fq/pkg/scalar"
|
||||
)
|
||||
|
||||
//go:embed avro_ocf.jq
|
||||
var avroOcfFS embed.FS
|
||||
|
||||
func init() {
|
||||
registry.MustRegister(decode.Format{
|
||||
Name: format.AVRO_OCF,
|
||||
Description: "Avro object container file",
|
||||
Groups: []string{format.PROBE},
|
||||
DecodeFn: decodeAvroOCF,
|
||||
Functions: []string{"_help"},
|
||||
Files: avroOcfFS,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
12
format/avro/avro_ocf.jq
Normal file
12
format/avro/avro_ocf.jq
Normal file
@ -0,0 +1,12 @@
|
||||
def _avro_ocf__help:
|
||||
{ notes: "Supports reading Avro Object Container Format (OCF) files based on the 1.11.0 specification.
|
||||
|
||||
Capable of handling null, deflate, and snappy codecs for data compression.
|
||||
|
||||
Limitations:
|
||||
- Schema does not support self-referential types, only built-in types.
|
||||
- Decimal logical types are not supported for decoding, will just be treated as their primitive type",
|
||||
links: [
|
||||
{url: "https://avro.apache.org/docs/current/spec.html#Object+Container+Files"}
|
||||
]
|
||||
};
|
||||
@ -1,7 +0,0 @@
|
||||
Supports reading Avro Object Container Format (OCF) files based on the [1.11.0 specification](https://avro.apache.org/docs/current/spec.html#Object+Container+Files).
|
||||
|
||||
Capable of handling null, deflate, and snappy codecs for data compression.
|
||||
|
||||
Limitations:
|
||||
- Schema does not support self-referential types, only built-in types.
|
||||
- Decimal logical types are not supported for decoding, will just be treated as their primitive type
|
||||
@ -1,5 +0,0 @@
|
||||
Supports `torepr`:
|
||||
|
||||
```
|
||||
fq -d bencode torepr file.torrent
|
||||
```
|
||||
@ -1,5 +1,7 @@
|
||||
package bencode
|
||||
|
||||
// https://wiki.theory.org/BitTorrentSpecification#Bencoding
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"strconv"
|
||||
@ -10,7 +12,7 @@ import (
|
||||
"github.com/wader/fq/pkg/scalar"
|
||||
)
|
||||
|
||||
//go:embed *.jq
|
||||
//go:embed bencode.jq
|
||||
var bencodeFS embed.FS
|
||||
|
||||
func init() {
|
||||
@ -19,7 +21,7 @@ func init() {
|
||||
Description: "BitTorrent bencoding",
|
||||
DecodeFn: decodeBencode,
|
||||
Files: bencodeFS,
|
||||
ToRepr: "_bencode_torepr",
|
||||
Functions: []string{"torepr", "_help"},
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@ -9,3 +9,12 @@ def _bencode_torepr:
|
||||
)
|
||||
else error("unknown type \(.type)")
|
||||
end;
|
||||
|
||||
def _bencode__help:
|
||||
{ examples: [
|
||||
{comment: "bencode as JSON", shell: "fq -d bencode torepr file"}
|
||||
],
|
||||
links: [
|
||||
{url: "https://bsonspec.org/spec.html"}
|
||||
]
|
||||
};
|
||||
|
||||
@ -12,7 +12,7 @@ import (
|
||||
"github.com/wader/fq/pkg/scalar"
|
||||
)
|
||||
|
||||
//go:embed *.jq
|
||||
//go:embed bson.jq
|
||||
var bsonFS embed.FS
|
||||
|
||||
func init() {
|
||||
@ -21,7 +21,7 @@ func init() {
|
||||
Description: "Binary JSON",
|
||||
DecodeFn: decodeBSON,
|
||||
Files: bsonFS,
|
||||
ToRepr: "_bson_torepr",
|
||||
Functions: []string{"torepr", "_help"},
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@ -16,3 +16,12 @@ def _bson_torepr:
|
||||
( {type: "document", value: .}
|
||||
| _f
|
||||
);
|
||||
|
||||
def _bson__help:
|
||||
{ examples: [
|
||||
{comment: "BSON as JSON", shell: "fq -d bson torepr file"}
|
||||
],
|
||||
links: [
|
||||
{url: "https://wiki.theory.org/BitTorrentSpecification#Bencoding"}
|
||||
]
|
||||
};
|
||||
|
||||
@ -1,5 +0,0 @@
|
||||
Supports `torepr`:
|
||||
|
||||
```
|
||||
fq -d bson torepr file.bson
|
||||
```
|
||||
@ -20,7 +20,7 @@ import (
|
||||
"github.com/wader/fq/pkg/scalar"
|
||||
)
|
||||
|
||||
//go:embed *.jq
|
||||
//go:embed cbor.jq
|
||||
var cborFS embed.FS
|
||||
|
||||
func init() {
|
||||
@ -29,7 +29,7 @@ func init() {
|
||||
Description: "Concise Binary Object Representation",
|
||||
DecodeFn: decodeCBOR,
|
||||
Files: cborFS,
|
||||
ToRepr: "_cbor_torepr",
|
||||
Functions: []string{"torepr", "_help"},
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@ -8,3 +8,10 @@ def _cbor_torepr:
|
||||
elif .major_type == "bytes" then .value | tostring
|
||||
else .value | tovalue
|
||||
end;
|
||||
|
||||
def _cbor__help:
|
||||
{ links: [
|
||||
{url: "https://en.wikipedia.org/wiki/CBOR"},
|
||||
{url: "https://www.rfc-editor.org/rfc/rfc8949.html"}
|
||||
]
|
||||
};
|
||||
|
||||
@ -1,8 +0,0 @@
|
||||
Supports `torepr`:
|
||||
|
||||
```
|
||||
fq -d cbor torepr file.cbor
|
||||
fq -d cbor 'torepr.field' file.cbor
|
||||
fq -d cbor 'torepr | .field' file.cbor
|
||||
fq -d cbor 'torepr | grep("abc")' file.cbor
|
||||
```
|
||||
@ -52,7 +52,7 @@ func flacDecode(d *decode.D, in interface{}) interface{} {
|
||||
if flacMetadatablockOut.HasStreamInfo {
|
||||
streamInfo = flacMetadatablockOut.StreamInfo
|
||||
streamTotalSamples = streamInfo.TotalSamplesInStream
|
||||
flacFrameIn = format.FlacFrameIn{StreamInfo: streamInfo}
|
||||
flacFrameIn = format.FlacFrameIn{BitsPerSample: int(streamInfo.BitsPerSample)}
|
||||
}
|
||||
|
||||
md5Samples := md5.New()
|
||||
|
||||
@ -17,6 +17,9 @@ func init() {
|
||||
Name: format.FLAC_FRAME,
|
||||
Description: "FLAC frame",
|
||||
DecodeFn: frameDecode,
|
||||
DecodeInArg: format.FlacFrameIn{
|
||||
BitsPerSample: 16,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@ -98,20 +101,18 @@ func utf8Uint(d *decode.D) uint64 {
|
||||
|
||||
// in argument is an optional FlacFrameIn struct with stream info
|
||||
func frameDecode(d *decode.D, in interface{}) interface{} {
|
||||
var inStreamInfo *format.FlacStreamInfo
|
||||
ffi, ok := in.(format.FlacFrameIn)
|
||||
if ok {
|
||||
inStreamInfo = &ffi.StreamInfo
|
||||
}
|
||||
|
||||
frameStart := d.Pos()
|
||||
|
||||
blockSize := 0
|
||||
channelAssignment := uint64(0)
|
||||
channels := 0
|
||||
sampleSize := 0
|
||||
sideChannelIndex := -1
|
||||
|
||||
ffi, ok := in.(format.FlacFrameIn)
|
||||
if ok {
|
||||
sampleSize = ffi.BitsPerSample
|
||||
}
|
||||
|
||||
d.FieldStruct("header", func(d *decode.D) {
|
||||
// <14> 11111111111110
|
||||
d.FieldU14("sync", d.AssertU(0b11111111111110), scalar.Bin)
|
||||
@ -192,12 +193,6 @@ func frameDecode(d *decode.D, in interface{}) interface{} {
|
||||
0b1111: {Description: "invalid"},
|
||||
}
|
||||
sampleRateS := d.FieldScalarU4("sample_rate", sampleRateMap, scalar.Bin)
|
||||
switch sampleRateS.ActualU() {
|
||||
case SampleRateStreaminfo:
|
||||
if inStreamInfo == nil {
|
||||
d.Fatalf("streaminfo required for sample rate")
|
||||
}
|
||||
}
|
||||
|
||||
// <4> Channel assignment
|
||||
// 0000-0111 : (number of independent channels)-1. Where defined, the channel order follows SMPTE/ITU-R recommendations. The assignments are as follows:
|
||||
@ -271,10 +266,7 @@ func frameDecode(d *decode.D, in interface{}) interface{} {
|
||||
sampleSizeS := d.FieldScalarU3("sample_size", sampleSizeMap, scalar.Bin)
|
||||
switch sampleSizeS.ActualU() {
|
||||
case SampleSizeStreaminfo:
|
||||
if inStreamInfo == nil {
|
||||
d.Fatalf("streaminfo required for sample size")
|
||||
}
|
||||
sampleSize = int(inStreamInfo.BitPerSample)
|
||||
sampleSize = ffi.BitsPerSample
|
||||
default:
|
||||
if sampleSizeS.Sym != nil {
|
||||
sampleSize = int(sampleSizeS.SymU())
|
||||
|
||||
@ -24,7 +24,7 @@ func streaminfoDecode(d *decode.D, in interface{}) interface{} {
|
||||
// <3> (number of channels)-1. FLAC supports from 1 to 8 channels
|
||||
d.FieldU3("channels", scalar.UAdd(1))
|
||||
// <5> (bits per sample)-1. FLAC supports from 4 to 32 bits per sample. Currently the reference encoder and decoders only support up to 24 bits per sample.
|
||||
bitPerSample := d.FieldU5("bits_per_sample", scalar.UAdd(1))
|
||||
bitsPerSample := d.FieldU5("bits_per_sample", scalar.UAdd(1))
|
||||
totalSamplesInStream := d.FieldU("total_samples_in_stream", 36)
|
||||
md5BR := d.FieldRawLen("md5", 16*8, scalar.RawHex)
|
||||
md5b := d.MustReadAllBits(md5BR)
|
||||
@ -32,7 +32,7 @@ func streaminfoDecode(d *decode.D, in interface{}) interface{} {
|
||||
return format.FlacStreaminfoOut{
|
||||
StreamInfo: format.FlacStreamInfo{
|
||||
SampleRate: sampleRate,
|
||||
BitPerSample: bitPerSample,
|
||||
BitsPerSample: bitsPerSample,
|
||||
TotalSamplesInStream: totalSamplesInStream,
|
||||
MD5: md5b,
|
||||
},
|
||||
|
||||
@ -111,7 +111,7 @@ const (
|
||||
|
||||
type FlacStreamInfo struct {
|
||||
SampleRate uint64
|
||||
BitPerSample uint64
|
||||
BitsPerSample uint64
|
||||
TotalSamplesInStream uint64
|
||||
MD5 []byte
|
||||
}
|
||||
@ -120,12 +120,6 @@ type FlacStreaminfoOut struct {
|
||||
StreamInfo FlacStreamInfo
|
||||
}
|
||||
|
||||
type FlacMetadatablockStreamInfo struct {
|
||||
SampleRate uint64
|
||||
BitPerSample uint64
|
||||
TotalSamplesInStream uint64
|
||||
}
|
||||
|
||||
type FlacMetadatablockOut struct {
|
||||
IsLastBlock bool
|
||||
HasStreamInfo bool
|
||||
@ -138,8 +132,8 @@ type FlacMetadatablocksOut struct {
|
||||
}
|
||||
|
||||
type FlacFrameIn struct {
|
||||
SamplesBuf []byte
|
||||
StreamInfo FlacStreamInfo
|
||||
SamplesBuf []byte
|
||||
BitsPerSample int `doc:"Bits per sample"`
|
||||
}
|
||||
|
||||
type FlacFrameOut struct {
|
||||
@ -158,16 +152,16 @@ type OggPageOut struct {
|
||||
Segments [][]byte
|
||||
}
|
||||
|
||||
type AvcIn struct {
|
||||
LengthSize uint64
|
||||
type AvcAuIn struct {
|
||||
LengthSize uint64 `doc:"Length value size"`
|
||||
}
|
||||
|
||||
type AvcDcrOut struct {
|
||||
LengthSize uint64
|
||||
}
|
||||
|
||||
type HevcIn struct {
|
||||
LengthSize uint64
|
||||
type HevcAuIn struct {
|
||||
LengthSize uint64 `doc:"Length value size"`
|
||||
}
|
||||
|
||||
type HevcDcrOut struct {
|
||||
@ -192,7 +186,12 @@ type MPEGASCOut struct {
|
||||
}
|
||||
|
||||
type AACFrameIn struct {
|
||||
ObjectType int
|
||||
ObjectType int `doc:"Audio object type"`
|
||||
}
|
||||
|
||||
type Mp3In struct {
|
||||
MaxUniqueHeaderConfigs int `doc:"Max number of unique frame header configs allowed"`
|
||||
MaxSyncSeek int `doc:"Max byte distance to next sync"`
|
||||
}
|
||||
|
||||
type MP3FrameOut struct {
|
||||
@ -261,12 +260,7 @@ func (t TCPStreamIn) MustIsPort(fn func(format string, a ...interface{}), ports
|
||||
}
|
||||
}
|
||||
|
||||
type X86_64In struct {
|
||||
Base int64
|
||||
SymLookup func(uint64) (string, uint64)
|
||||
}
|
||||
|
||||
type ARM64In struct {
|
||||
Base int64
|
||||
SymLookup func(uint64) (string, uint64)
|
||||
type Mp4In struct {
|
||||
DecodeSamples bool `doc:"Decode supported media samples"`
|
||||
AllowTruncated bool `doc:"Allow box to be truncated"`
|
||||
}
|
||||
|
||||
@ -3,6 +3,7 @@ package macho
|
||||
// https://github.com/aidansteele/osx-abi-macho-file-format-reference
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"time"
|
||||
|
||||
"github.com/wader/fq/format"
|
||||
@ -12,12 +13,17 @@ import (
|
||||
"github.com/wader/fq/pkg/scalar"
|
||||
)
|
||||
|
||||
//go:embed macho.jq
|
||||
var machoFS embed.FS
|
||||
|
||||
func init() {
|
||||
registry.MustRegister(decode.Format{
|
||||
Name: format.MACHO,
|
||||
Description: "Mach-O macOS executable",
|
||||
Groups: []string{format.PROBE},
|
||||
DecodeFn: machoDecode,
|
||||
Files: machoFS,
|
||||
Functions: []string{"_help"},
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
9
format/macho/macho.jq
Normal file
9
format/macho/macho.jq
Normal file
@ -0,0 +1,9 @@
|
||||
def _macho__help:
|
||||
{ notes: "Supports decoding vanilla and FAT Mach-O binaries.",
|
||||
examples: [
|
||||
{comment: "Select 64bit load segments", shell: "fq '.load_commands[] | select(.cmd==\"segment_64\")' file"}
|
||||
],
|
||||
links: [
|
||||
{url: "https://github.com/aidansteele/osx-abi-macho-file-format-reference"}
|
||||
]
|
||||
};
|
||||
@ -1,18 +0,0 @@
|
||||
Supports decoding vanilla and FAT Mach-O binaries.
|
||||
|
||||
#### Examples
|
||||
|
||||
To decode the macOS build of `fq`:
|
||||
|
||||
```
|
||||
fq . /path/to/fq
|
||||
```
|
||||
|
||||
```
|
||||
fq '.load_commands[] | select(.cmd=="segment_64")' /path/to/fq
|
||||
```
|
||||
|
||||
Note you can use `-d macho` to decode a broken Mach-O binary.
|
||||
|
||||
#### References:
|
||||
- https://github.com/aidansteele/osx-abi-macho-file-format-reference
|
||||
@ -26,7 +26,7 @@ import (
|
||||
"github.com/wader/fq/pkg/scalar"
|
||||
)
|
||||
|
||||
//go:embed *.jq
|
||||
//go:embed matroska.jq
|
||||
var matroskaFS embed.FS
|
||||
|
||||
var aacFrameFormat decode.Group
|
||||
@ -78,7 +78,8 @@ func init() {
|
||||
{Names: []string{format.VP9_CFM}, Group: &vp9CFMFormat},
|
||||
{Names: []string{format.VP9_FRAME}, Group: &vp9FrameFormat},
|
||||
},
|
||||
Files: matroskaFS,
|
||||
Functions: []string{"_help"},
|
||||
Files: matroskaFS,
|
||||
})
|
||||
|
||||
codecToFormat = map[string]*decode.Group{
|
||||
@ -369,7 +370,7 @@ func matroskaDecode(d *decode.D, in interface{}) interface{} {
|
||||
panic(fmt.Sprintf("expected FlacMetadatablockOut got %#+v", v))
|
||||
}
|
||||
if flacMetadatablockOut.HasStreamInfo {
|
||||
t.formatInArg = format.FlacFrameIn{StreamInfo: flacMetadatablockOut.StreamInfo}
|
||||
t.formatInArg = format.FlacFrameIn{BitsPerSample: int(flacMetadatablockOut.StreamInfo.BitsPerSample)}
|
||||
}
|
||||
})
|
||||
})
|
||||
@ -379,14 +380,14 @@ func matroskaDecode(d *decode.D, in interface{}) interface{} {
|
||||
if dv != nil && !ok {
|
||||
panic(fmt.Sprintf("expected AvcDcrOut got %#+v", v))
|
||||
}
|
||||
t.formatInArg = format.AvcIn{LengthSize: avcDcrOut.LengthSize} //nolint:gosimple
|
||||
t.formatInArg = format.AvcAuIn{LengthSize: avcDcrOut.LengthSize} //nolint:gosimple
|
||||
case "V_MPEGH/ISO/HEVC":
|
||||
dv, v := t.parentD.FieldFormatRange("value", t.codecPrivatePos, t.codecPrivateTagSize, mpegHEVCDCRFormat, nil)
|
||||
hevcDcrOut, ok := v.(format.HevcDcrOut)
|
||||
if dv != nil && !ok {
|
||||
panic(fmt.Sprintf("expected HevcDcrOut got %#+v", v))
|
||||
}
|
||||
t.formatInArg = format.HevcIn{LengthSize: hevcDcrOut.LengthSize} //nolint:gosimple
|
||||
t.formatInArg = format.HevcAuIn{LengthSize: hevcDcrOut.LengthSize} //nolint:gosimple
|
||||
case "V_AV1":
|
||||
t.parentD.FieldFormatRange("value", t.codecPrivatePos, t.codecPrivateTagSize, av1CCRFormat, nil)
|
||||
case "V_VP9":
|
||||
|
||||
@ -4,11 +4,25 @@
|
||||
def matroska_path(p):
|
||||
_decode_value(
|
||||
( if format != "matroska" then error("not matroska format") end
|
||||
| tree_path(.elements; .id; p)
|
||||
| _tree_path(.elements; .id; p)
|
||||
)
|
||||
);
|
||||
def matroska_path:
|
||||
( . as $c
|
||||
| format_root
|
||||
| matroska_path($c)
|
||||
);
|
||||
);
|
||||
|
||||
def _matroska__help:
|
||||
{ examples: [
|
||||
{comment: "Lookup element decode value using `matroska_path`", expr: "matroska_path(\".Segment.Tracks[0)\""},
|
||||
{comment: "Return `matroska_path` string for a box decode value", expr: "grep_by(.id == \"Tracks\") | matroska_path"}
|
||||
],
|
||||
links: [
|
||||
{url: "https://tools.ietf.org/html/draft-ietf-cellar-ebml-00"},
|
||||
{url: "https://matroska.org/technical/specs/index.html"},
|
||||
{url: "https://www.matroska.org/technical/basics.html"},
|
||||
{url: "https://www.matroska.org/technical/codec_specs.html"},
|
||||
{url: "https://wiki.xiph.org/MatroskaOpus"}
|
||||
]
|
||||
};
|
||||
@ -1,18 +0,0 @@
|
||||
Supports `matroska_path`:
|
||||
|
||||
```
|
||||
$ fq 'matroska_path(".Segment.Tracks[0]")' file.mkv
|
||||
│00 01 02 03 04 05 06 07 08 09│0123456789│.elements[1].elements[3]{}:
|
||||
0x122│ 16 54 ae 6b │ .T.k │ id: "Tracks" (0x1654ae6b) (A Top-Level Element of information with many tracks described.)
|
||||
│ │ │ type: "master" (7)
|
||||
0x122│ 4d bf │ M. │ size: 3519
|
||||
0x122│ bf│ .│ elements[0:3]:
|
||||
0x12c│84 cf 8b db a0 ae 01 00 00 00│..........│
|
||||
0x136│00 00 00 78 d7 81 01 73 c5 88│...x...s..│
|
||||
* │until 0xee9.7 (3519) │ │
|
||||
```
|
||||
|
||||
```
|
||||
$ fq 'first(grep_by(.id == "Tracks")) | matroska_path' test.mkv
|
||||
".Segment.Tracks"
|
||||
```
|
||||
@ -14,10 +14,6 @@ var headerFormat decode.Group
|
||||
var footerFormat decode.Group
|
||||
var mp3Frame decode.Group
|
||||
|
||||
// TODO: format options default
|
||||
const maxUniqueHeaderConfigs = 5
|
||||
const maxSyncSeek = 4 * 1024 * 8
|
||||
|
||||
func init() {
|
||||
registry.MustRegister(decode.Format{
|
||||
Name: format.MP3,
|
||||
@ -25,6 +21,10 @@ func init() {
|
||||
Description: "MP3 file",
|
||||
Groups: []string{format.PROBE},
|
||||
DecodeFn: mp3Decode,
|
||||
DecodeInArg: format.Mp3In{
|
||||
MaxUniqueHeaderConfigs: 5,
|
||||
MaxSyncSeek: 4 * 1024 * 8,
|
||||
},
|
||||
Dependencies: []decode.Dependency{
|
||||
{Names: []string{format.ID3V2}, Group: &headerFormat},
|
||||
{
|
||||
@ -41,6 +41,8 @@ func init() {
|
||||
}
|
||||
|
||||
func mp3Decode(d *decode.D, in interface{}) interface{} {
|
||||
mi, _ := in.(format.Mp3In)
|
||||
|
||||
// things in a mp3 stream usually have few unique combinations of.
|
||||
// does not include bitrate on purpose
|
||||
type headerConfig struct {
|
||||
@ -66,7 +68,7 @@ func mp3Decode(d *decode.D, in interface{}) interface{} {
|
||||
decodeFailures := 0
|
||||
d.FieldArray("frames", func(d *decode.D) {
|
||||
for d.NotEnd() {
|
||||
syncLen, _, err := d.TryPeekFind(16, 8, maxSyncSeek, func(v uint64) bool {
|
||||
syncLen, _, err := d.TryPeekFind(16, 8, int64(mi.MaxSyncSeek), func(v uint64) bool {
|
||||
return (v&0b1111_1111_1110_0000 == 0b1111_1111_1110_0000 && // sync header
|
||||
v&0b0000_0000_0001_1000 != 0b0000_0000_0000_1000 && // not reserved mpeg version
|
||||
v&0b0000_0000_0000_0110 == 0b0000_0000_0000_0010) // layer 3
|
||||
@ -99,7 +101,7 @@ func mp3Decode(d *decode.D, in interface{}) interface{} {
|
||||
lastValidEnd = d.Pos()
|
||||
validFrames++
|
||||
|
||||
if len(uniqueHeaderConfigs) >= maxUniqueHeaderConfigs {
|
||||
if len(uniqueHeaderConfigs) >= mi.MaxUniqueHeaderConfigs {
|
||||
d.Errorf("too many unique header configurations")
|
||||
}
|
||||
}
|
||||
|
||||
@ -136,10 +136,10 @@ func decodeBoxWithParentData(ctx *decodeContext, d *decode.D, parentData interfa
|
||||
dataSize = boxSize - 8
|
||||
}
|
||||
|
||||
// TODO: add truncate to size option?
|
||||
// if dataSize > uint64(d.BitsLeft()/8) {
|
||||
// dataSize = uint64(d.BitsLeft() / 8)
|
||||
// }
|
||||
if ctx.opts.AllowTruncated && dataSize > uint64(d.BitsLeft()/8) {
|
||||
dataSize = uint64(d.BitsLeft() / 8)
|
||||
|
||||
}
|
||||
|
||||
// TODO: not sure about this
|
||||
switch {
|
||||
@ -475,7 +475,7 @@ func init() {
|
||||
panic(fmt.Sprintf("expected AvcDcrOut got %#+v", v))
|
||||
}
|
||||
if ctx.currentTrack != nil {
|
||||
ctx.currentTrack.formatInArg = format.AvcIn{LengthSize: avcDcrOut.LengthSize} //nolint:gosimple
|
||||
ctx.currentTrack.formatInArg = format.AvcAuIn{LengthSize: avcDcrOut.LengthSize} //nolint:gosimple
|
||||
}
|
||||
},
|
||||
"hvcC": func(ctx *decodeContext, d *decode.D) {
|
||||
@ -485,7 +485,7 @@ func init() {
|
||||
panic(fmt.Sprintf("expected HevcDcrOut got %#+v", v))
|
||||
}
|
||||
if ctx.currentTrack != nil {
|
||||
ctx.currentTrack.formatInArg = format.HevcIn{LengthSize: hevcDcrOut.LengthSize} //nolint:gosimple
|
||||
ctx.currentTrack.formatInArg = format.HevcAuIn{LengthSize: hevcDcrOut.LengthSize} //nolint:gosimple
|
||||
}
|
||||
},
|
||||
"dfLa": func(ctx *decodeContext, d *decode.D) {
|
||||
@ -498,7 +498,7 @@ func init() {
|
||||
}
|
||||
if flacMetadatablockOut.HasStreamInfo {
|
||||
if ctx.currentTrack != nil {
|
||||
ctx.currentTrack.formatInArg = format.FlacFrameIn{StreamInfo: flacMetadatablockOut.StreamInfo}
|
||||
ctx.currentTrack.formatInArg = format.FlacFrameIn{BitsPerSample: int(flacMetadatablockOut.StreamInfo.BitsPerSample)}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
@ -25,7 +25,7 @@ import (
|
||||
"github.com/wader/fq/pkg/decode"
|
||||
)
|
||||
|
||||
//go:embed *.jq
|
||||
//go:embed mp4.jq
|
||||
var mp4FS embed.FS
|
||||
|
||||
var aacFrameFormat decode.Group
|
||||
@ -60,6 +60,10 @@ func init() {
|
||||
format.IMAGE, // avif
|
||||
},
|
||||
DecodeFn: mp4Decode,
|
||||
DecodeInArg: format.Mp4In{
|
||||
DecodeSamples: true,
|
||||
AllowTruncated: false,
|
||||
},
|
||||
Dependencies: []decode.Dependency{
|
||||
{Names: []string{format.AAC_FRAME}, Group: &aacFrameFormat},
|
||||
{Names: []string{format.AV1_CCR}, Group: &av1CCRFormat},
|
||||
@ -84,7 +88,8 @@ func init() {
|
||||
{Names: []string{format.VPX_CCR}, Group: &vpxCCRFormat},
|
||||
{Names: []string{format.ICC_PROFILE}, Group: &iccProfileFormat},
|
||||
},
|
||||
Files: mp4FS,
|
||||
Files: mp4FS,
|
||||
Functions: []string{"_help"},
|
||||
})
|
||||
}
|
||||
|
||||
@ -131,6 +136,7 @@ type pathEntry struct {
|
||||
}
|
||||
|
||||
type decodeContext struct {
|
||||
opts format.Mp4In
|
||||
path []pathEntry
|
||||
tracks map[uint32]*track
|
||||
currentTrack *track
|
||||
@ -145,10 +151,10 @@ func (ctx *decodeContext) parent() pathEntry {
|
||||
return ctx.path[len(ctx.path)-2]
|
||||
}
|
||||
|
||||
func mp4Tracks(d *decode.D, tracks map[uint32]*track) {
|
||||
func mp4Tracks(d *decode.D, ctx *decodeContext) {
|
||||
// keep track order stable
|
||||
var sortedTracks []*track
|
||||
for _, t := range tracks {
|
||||
for _, t := range ctx.tracks {
|
||||
sortedTracks = append(sortedTracks, t)
|
||||
}
|
||||
sort.Slice(sortedTracks, func(i, j int) bool { return sortedTracks[i].id < sortedTracks[j].id })
|
||||
@ -157,6 +163,11 @@ func mp4Tracks(d *decode.D, tracks map[uint32]*track) {
|
||||
for _, t := range sortedTracks {
|
||||
decodeSampleRange := func(d *decode.D, t *track, dataFormat string, name string, firstBit int64, nBits int64, inArg interface{}) {
|
||||
d.RangeFn(firstBit, nBits, func(d *decode.D) {
|
||||
if !ctx.opts.DecodeSamples {
|
||||
d.FieldRawLen(name, d.BitsLeft())
|
||||
return
|
||||
}
|
||||
|
||||
switch {
|
||||
case dataFormat == "fLaC":
|
||||
d.FieldFormatLen(name, nBits, flacFrameFormat, inArg)
|
||||
@ -290,7 +301,10 @@ func mp4Tracks(d *decode.D, tracks map[uint32]*track) {
|
||||
}
|
||||
|
||||
func mp4Decode(d *decode.D, in interface{}) interface{} {
|
||||
mi, _ := in.(format.Mp4In)
|
||||
|
||||
ctx := &decodeContext{
|
||||
opts: mi,
|
||||
path: []pathEntry{{typ: "root"}},
|
||||
tracks: map[uint32]*track{},
|
||||
}
|
||||
@ -317,9 +331,8 @@ func mp4Decode(d *decode.D, in interface{}) interface{} {
|
||||
|
||||
decodeBoxes(ctx, d)
|
||||
if len(ctx.tracks) > 0 {
|
||||
mp4Tracks(d, ctx.tracks)
|
||||
mp4Tracks(d, ctx)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
@ -2,13 +2,25 @@
|
||||
# box -> | mp4_path -> ".moov.trak[1]"
|
||||
# box -> | mp4_path(<mp4 root>) -> ".moov.trak[1]"
|
||||
def mp4_path(p):
|
||||
_decode_value(
|
||||
( if format != "mp4" then error("not mp4 format") end
|
||||
| tree_path(.boxes; .type; p)
|
||||
)
|
||||
);
|
||||
_decode_value(
|
||||
( if format != "mp4" then error("not mp4 format") end
|
||||
| _tree_path(.boxes; .type; p)
|
||||
)
|
||||
);
|
||||
def mp4_path:
|
||||
( . as $c
|
||||
| format_root
|
||||
| mp4_path($c)
|
||||
);
|
||||
( . as $c
|
||||
| format_root
|
||||
| mp4_path($c)
|
||||
);
|
||||
|
||||
def _mp4__help:
|
||||
{ notes: "Support `mp4_path`",
|
||||
examples: [
|
||||
{comment: "Lookup box decode value using `mp4_path`", expr: "mp4_path(\".moov.trak[1]\")"},
|
||||
{comment: "Return `mp4_path` string for a box decode value", expr: "grep_by(.type == \"trak\") | mp4_path"}
|
||||
],
|
||||
links: [
|
||||
{title: "ISO/IEC base media file format (MPEG-4 Part 12)", url: "https://en.wikipedia.org/wiki/ISO/IEC_base_media_file_format"},
|
||||
{title: "Quicktime file format", url: "https://developer.apple.com/standards/qtff-2001.pdf"}
|
||||
]
|
||||
};
|
||||
|
||||
@ -1,18 +0,0 @@
|
||||
Supports `mp4_path`:
|
||||
|
||||
```
|
||||
$ fq 'mp4_path(".moov.trak[1]")' file.mp4
|
||||
│00 01 02 03 04 05 06 07 08 09│0123456789│.boxes[3].boxes[1]{}:
|
||||
0x4f6│ 00 00 02│ ...│ size: 573
|
||||
0x500│3d │= │
|
||||
0x500│ 74 72 61 6b │ trak │ type: "trak" (Container for an individual track or stream)
|
||||
0x500│ 00 00 00 5c 74│ ...\t│ boxes[0:3]:
|
||||
0x50a│6b 68 64 00 00 00 03 00 00 00│khd.......│
|
||||
0x514│00 00 00 00 00 00 00 00 01 00│..........│
|
||||
* │until 0x739.7 (565) │ │
|
||||
```
|
||||
|
||||
```
|
||||
$ fq 'first(grep_by(.type == "trak")) | mp4_path' file.mp4
|
||||
".moov.trak"
|
||||
```
|
||||
15
format/mp4/testdata/decode_samples.fqtest
vendored
Normal file
15
format/mp4/testdata/decode_samples.fqtest
vendored
Normal file
@ -0,0 +1,15 @@
|
||||
$ fq -o decode_samples=false -d mp4 '.tracks | dv' /aac.mp4
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.tracks[0:1]: 0x2c-0x291.7 (614)
|
||||
| | | [0]{}: track 0x2c-0x291.7 (614)
|
||||
| | | samples[0:4]: 0x2c-0x291.7 (614)
|
||||
0x020| de 02 00 4c| ...L| [0]: raw bits sample 0x2c-0xf8.7 (205)
|
||||
0x030|61 76 63 35 38 2e 39 31 2e 31 30 30 00 02 5c ab|avc58.91.100..\.|
|
||||
* |until 0xf8.7 (205) | |
|
||||
0x0f0| 01 22 98 da d8 3d d6| ."...=.| [1]: raw bits sample 0xf9-0x1d2.7 (218)
|
||||
0x100|93 80 76 db 22 13 6a 38 46 1c 9c 5e ae 85 f1 ab|..v.".j8F..^....|
|
||||
* |until 0x1d2.7 (218) | |
|
||||
0x1d0| 01 1a 99 a6 d3 21 41 ad 34 86 c8 cd 9a| .....!A.4....| [2]: raw bits sample 0x1d3-0x28c.7 (186)
|
||||
0x1e0|f0 3d 04 a1 e7 5f 1d 0c ff 81 d6 bd bc da b0 65|.=..._.........e|
|
||||
* |until 0x28c.7 (186) | |
|
||||
0x280| 01 18 81| ...| [3]: raw bits sample 0x28d-0x291.7 (5)
|
||||
0x290|b4 70 |.p |
|
||||
@ -18,8 +18,11 @@ func init() {
|
||||
Name: format.AAC_FRAME,
|
||||
Description: "Advanced Audio Coding frame",
|
||||
DecodeFn: aacDecode,
|
||||
RootArray: true,
|
||||
RootName: "elements",
|
||||
DecodeInArg: format.AACFrameIn{
|
||||
ObjectType: format.MPEGAudioObjectTypeMain,
|
||||
},
|
||||
RootArray: true,
|
||||
RootName: "elements",
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@ -15,8 +15,11 @@ func init() {
|
||||
Name: format.AVC_AU,
|
||||
Description: "H.264/AVC Access Unit",
|
||||
DecodeFn: avcAUDecode,
|
||||
RootArray: true,
|
||||
RootName: "access_unit",
|
||||
DecodeInArg: format.AvcAuIn{
|
||||
LengthSize: 4,
|
||||
},
|
||||
RootArray: true,
|
||||
RootName: "access_unit",
|
||||
Dependencies: []decode.Dependency{
|
||||
{Names: []string{format.AVC_NALU}, Group: &avcNALUFormat},
|
||||
},
|
||||
@ -24,7 +27,7 @@ func init() {
|
||||
}
|
||||
|
||||
func avcAUDecode(d *decode.D, in interface{}) interface{} {
|
||||
avcIn, ok := in.(format.AvcIn)
|
||||
avcIn, ok := in.(format.AvcAuIn)
|
||||
if !ok {
|
||||
d.Fatalf("avcIn required")
|
||||
}
|
||||
|
||||
@ -13,8 +13,11 @@ func init() {
|
||||
Name: format.HEVC_AU,
|
||||
Description: "H.265/HEVC Access Unit",
|
||||
DecodeFn: hevcAUDecode,
|
||||
RootArray: true,
|
||||
RootName: "access_unit",
|
||||
DecodeInArg: format.HevcAuIn{
|
||||
LengthSize: 4,
|
||||
},
|
||||
RootArray: true,
|
||||
RootName: "access_unit",
|
||||
Dependencies: []decode.Dependency{
|
||||
{Names: []string{format.HEVC_NALU}, Group: &hevcAUNALFormat},
|
||||
},
|
||||
@ -22,9 +25,9 @@ func init() {
|
||||
}
|
||||
|
||||
func hevcAUDecode(d *decode.D, in interface{}) interface{} {
|
||||
hevcIn, ok := in.(format.HevcIn)
|
||||
hevcIn, ok := in.(format.HevcAuIn)
|
||||
if !ok {
|
||||
d.Errorf("hevcIn required")
|
||||
d.Errorf("HevcAuIn required")
|
||||
}
|
||||
|
||||
for d.NotEnd() {
|
||||
|
||||
BIN
format/mpeg/testdata/aac_frame
vendored
Normal file
BIN
format/mpeg/testdata/aac_frame
vendored
Normal file
Binary file not shown.
19
format/mpeg/testdata/aac_frame.fqtest
vendored
Normal file
19
format/mpeg/testdata/aac_frame.fqtest
vendored
Normal file
@ -0,0 +1,19 @@
|
||||
$ fq -d aac_frame dv aac_frame
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.[0:4]: aac_frame (aac_frame) 0x0-0x14c.7 (333)
|
||||
| | | [0]{}: element 0x0-0x11.6 (17.7)
|
||||
0x000|de |. | syntax_element: "FIL" (6) 0x0-0x0.2 (0.3)
|
||||
| | | cnt{}: 0x0.3-0x1.6 (1.4)
|
||||
0x000|de |. | count: 15 0x0.3-0x0.6 (0.4)
|
||||
0x000|de 04 |.. | esc_count: 2 0x0.7-0x1.6 (1)
|
||||
| | | payload_length: 16 0x1.7-NA (0)
|
||||
| | | extension_payload{}: 0x1.7-0x11.6 (16)
|
||||
0x000| 04 00 | .. | extension_type: "EXT_FILL" (0) 0x1.7-0x2.2 (0.4)
|
||||
0x000| 00 | . | fill_nibble: 0 0x2.3-0x2.6 (0.4)
|
||||
0x000| 00 4c 61 76 63 35 38 2e 31 33 34 2e 31 30| .Lavc58.134.10| fill_byte: raw bits 0x2.7-0x11.6 (15)
|
||||
0x010|30 00 |0. |
|
||||
| | | [1]{}: element 0x11.7-0x12.1 (0.3)
|
||||
0x010| 00 42 | .B | syntax_element: "CPE" (1) 0x11.7-0x12.1 (0.3)
|
||||
0x010| 42 | B | [2]: raw bits byte_align 0x12.2-0x12.7 (0.6)
|
||||
0x010| 55 9f ff ff ff c0 01 29 68 a7 33 11 20| U......)h.3. | [3]: raw bits data 0x13-0x14c.7 (314)
|
||||
0x020|02 6a e5 c4 96 89 11 11 04 20 36 76 e1 e2 ee 35|.j....... 6v...5|
|
||||
* |until 0x14c.7 (end) (314) | |
|
||||
@ -13,7 +13,7 @@ import (
|
||||
"github.com/wader/fq/pkg/scalar"
|
||||
)
|
||||
|
||||
//go:embed *.jq
|
||||
//go:embed msgpack.jq
|
||||
var msgPackFS embed.FS
|
||||
|
||||
func init() {
|
||||
@ -22,7 +22,7 @@ func init() {
|
||||
Description: "MessagePack",
|
||||
DecodeFn: decodeMsgPack,
|
||||
Files: msgPackFS,
|
||||
ToRepr: "_msgpack_torepr",
|
||||
Functions: []string{"torepr", "_help"},
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@ -8,3 +8,9 @@ def _msgpack_torepr:
|
||||
elif .type | . == "bin8" or . == "bin16" or . == "bin32" then .value | tostring
|
||||
else .value | tovalue
|
||||
end;
|
||||
|
||||
def _msgpack__help:
|
||||
{ links: [
|
||||
{url: "https://github.com/msgpack/msgpack/blob/master/spec.md"}
|
||||
]
|
||||
};
|
||||
|
||||
@ -1,5 +0,0 @@
|
||||
Supports `torepr`:
|
||||
|
||||
```
|
||||
fq -d msgpack torepr file.msgpack
|
||||
```
|
||||
@ -3,6 +3,8 @@ package protobuf
|
||||
// https://developers.google.com/protocol-buffers/docs/encoding
|
||||
|
||||
import (
|
||||
"embed"
|
||||
|
||||
"github.com/wader/fq/format"
|
||||
"github.com/wader/fq/format/registry"
|
||||
"github.com/wader/fq/internal/mathextra"
|
||||
@ -10,11 +12,16 @@ import (
|
||||
"github.com/wader/fq/pkg/scalar"
|
||||
)
|
||||
|
||||
//go:embed protobuf.jq
|
||||
var protobufFS embed.FS
|
||||
|
||||
func init() {
|
||||
registry.MustRegister(decode.Format{
|
||||
Name: format.PROTOBUF,
|
||||
Description: "Protobuf",
|
||||
DecodeFn: protobufDecode,
|
||||
Functions: []string{"_help"},
|
||||
Files: protobufFS,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
8
format/protobuf/protobuf.jq
Normal file
8
format/protobuf/protobuf.jq
Normal file
@ -0,0 +1,8 @@
|
||||
def _protobuf__help:
|
||||
{ examples: [
|
||||
{comment: "Can be used to decode sub messages", shell: "fq -d protobuf '.fields[6].wire_value | protobuf | d'"}
|
||||
],
|
||||
links: [
|
||||
{url: "https://developers.google.com/protocol-buffers/docs/encoding"}
|
||||
]
|
||||
};
|
||||
@ -1,5 +0,0 @@
|
||||
`protobuf` decoder can be used to decode sub messages:
|
||||
|
||||
```
|
||||
fq -d protobuf '.fields[6].wire_value | protobuf | d'
|
||||
```
|
||||
@ -9,6 +9,7 @@ package rtmp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"embed"
|
||||
|
||||
"github.com/wader/fq/format"
|
||||
"github.com/wader/fq/format/registry"
|
||||
@ -20,6 +21,9 @@ import (
|
||||
var rtmpAmf0Group decode.Group
|
||||
var rtmpMpegASCFormat decode.Group
|
||||
|
||||
//go:embed rtmp.jq
|
||||
var rtmpFS embed.FS
|
||||
|
||||
func init() {
|
||||
registry.MustRegister(decode.Format{
|
||||
Name: format.RTMP,
|
||||
@ -32,6 +36,8 @@ func init() {
|
||||
{Names: []string{format.AMF0}, Group: &rtmpAmf0Group},
|
||||
{Names: []string{format.MPEG_ASC}, Group: &rtmpMpegASCFormat},
|
||||
},
|
||||
Functions: []string{"_help"},
|
||||
Files: rtmpFS,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
7
format/rtmp/rtmp.jq
Normal file
7
format/rtmp/rtmp.jq
Normal file
@ -0,0 +1,7 @@
|
||||
def _rtmp__help:
|
||||
{ notes: "Current only supports plain RTMP (not RTMPT or encrypted variants etc) with AMF0 (not AMF3).",
|
||||
links: [
|
||||
{url: "https://rtmp.veriskope.com/docs/spec/"},
|
||||
{url: "https://rtmp.veriskope.com/pdf/video_file_format_spec_v10.pdf"}
|
||||
]
|
||||
};
|
||||
@ -1 +0,0 @@
|
||||
Current only supports plain RTMP (not RTMPT or encrypted variants etc) with AMF0 (not AMF3).
|
||||
5
go.mod
5
go.mod
@ -18,6 +18,10 @@ require (
|
||||
// bump: gomod-gopacket command go get -d github.com/google/gopacket@v$LATEST && go mod tidy
|
||||
// bump: gomod-gopacket link "Release notes" https://github.com/google/gopacket/releases/tag/v$LATEST
|
||||
github.com/google/gopacket v1.1.19
|
||||
// bump: gomod-copystructure /github.com\/mitchellh\/copystructure v(.*)/ https://github.com/mitchellh/copystructure.git|^1
|
||||
// bump: gomod-copystructure command go get -d github.com/mitchellh/copystructure@v$LATEST && go mod tidy
|
||||
// bump: gomod-copystructure link "CHANGELOG" https://github.com/mitchellh/copystructure/blob/master/CHANGELOG.md
|
||||
github.com/mitchellh/copystructure v1.2.0
|
||||
// bump: gomod-mapstructure /github.com\/mitchellh\/mapstructure v(.*)/ https://github.com/mitchellh/mapstructure.git|^1
|
||||
// bump: gomod-mapstructure command go get -d github.com/mitchellh/mapstructure@v$LATEST && go mod tidy
|
||||
// bump: gomod-mapstructure link "CHANGELOG" https://github.com/mitchellh/mapstructure/blob/master/CHANGELOG.md
|
||||
@ -34,5 +38,6 @@ require (
|
||||
|
||||
require (
|
||||
github.com/itchyny/timefmt-go v0.1.3 // indirect
|
||||
github.com/mitchellh/reflectwalk v1.0.2 // indirect
|
||||
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect
|
||||
)
|
||||
|
||||
4
go.sum
4
go.sum
@ -7,8 +7,12 @@ github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921i
|
||||
github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A=
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
|
||||
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
|
||||
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
|
||||
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
|
||||
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/wader/gojq v0.12.1-0.20220419165350-574a3e400098 h1:lSY999PYhVjCCqhC6ltGkGY6HxHBYpJMvaVK3rt7sA4=
|
||||
|
||||
@ -140,12 +140,16 @@ func ToGoJQValue(v interface{}) (interface{}, bool) {
|
||||
return big.NewInt(vv), true
|
||||
case uint64:
|
||||
return new(big.Int).SetUint64(vv), true
|
||||
case float32:
|
||||
return float64(vv), true
|
||||
case float64:
|
||||
return vv, true
|
||||
case *big.Int:
|
||||
return vv, true
|
||||
case string:
|
||||
return vv, true
|
||||
case []byte:
|
||||
return string(vv), true
|
||||
case gojq.JQValue:
|
||||
return ToGoJQValue(vv.JQValueToGoJQ())
|
||||
case []interface{}:
|
||||
|
||||
@ -34,8 +34,8 @@ type Options struct {
|
||||
FillGaps bool
|
||||
IsRoot bool
|
||||
Range ranges.Range // if zero use whole buffer
|
||||
FormatOptions map[string]interface{}
|
||||
FormatInArg interface{}
|
||||
FormatInArgFn func(f Format) (interface{}, error)
|
||||
ReadBuf *[]byte
|
||||
}
|
||||
|
||||
@ -61,17 +61,31 @@ func decode(ctx context.Context, br bitio.ReaderAtSeeker, group Group, opts Opti
|
||||
|
||||
formatsErr := FormatsError{}
|
||||
|
||||
for _, g := range group {
|
||||
for _, f := range group {
|
||||
var formatInArg interface{}
|
||||
if opts.FormatInArgFn != nil {
|
||||
var err error
|
||||
formatInArg, err = opts.FormatInArgFn(f)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
} else {
|
||||
formatInArg = opts.FormatInArg
|
||||
if formatInArg == nil {
|
||||
formatInArg = f.DecodeInArg
|
||||
}
|
||||
}
|
||||
|
||||
cBR, err := bitioextra.Range(br, decodeRange.Start, decodeRange.Len)
|
||||
if err != nil {
|
||||
return nil, nil, IOError{Err: err, Op: "BitBufRange", ReadSize: decodeRange.Len, Pos: decodeRange.Start}
|
||||
}
|
||||
|
||||
d := newDecoder(ctx, g, cBR, opts)
|
||||
d := newDecoder(ctx, f, cBR, opts)
|
||||
|
||||
var decodeV interface{}
|
||||
r, rOk := recoverfn.Run(func() {
|
||||
decodeV = g.DecodeFn(d, opts.FormatInArg)
|
||||
decodeV = f.DecodeFn(d, formatInArg)
|
||||
})
|
||||
|
||||
if ctx != nil && ctx.Err() != nil {
|
||||
@ -83,7 +97,7 @@ func decode(ctx context.Context, br bitio.ReaderAtSeeker, group Group, opts Opti
|
||||
panicErr, _ := re.(error)
|
||||
formatErr := FormatError{
|
||||
Err: panicErr,
|
||||
Format: g,
|
||||
Format: f,
|
||||
Stacktrace: r,
|
||||
}
|
||||
formatsErr.Errs = append(formatsErr.Errs, formatErr)
|
||||
|
||||
@ -10,16 +10,40 @@ type Dependency struct {
|
||||
}
|
||||
|
||||
type Format struct {
|
||||
Name string
|
||||
ProbeOrder int // probe order is from low to hi value then by name
|
||||
Description string
|
||||
Groups []string
|
||||
DecodeFn func(d *D, in interface{}) interface{}
|
||||
RootArray bool
|
||||
RootName string
|
||||
Dependencies []Dependency
|
||||
Files fs.ReadDirFS
|
||||
ToRepr string
|
||||
Name string
|
||||
ProbeOrder int // probe order is from low to hi value then by name
|
||||
Description string
|
||||
Groups []string
|
||||
DecodeFn func(d *D, in interface{}) interface{}
|
||||
DecodeInArg interface{}
|
||||
DecodeOutType interface{}
|
||||
RootArray bool
|
||||
RootName string
|
||||
Dependencies []Dependency
|
||||
Files fs.ReadDirFS
|
||||
Help FormatHelp
|
||||
Functions []string
|
||||
}
|
||||
|
||||
type HelpExample struct {
|
||||
Comment string
|
||||
Code string
|
||||
}
|
||||
|
||||
type HelpFunction struct {
|
||||
Name string
|
||||
Examples []HelpExample
|
||||
}
|
||||
|
||||
type HelpReference struct {
|
||||
Title string
|
||||
URL string
|
||||
}
|
||||
|
||||
type FormatHelp struct {
|
||||
Notes string
|
||||
Functions []HelpFunction
|
||||
References []HelpReference
|
||||
}
|
||||
|
||||
func FormatFn(d func(d *D, in interface{}) interface{}) Group {
|
||||
|
||||
@ -6,6 +6,7 @@ def _eval($expr; $filename): empty;
|
||||
def _eval($expr): empty;
|
||||
def _extkeys: empty;
|
||||
def _exttype: empty;
|
||||
def _format_func($format; $func): empty;
|
||||
def _global_state: empty;
|
||||
def _global_state($v): empty;
|
||||
def _hexdump($opts): empty;
|
||||
@ -28,5 +29,7 @@ def _tobits($bits; $is_range; $pad): empty;
|
||||
def _tovalue: empty;
|
||||
def _tovalue($opts): empty;
|
||||
def base64: empty;
|
||||
def format: empty;
|
||||
def open: empty;
|
||||
def scope: empty;
|
||||
def torepr: empty;
|
||||
|
||||
@ -7,9 +7,11 @@ import (
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math/big"
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mitchellh/copystructure"
|
||||
"github.com/mitchellh/mapstructure"
|
||||
"github.com/wader/fq/internal/bitioextra"
|
||||
"github.com/wader/fq/internal/gojqextra"
|
||||
@ -75,7 +77,6 @@ func (i *Interp) _registry(c interface{}, a []interface{}) interface{} {
|
||||
"probe_order": f.ProbeOrder,
|
||||
"root_name": f.RootName,
|
||||
"root_array": f.RootArray,
|
||||
"to_repr": f.ToRepr,
|
||||
}
|
||||
|
||||
var dependenciesVs []interface{}
|
||||
@ -96,9 +97,33 @@ func (i *Interp) _registry(c interface{}, a []interface{}) interface{} {
|
||||
if len(groupsVs) > 0 {
|
||||
vf["groups"] = groupsVs
|
||||
}
|
||||
if f.DecodeInArg != nil {
|
||||
doc := map[string]interface{}{}
|
||||
st := reflect.TypeOf(f.DecodeInArg)
|
||||
for i := 0; i < st.NumField(); i++ {
|
||||
f := st.Field(i)
|
||||
if v, ok := f.Tag.Lookup("doc"); ok {
|
||||
doc[camelToSnake(f.Name)] = v
|
||||
}
|
||||
}
|
||||
vf["decode_in_arg_doc"] = doc
|
||||
|
||||
args, err := structToMap(f.DecodeInArg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// filter out internal field without documentation
|
||||
for k := range args {
|
||||
if _, ok := doc[k]; !ok {
|
||||
delete(args, k)
|
||||
}
|
||||
}
|
||||
vf["decode_in_arg"] = args
|
||||
}
|
||||
|
||||
if f.Files != nil {
|
||||
files := map[string]interface{}{}
|
||||
files := []interface{}{}
|
||||
|
||||
entries, err := f.Files.ReadDir(".")
|
||||
if err != nil {
|
||||
@ -114,12 +139,24 @@ func (i *Interp) _registry(c interface{}, a []interface{}) interface{} {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
files[e.Name()] = string(b)
|
||||
|
||||
files = append(files, map[string]interface{}{
|
||||
"name": e.Name(),
|
||||
"data": string(b),
|
||||
})
|
||||
}
|
||||
|
||||
vf["files"] = files
|
||||
}
|
||||
|
||||
if f.Functions != nil {
|
||||
var ss []interface{}
|
||||
for _, f := range f.Functions {
|
||||
ss = append(ss, f)
|
||||
}
|
||||
vf["functions"] = ss
|
||||
}
|
||||
|
||||
formats[f.Name] = vf
|
||||
}
|
||||
|
||||
@ -139,19 +176,20 @@ func (i *Interp) _toValue(c interface{}, a []interface{}) interface{} {
|
||||
|
||||
func (i *Interp) _decode(c interface{}, a []interface{}) interface{} {
|
||||
var opts struct {
|
||||
Filename string `mapstructure:"filename"`
|
||||
Force bool `mapstructure:"force"`
|
||||
Progress string `mapstructure:"_progress"`
|
||||
Remain map[string]interface{} `mapstructure:",remain"`
|
||||
}
|
||||
_ = mapstructure.Decode(a[1], &opts)
|
||||
|
||||
var filename string
|
||||
|
||||
// TODO: progress hack
|
||||
// would be nice to move all progress code into decode but it might be
|
||||
// tricky to keep track of absolute positions in the underlaying readers
|
||||
// when it uses BitBuf slices, maybe only in Pos()?
|
||||
if bbf, ok := c.(*openFile); ok {
|
||||
opts.Filename = bbf.filename
|
||||
filename = bbf.filename
|
||||
|
||||
if opts.Progress != "" {
|
||||
evalProgress := func(c interface{}) {
|
||||
@ -201,14 +239,35 @@ func (i *Interp) _decode(c interface{}, a []interface{}) interface{} {
|
||||
return err
|
||||
}
|
||||
|
||||
dv, _, err := decode.Decode(i.evalInstance.ctx, bv.br, decodeFormat,
|
||||
dv, formatOut, err := decode.Decode(i.evalInstance.ctx, bv.br, decodeFormat,
|
||||
decode.Options{
|
||||
IsRoot: true,
|
||||
FillGaps: true,
|
||||
Force: opts.Force,
|
||||
Range: bv.r,
|
||||
Description: opts.Filename,
|
||||
FormatOptions: opts.Remain,
|
||||
IsRoot: true,
|
||||
FillGaps: true,
|
||||
Force: opts.Force,
|
||||
Range: bv.r,
|
||||
Description: filename,
|
||||
FormatInArgFn: func(f decode.Format) (interface{}, error) {
|
||||
inArg := f.DecodeInArg
|
||||
if inArg == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var err error
|
||||
inArg, err = copystructure.Copy(inArg)
|
||||
if err != nil {
|
||||
return f.DecodeInArg, err
|
||||
}
|
||||
|
||||
if len(opts.Remain) > 0 {
|
||||
if err := mapToStruct(opts.Remain, &inArg); err != nil {
|
||||
// TODO: currently ignores failed struct mappings
|
||||
//nolint: nilerr
|
||||
return f.DecodeInArg, nil
|
||||
}
|
||||
}
|
||||
|
||||
return inArg, nil
|
||||
},
|
||||
},
|
||||
)
|
||||
if dv == nil {
|
||||
@ -225,7 +284,16 @@ func (i *Interp) _decode(c interface{}, a []interface{}) interface{} {
|
||||
return valueError{err}
|
||||
}
|
||||
|
||||
return makeDecodeValue(dv)
|
||||
var formatOutMap interface{}
|
||||
|
||||
if formatOut != nil {
|
||||
formatOutMap, err = structToMap(formatOut)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return makeDecodeValueOut(dv, formatOutMap)
|
||||
}
|
||||
|
||||
func valueKey(name string, a, b func(name string) interface{}) interface{} {
|
||||
@ -260,12 +328,16 @@ func toValue(optsFn func() Options, v interface{}) (interface{}, bool) {
|
||||
}
|
||||
|
||||
func makeDecodeValue(dv *decode.Value) interface{} {
|
||||
return makeDecodeValueOut(dv, nil)
|
||||
}
|
||||
|
||||
func makeDecodeValueOut(dv *decode.Value, out interface{}) interface{} {
|
||||
switch vv := dv.V.(type) {
|
||||
case *decode.Compound:
|
||||
if vv.IsArray {
|
||||
return NewArrayDecodeValue(dv, vv)
|
||||
return NewArrayDecodeValue(dv, out, vv)
|
||||
}
|
||||
return NewStructDecodeValue(dv, vv)
|
||||
return NewStructDecodeValue(dv, out, vv)
|
||||
case *scalar.S:
|
||||
switch vv := vv.Value().(type) {
|
||||
case bitio.ReaderAtSeeker:
|
||||
@ -288,58 +360,58 @@ func makeDecodeValue(dv *decode.Value) interface{} {
|
||||
return gojqextra.String([]rune(buf.String())), nil
|
||||
},
|
||||
},
|
||||
decodeValueBase: decodeValueBase{dv},
|
||||
decodeValueBase: decodeValueBase{dv: dv},
|
||||
bitsFormat: true,
|
||||
}
|
||||
case bool:
|
||||
return decodeValue{
|
||||
JQValue: gojqextra.Boolean(vv),
|
||||
decodeValueBase: decodeValueBase{dv},
|
||||
decodeValueBase: decodeValueBase{dv: dv},
|
||||
}
|
||||
case int:
|
||||
return decodeValue{
|
||||
JQValue: gojqextra.Number{V: vv},
|
||||
decodeValueBase: decodeValueBase{dv},
|
||||
decodeValueBase: decodeValueBase{dv: dv},
|
||||
}
|
||||
case int64:
|
||||
return decodeValue{
|
||||
JQValue: gojqextra.Number{V: big.NewInt(vv)},
|
||||
decodeValueBase: decodeValueBase{dv},
|
||||
decodeValueBase: decodeValueBase{dv: dv},
|
||||
}
|
||||
case uint64:
|
||||
return decodeValue{
|
||||
JQValue: gojqextra.Number{V: new(big.Int).SetUint64(vv)},
|
||||
decodeValueBase: decodeValueBase{dv},
|
||||
decodeValueBase: decodeValueBase{dv: dv},
|
||||
}
|
||||
case float64:
|
||||
return decodeValue{
|
||||
JQValue: gojqextra.Number{V: vv},
|
||||
decodeValueBase: decodeValueBase{dv},
|
||||
decodeValueBase: decodeValueBase{dv: dv},
|
||||
}
|
||||
case string:
|
||||
return decodeValue{
|
||||
JQValue: gojqextra.String(vv),
|
||||
decodeValueBase: decodeValueBase{dv},
|
||||
decodeValueBase: decodeValueBase{dv: dv},
|
||||
}
|
||||
case []interface{}:
|
||||
return decodeValue{
|
||||
JQValue: gojqextra.Array(vv),
|
||||
decodeValueBase: decodeValueBase{dv},
|
||||
decodeValueBase: decodeValueBase{dv: dv},
|
||||
}
|
||||
case map[string]interface{}:
|
||||
return decodeValue{
|
||||
JQValue: gojqextra.Object(vv),
|
||||
decodeValueBase: decodeValueBase{dv},
|
||||
decodeValueBase: decodeValueBase{dv: dv},
|
||||
}
|
||||
case nil:
|
||||
return decodeValue{
|
||||
JQValue: gojqextra.Null{},
|
||||
decodeValueBase: decodeValueBase{dv},
|
||||
decodeValueBase: decodeValueBase{dv: dv},
|
||||
}
|
||||
case *big.Int:
|
||||
return decodeValue{
|
||||
JQValue: gojqextra.Number{V: vv},
|
||||
decodeValueBase: decodeValueBase{dv},
|
||||
decodeValueBase: decodeValueBase{dv: dv},
|
||||
}
|
||||
default:
|
||||
panic(fmt.Sprintf("unreachable vv %#+v", vv))
|
||||
@ -350,7 +422,8 @@ func makeDecodeValue(dv *decode.Value) interface{} {
|
||||
}
|
||||
|
||||
type decodeValueBase struct {
|
||||
dv *decode.Value
|
||||
dv *decode.Value
|
||||
out interface{}
|
||||
}
|
||||
|
||||
func (dvb decodeValueBase) DecodeValue() *decode.Value {
|
||||
@ -386,6 +459,7 @@ func (dvb decodeValueBase) ExtKeys() []string {
|
||||
kv = append(kv,
|
||||
"_error",
|
||||
"_format",
|
||||
"_out",
|
||||
)
|
||||
|
||||
if dvb.dv.Index != -1 {
|
||||
@ -502,6 +576,8 @@ func (dvb decodeValueBase) JQValueKey(name string) interface{} {
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
case "_out":
|
||||
return dvb.out
|
||||
case "_unknown":
|
||||
switch vv := dv.V.(type) {
|
||||
case *scalar.S:
|
||||
@ -568,9 +644,9 @@ type ArrayDecodeValue struct {
|
||||
*decode.Compound
|
||||
}
|
||||
|
||||
func NewArrayDecodeValue(dv *decode.Value, c *decode.Compound) ArrayDecodeValue {
|
||||
func NewArrayDecodeValue(dv *decode.Value, out interface{}, c *decode.Compound) ArrayDecodeValue {
|
||||
return ArrayDecodeValue{
|
||||
decodeValueBase: decodeValueBase{dv},
|
||||
decodeValueBase: decodeValueBase{dv: dv, out: out},
|
||||
Base: gojqextra.Base{Typ: "array"},
|
||||
Compound: c,
|
||||
}
|
||||
@ -639,9 +715,9 @@ type StructDecodeValue struct {
|
||||
*decode.Compound
|
||||
}
|
||||
|
||||
func NewStructDecodeValue(dv *decode.Value, c *decode.Compound) StructDecodeValue {
|
||||
func NewStructDecodeValue(dv *decode.Value, out interface{}, c *decode.Compound) StructDecodeValue {
|
||||
return StructDecodeValue{
|
||||
decodeValueBase: decodeValueBase{dv},
|
||||
decodeValueBase: decodeValueBase{dv: dv, out: out},
|
||||
Base: gojqextra.Base{Typ: "object"},
|
||||
Compound: c,
|
||||
}
|
||||
|
||||
@ -37,16 +37,17 @@ def decode($name; $decode_opts):
|
||||
( options as $opts
|
||||
| _decode(
|
||||
$name;
|
||||
$opts +
|
||||
{
|
||||
_progress: (
|
||||
if $opts.decode_progress and $opts.repl and stdout_tty.is_terminal then
|
||||
"_decode_progress"
|
||||
else null
|
||||
end
|
||||
),
|
||||
} +
|
||||
$decode_opts
|
||||
( {
|
||||
_progress: (
|
||||
if $opts.decode_progress and $opts.repl and stdout_tty.is_terminal then
|
||||
"_decode_progress"
|
||||
else null
|
||||
end
|
||||
),
|
||||
}
|
||||
+ $opts
|
||||
+ $decode_opts
|
||||
)
|
||||
)
|
||||
);
|
||||
def decode($name): decode($name; {});
|
||||
|
||||
9
pkg/interp/format_decode.jq
Normal file
9
pkg/interp/format_decode.jq
Normal file
@ -0,0 +1,9 @@
|
||||
# note this is a "dynamic" include, output string will be used as source
|
||||
|
||||
[ _registry.groups
|
||||
| to_entries[]
|
||||
# TODO: nicer way to skip "all" which also would override builtin all/*
|
||||
| select(.key != "all")
|
||||
| "def \(.key)($opts): decode(\(.key | tojson); $opts);"
|
||||
, "def \(.key): decode(\(.key | tojson); {});"
|
||||
] | join("\n")
|
||||
13
pkg/interp/format_func.jq
Normal file
13
pkg/interp/format_func.jq
Normal file
@ -0,0 +1,13 @@
|
||||
# note this is a "dynamic" include, output string will be used as source
|
||||
|
||||
[ "def _format_func($format; $func):"
|
||||
, " ( [$format, $func] as $ff"
|
||||
, " | if false then error(\"unreachable\")"
|
||||
, ( _registry.formats[] as $f
|
||||
| $f.functions[]?
|
||||
| " elif $ff == \([$f.name, .] | tojson) then _\($f.name)_\(.)"
|
||||
)
|
||||
, " else error(\"\\($format) has no \\($func)\")"
|
||||
, " end"
|
||||
, " );"
|
||||
] | join("\n")
|
||||
8
pkg/interp/format_include.jq
Normal file
8
pkg/interp/format_include.jq
Normal file
@ -0,0 +1,8 @@
|
||||
# note this is a "dynamic" include, output string will be used as source
|
||||
|
||||
[ _registry.formats[]
|
||||
| select(.files)
|
||||
| .files[]
|
||||
| select(.name | endswith(".jq"))
|
||||
| .data
|
||||
] | join("\n")
|
||||
@ -1,31 +0,0 @@
|
||||
# note this is a "dynamic" include, outputted string will be used as source
|
||||
|
||||
def _formats_source:
|
||||
( [ ( _registry.groups
|
||||
| to_entries[]
|
||||
# TODO: nicer way to skip "all" which also would override builtin all/*
|
||||
| select(.key != "all")
|
||||
| "def \(.key)($opts): decode(\(.key | tojson); $opts);"
|
||||
, "def \(.key): decode(\(.key | tojson); {});"
|
||||
)
|
||||
, ( _registry.formats[]
|
||||
| select(.files)
|
||||
| .files[]
|
||||
)
|
||||
, ( "def torepr:"
|
||||
, " ( format as $f"
|
||||
, " | if $f == null then error(\"value is not a format root\") end"
|
||||
, " | if false then error(\"unreachable\")"
|
||||
, ( _registry.formats[]
|
||||
| select(.to_repr != "")
|
||||
| " elif $f == \(.name | tojson) then \(.to_repr)"
|
||||
)
|
||||
, " else error(\"format has no torepr\")"
|
||||
, " end"
|
||||
, " );"
|
||||
)
|
||||
]
|
||||
| join("\n")
|
||||
);
|
||||
|
||||
_formats_source
|
||||
@ -1,7 +1,6 @@
|
||||
include "internal";
|
||||
include "options";
|
||||
include "binary";
|
||||
include "ansi";
|
||||
|
||||
def _display_default_opts:
|
||||
options({depth: 1});
|
||||
@ -30,37 +29,6 @@ def hd: hexdump;
|
||||
|
||||
def intdiv(a; b): _intdiv(a; b);
|
||||
|
||||
# TODO: escape for safe key names
|
||||
# path ["a", 1, "b"] -> "a[1].b"
|
||||
def path_to_expr($opts):
|
||||
( if length == 0 or (.[0] | type) != "string" then
|
||||
[""] + .
|
||||
end
|
||||
| map(
|
||||
if type == "number" then
|
||||
( ("[" | _ansi_if($opts; "array"))
|
||||
, _ansi_if($opts; "number")
|
||||
, ("]" | _ansi_if($opts; "array"))
|
||||
) else
|
||||
( "."
|
||||
, # empty (special case for leading index or empty path) or key
|
||||
if . == "" or _is_ident then _ansi_if($opts; "objectkey")
|
||||
else
|
||||
"\"\(_escape_ident)\"" | _ansi_if($opts; "string")
|
||||
end
|
||||
)
|
||||
end
|
||||
)
|
||||
| join("")
|
||||
);
|
||||
def path_to_expr: path_to_expr(null);
|
||||
|
||||
# TODO: don't use eval? should support '.a.b[1]."c.c"' and escapes?
|
||||
def expr_to_path:
|
||||
( if type != "string" then error("require string argument") end
|
||||
| _eval("null | path(\(.))")
|
||||
);
|
||||
|
||||
def trim: capture("^\\s*(?<str>.*?)\\s*$"; "").str;
|
||||
|
||||
# does +1 and [:1] as " "*0 is null
|
||||
@ -133,58 +101,6 @@ def chunk($size):
|
||||
]
|
||||
end;
|
||||
|
||||
# helper to build path query/generate functions for tree structures with
|
||||
# non-unique children, ex: mp4_path
|
||||
def tree_path(children; name; $v):
|
||||
def _lookup:
|
||||
# add implicit zeros to get first value
|
||||
# ["a", "b", 1] => ["a", 0, "b", 1]
|
||||
def _normalize_path:
|
||||
( . as $np
|
||||
| if $np | last | type == "string" then $np+[0] end
|
||||
# state is [path acc, possible pending zero index]
|
||||
| ( reduce .[] as $np ([[], []];
|
||||
if $np | type == "string" then
|
||||
[(.[0]+.[1]+[$np]), [0]]
|
||||
else
|
||||
[.[0]+[$np], []]
|
||||
end
|
||||
))
|
||||
)[0];
|
||||
( . as $c
|
||||
| $v
|
||||
| expr_to_path
|
||||
| _normalize_path
|
||||
| reduce .[] as $n ($c;
|
||||
if $n | type == "string" then
|
||||
children | map(select(name == $n))
|
||||
else
|
||||
.[$n]
|
||||
end
|
||||
)
|
||||
);
|
||||
def _path:
|
||||
[ . as $r
|
||||
| $v._path as $p
|
||||
| foreach range(($p | length)/2) as $i (null; null;
|
||||
( ($r | getpath($p[0:($i+1)*2]) | name) as $name
|
||||
| [($r | getpath($p[0:($i+1)*2-1]))[] | name][0:$p[($i*2)+1]+1] as $before
|
||||
| [ $name
|
||||
, ($before | map(select(. == $name)) | length)-1
|
||||
]
|
||||
)
|
||||
)
|
||||
| [ ".", .[0],
|
||||
(.[1] | if . == 0 then empty else "[", ., "]" end)
|
||||
]
|
||||
]
|
||||
| flatten
|
||||
| join("");
|
||||
if $v | type == "string" then _lookup
|
||||
else _path
|
||||
end;
|
||||
|
||||
|
||||
# [{a: 123, ...}, ...]
|
||||
# colmap maps something into [col, ...]
|
||||
# render maps [{column: 0, string: "coltext", maxwidth: 12}, ..] into a row
|
||||
@ -323,3 +239,102 @@ def paste:
|
||||
| join("")
|
||||
)
|
||||
end;
|
||||
|
||||
def tojq($style):
|
||||
def _is_ident: test("^[a-zA-Z_][a-zA-Z_0-9]*$");
|
||||
def _key: if _is_ident | not then tojson end;
|
||||
def _f($style):
|
||||
def _r($indent):
|
||||
( type as $t
|
||||
| if $t == "null" then tojson
|
||||
elif $t == "string" then tojson
|
||||
elif $t == "number" then tojson
|
||||
elif $t == "boolean" then tojson
|
||||
elif $t == "array" then
|
||||
[ "[", $style.compound_newline
|
||||
, ( [ .[]
|
||||
| $indent, $style.indent
|
||||
, _r($indent+$style.indent), $style.array_sep
|
||||
]
|
||||
| .[0:-1]
|
||||
)
|
||||
, $style.compound_newline
|
||||
, $indent, "]"
|
||||
]
|
||||
elif $t == "object" then
|
||||
[ "{", $style.compound_newline
|
||||
, ( [ to_entries[]
|
||||
| $indent, $style.indent
|
||||
, (.key | _key), $style.key_sep
|
||||
, (.value | _r($indent+$style.indent)), $style.value_sep
|
||||
]
|
||||
| .[0:-1]
|
||||
)
|
||||
, $style.compound_newline
|
||||
, $indent, "}"
|
||||
]
|
||||
else error("unknown type \($t)")
|
||||
end
|
||||
);
|
||||
_r("");
|
||||
( {
|
||||
compact: {
|
||||
indent: "",
|
||||
key_sep: ":",
|
||||
value_sep: ",",
|
||||
array_sep: ",",
|
||||
compound_newline: "",
|
||||
},
|
||||
fancy_compact: {
|
||||
indent: "",
|
||||
key_sep: ": ",
|
||||
value_sep: ", ",
|
||||
array_sep: ", ",
|
||||
compound_newline: "",
|
||||
},
|
||||
verbose: {
|
||||
indent: " ",
|
||||
key_sep: ": ",
|
||||
value_sep: ",\n",
|
||||
array_sep: ",\n",
|
||||
compound_newline: "\n",
|
||||
}
|
||||
} as $styles
|
||||
| _f(
|
||||
( $style // "compact"
|
||||
| if type == "string" then $styles[.]
|
||||
elif type == "object" then .
|
||||
else error("invalid style")
|
||||
end
|
||||
)
|
||||
)
|
||||
| flatten
|
||||
| join("")
|
||||
);
|
||||
def tojq: tojq(null);
|
||||
|
||||
# very simple markdown to text converter
|
||||
# assumes very basic markdown as input
|
||||
def _markdown_to_text:
|
||||
( .
|
||||
# ```
|
||||
# code
|
||||
# ```
|
||||
# -> code
|
||||
| gsub("\\n```\\n"; "\n"; "m")
|
||||
# #, ##, ###, ... -> #
|
||||
| gsub("(?<line>\\n)?#+(?<title>.*)\\n"; "\(.line // "")#\(.title)\n"; "m")
|
||||
# [title](url) -> title (url)
|
||||
| gsub("\\[(?<title>.*)\\]\\((?<url>.*)\\)"; "\(.title) (\(.url))")
|
||||
# `code` -> code
|
||||
| gsub("`(?<code>.*)`"; .code)
|
||||
);
|
||||
|
||||
def expr_to_path: _expr_to_path;
|
||||
def path_to_expr: _path_to_expr;
|
||||
|
||||
def torepr:
|
||||
( format as $f
|
||||
| if $f == null then error("value is not a format root") end
|
||||
| _format_func($f; "torepr")
|
||||
);
|
||||
|
||||
@ -56,6 +56,33 @@ Same as recurse without argument.
|
||||
def help($_): error("help must be alone or last in pipeline. ex: help(length) or ... | help");
|
||||
def help: help(null);
|
||||
|
||||
def _help_format_enrich($arg0; $f; $include_basic):
|
||||
( if $include_basic then
|
||||
.examples +=
|
||||
[ {comment: "Decode file as \($f.name)", shell: "fq -d \($f.name) file"}
|
||||
, {comment: "Decode value as \($f.name)", expr: "\($f.name)"}
|
||||
]
|
||||
end
|
||||
| (($f.functions // []) | map(select(startswith("_") | not))) as $public_functions
|
||||
| if ($public_functions | length) > 0 then
|
||||
.examples +=
|
||||
[ $public_functions[]
|
||||
| {comment: "Supports `\(.)`", shell: "fq -d \($f.name) torepr file"}
|
||||
, {comment: "Supports `\(.)`", expr: "\($f.name) | torepr"}
|
||||
]
|
||||
end
|
||||
| if $f.decode_in_arg then
|
||||
.examples +=
|
||||
[ { comment: "Decode file using options"
|
||||
, shell: "\($arg0) -d \($f.name)\($f.decode_in_arg | to_entries | map(" -o ", .key, "=", (.value | tojson)) | join("")) file"
|
||||
}
|
||||
, { comment: "Decode value as \($f.name)"
|
||||
, expr: "\($f.name)(\($f.decode_in_arg | tojq("fancy_compact")))"
|
||||
}
|
||||
]
|
||||
end
|
||||
);
|
||||
|
||||
def _help($arg0; $topic):
|
||||
( $topic
|
||||
| if . == "usage" then
|
||||
@ -76,9 +103,9 @@ def _help($arg0; $topic):
|
||||
)
|
||||
elif . == "args" then
|
||||
args_help_text(_opt_cli_opts)
|
||||
elif . == "options" then
|
||||
elif . == "options" then
|
||||
( [ ( options
|
||||
| _opt_cli_arg_fromoptions
|
||||
| _opt_cli_arg_from_options
|
||||
)
|
||||
| to_entries[]
|
||||
| [(.key+" "), .value | tostring]
|
||||
@ -97,9 +124,9 @@ def _help($arg0; $topic):
|
||||
)
|
||||
elif . == "formats" then
|
||||
( [ formats
|
||||
| to_entries[]
|
||||
| [(.key+" "), .value.description]
|
||||
]
|
||||
| to_entries[]
|
||||
| [(.key+" "), .value.description]
|
||||
]
|
||||
| table(
|
||||
.;
|
||||
map(
|
||||
@ -112,8 +139,98 @@ def _help($arg0; $topic):
|
||||
) | join("")
|
||||
)
|
||||
)
|
||||
elif _registry.formats | has($topic) then
|
||||
( _registry.formats[$topic] as $f
|
||||
| (_format_func($f.name; "_help")? // {} | _help_format_enrich($arg0; $f; true)) as $fhelp
|
||||
| "\($f.name): \($f.description) decoder"
|
||||
, ($fhelp.notes | if . then _markdown_to_text else empty end)
|
||||
, if $f.decode_in_arg then
|
||||
( $f.decode_in_arg
|
||||
| to_entries
|
||||
| map([" \(.key)=\(.value) ", $f.decode_in_arg_doc[.key]])
|
||||
| "Options:"
|
||||
, table(
|
||||
.;
|
||||
map(
|
||||
( . as $rc
|
||||
# right pad format name to align description
|
||||
| if .column == 0 then .string | rpad(" "; $rc.maxwidth)
|
||||
else $rc.string
|
||||
end
|
||||
)
|
||||
) | join("")
|
||||
)
|
||||
)
|
||||
else empty
|
||||
end
|
||||
, "Examples:"
|
||||
, ( $fhelp.examples[]
|
||||
| " # \(.comment | _markdown_to_text)"
|
||||
, if .shell then " $ \(.shell)"
|
||||
elif .expr then " ... | \(.expr)"
|
||||
else empty
|
||||
end
|
||||
)
|
||||
, if isempty($f.functions | select(. == "torepr")) | not then
|
||||
( "Supports torepr:"
|
||||
, " ... | \($f.name) | torepr"
|
||||
)
|
||||
else empty
|
||||
end
|
||||
, if $fhelp.links then
|
||||
( "References and links"
|
||||
, ( $fhelp.links[]
|
||||
| if .title then " \(.title) \(.url)"
|
||||
else " \(.url)"
|
||||
end
|
||||
)
|
||||
)
|
||||
else empty
|
||||
end
|
||||
)
|
||||
elif _help_functions | has($topic) then
|
||||
( _help_functions[$topic] as $hf
|
||||
| "\($topic): \($hf.summary)"
|
||||
, $hf.doc
|
||||
, if $hf.examples then
|
||||
( "Examples:"
|
||||
, ( $hf.examples[]
|
||||
| . as $e
|
||||
| if length == 1 then
|
||||
( "> \($e[0])"
|
||||
, (null | try (_eval($e[0]) | tojson) catch "error: \(.)")
|
||||
)
|
||||
else
|
||||
( "> \($e[0] | tojson) | \($e[1])"
|
||||
, ($e[0] | try (_eval($e[1]) | tojson) catch "error: \(.)")
|
||||
)
|
||||
end
|
||||
)
|
||||
)
|
||||
end
|
||||
)
|
||||
else
|
||||
error("unknown topic: \($topic)")
|
||||
# help(unknown)
|
||||
# TODO: check builtin
|
||||
( ( . # TODO: extract
|
||||
| builtins
|
||||
| map(split("/") | {key: .[0], value: true})
|
||||
| from_entries
|
||||
) as $builtins
|
||||
| ( . # TODO: extract
|
||||
| scope
|
||||
| map({key: ., value: true})
|
||||
| from_entries
|
||||
) as $scope
|
||||
| if $builtins | has($topic) then
|
||||
"\($topic) is builtin function"
|
||||
elif $scope | has($topic) then
|
||||
"\($topic) is a function or variable"
|
||||
else
|
||||
"don't know what \($topic) is "
|
||||
end
|
||||
| println
|
||||
)
|
||||
end
|
||||
);
|
||||
|
||||
@ -121,6 +238,7 @@ def _help($arg0; $topic):
|
||||
def _help_slurp($query):
|
||||
def _name:
|
||||
if _query_is_func then _query_func_name
|
||||
elif _query_is_string then _query_string_str
|
||||
else _query_tostring
|
||||
end;
|
||||
if $query.orig | _query_is_func then
|
||||
@ -129,60 +247,16 @@ def _help($arg0; $topic):
|
||||
| if $args == null then
|
||||
# help
|
||||
( "Type expression to evaluate"
|
||||
, "help(...) Help for topic. Ex: help(mp4), help(\"mp4\")"
|
||||
, "\\t Completion"
|
||||
, "Up/Down History"
|
||||
, "^C Interrupt execution"
|
||||
, "... | repl Start a new REPL"
|
||||
, "^C Interrupt execution"
|
||||
, "^D Exit REPL"
|
||||
) | println
|
||||
elif $argc == 1 then
|
||||
# help(...)
|
||||
( ($args[0] | _name) as $name
|
||||
| _help_functions[$name] as $hf
|
||||
| if $hf then
|
||||
# help(name)
|
||||
( "\($name): \($hf.summary)"
|
||||
, $hf.doc
|
||||
, if $hf.examples then
|
||||
( "Examples:"
|
||||
, ( $hf.examples[]
|
||||
| . as $e
|
||||
| if length == 1 then
|
||||
( "> \($e[0])"
|
||||
, (null | try (_eval($e[0]) | tojson) catch "error: \(.)")
|
||||
)
|
||||
else
|
||||
( "> \($e[0] | tojson) | \($e[1])"
|
||||
, ($e[0] | try (_eval($e[1]) | tojson) catch "error: \(.)")
|
||||
)
|
||||
end
|
||||
)
|
||||
)
|
||||
end
|
||||
) | println
|
||||
else
|
||||
# help(unknown)
|
||||
# TODO: check builtin
|
||||
( ( . # TODO: extract
|
||||
| builtins
|
||||
| map(split("/") | {key: .[0], value: true})
|
||||
| from_entries
|
||||
) as $builtins
|
||||
| ( . # TODO: extract
|
||||
| scope
|
||||
| map({key: ., value: true})
|
||||
| from_entries
|
||||
) as $scope
|
||||
| if $builtins | has($name) then
|
||||
"\($name) is builtin function"
|
||||
elif $scope | has($name) then
|
||||
"\($name) is a function or variable"
|
||||
else
|
||||
"don't know what \($name) is "
|
||||
end
|
||||
| println
|
||||
)
|
||||
end
|
||||
( _help("fq"; $args[0] | _name)
|
||||
| println
|
||||
)
|
||||
else
|
||||
_eval_error("compile"; "help must be last in pipeline. ex: help(length) or ... | help")
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
include "ansi";
|
||||
|
||||
# is here to be defined as early as possible to allow debugging
|
||||
# TODO: move some _* to builtin.jq etc?
|
||||
|
||||
@ -136,3 +138,86 @@ def _is_context_canceled_error: . == "context canceled";
|
||||
|
||||
def _error_str($contexts): (["error"] + $contexts + [.]) | join(": ");
|
||||
def _error_str: _error_str([]);
|
||||
|
||||
# TODO: escape for safe key names
|
||||
# path ["a", 1, "b"] -> "a[1].b"
|
||||
def _path_to_expr($opts):
|
||||
( if length == 0 or (.[0] | type) != "string" then
|
||||
[""] + .
|
||||
end
|
||||
| map(
|
||||
if type == "number" then
|
||||
( ("[" | _ansi_if($opts; "array"))
|
||||
, _ansi_if($opts; "number")
|
||||
, ("]" | _ansi_if($opts; "array"))
|
||||
) else
|
||||
( "."
|
||||
, # empty (special case for leading index or empty path) or key
|
||||
if . == "" or _is_ident then _ansi_if($opts; "objectkey")
|
||||
else
|
||||
"\"\(_escape_ident)\"" | _ansi_if($opts; "string")
|
||||
end
|
||||
)
|
||||
end
|
||||
)
|
||||
| join("")
|
||||
);
|
||||
def _path_to_expr: _path_to_expr(null);
|
||||
|
||||
# TODO: don't use eval? should support '.a.b[1]."c.c"' and escapes?
|
||||
def _expr_to_path:
|
||||
( if type != "string" then error("require string argument") end
|
||||
| _eval("null | path(\(.))")
|
||||
);
|
||||
|
||||
# helper to build path query/generate functions for tree structures with
|
||||
# non-unique children, ex: mp4_path
|
||||
def _tree_path(children; name; $v):
|
||||
def _lookup:
|
||||
# add implicit zeros to get first value
|
||||
# ["a", "b", 1] => ["a", 0, "b", 1]
|
||||
def _normalize_path:
|
||||
( . as $np
|
||||
| if $np | last | type == "string" then $np+[0] end
|
||||
# state is [path acc, possible pending zero index]
|
||||
| ( reduce .[] as $np ([[], []];
|
||||
if $np | type == "string" then
|
||||
[(.[0]+.[1]+[$np]), [0]]
|
||||
else
|
||||
[.[0]+[$np], []]
|
||||
end
|
||||
))
|
||||
)[0];
|
||||
( . as $c
|
||||
| $v
|
||||
| _expr_to_path
|
||||
| _normalize_path
|
||||
| reduce .[] as $n ($c;
|
||||
if $n | type == "string" then
|
||||
children | map(select(name == $n))
|
||||
else
|
||||
.[$n]
|
||||
end
|
||||
)
|
||||
);
|
||||
def _path:
|
||||
[ . as $r
|
||||
| $v._path as $p
|
||||
| foreach range(($p | length)/2) as $i (null; null;
|
||||
( ($r | getpath($p[0:($i+1)*2]) | name) as $name
|
||||
| [($r | getpath($p[0:($i+1)*2-1]))[] | name][0:$p[($i*2)+1]+1] as $before
|
||||
| [ $name
|
||||
, ($before | map(select(. == $name)) | length)-1
|
||||
]
|
||||
)
|
||||
)
|
||||
| [ ".", .[0],
|
||||
(.[1] | if . == 0 then empty else "[", ., "]" end)
|
||||
]
|
||||
]
|
||||
| flatten
|
||||
| join("");
|
||||
if $v | type == "string" then _lookup
|
||||
else _path
|
||||
end;
|
||||
|
||||
|
||||
@ -14,6 +14,8 @@ import (
|
||||
"io/ioutil"
|
||||
"math/big"
|
||||
"path"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@ -47,7 +49,9 @@ import (
|
||||
//go:embed query.jq
|
||||
//go:embed repl.jq
|
||||
//go:embed help.jq
|
||||
//go:embed formats.jq
|
||||
//go:embed format_decode.jq
|
||||
//go:embed format_func.jq
|
||||
//go:embed format_include.jq
|
||||
var builtinFS embed.FS
|
||||
|
||||
var initSource = `include "@builtin/interp";`
|
||||
@ -828,7 +832,7 @@ func (i *Interp) Eval(ctx context.Context, c interface{}, expr string, opts Eval
|
||||
}
|
||||
|
||||
// not identity body means it returns something, threat as dynamic include
|
||||
if q.Term.Type != gojq.TermTypeIdentity {
|
||||
if q.Term == nil || q.Term.Type != gojq.TermTypeIdentity {
|
||||
gc, err := gojq.Compile(q, funcCompilerOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -857,7 +861,7 @@ func (i *Interp) Eval(ctx context.Context, c interface{}, expr string, opts Eval
|
||||
p := queryErrorPosition(s, err)
|
||||
return nil, compileError{
|
||||
err: err,
|
||||
what: "parse",
|
||||
what: "dynamic include parse",
|
||||
filename: filenamePart,
|
||||
pos: p,
|
||||
}
|
||||
@ -1130,3 +1134,79 @@ func (i *Interp) NewColorJSON(opts Options) (*colorjson.Encoder, error) {
|
||||
},
|
||||
), nil
|
||||
}
|
||||
|
||||
var camelToSnakeRe = regexp.MustCompile(`[[:lower:]][[:upper:]]`)
|
||||
|
||||
// "AaaBbb" -> "aaa_bbb"
|
||||
func camelToSnake(s string) string {
|
||||
return strings.ToLower(camelToSnakeRe.ReplaceAllStringFunc(s, func(s string) string {
|
||||
return s[0:1] + "_" + s[1:2]
|
||||
}))
|
||||
}
|
||||
func mapToStruct(m map[string]interface{}, v interface{}) error {
|
||||
ms, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
|
||||
MatchName: func(mapKey, fieldName string) bool {
|
||||
return camelToSnake(fieldName) == mapKey
|
||||
},
|
||||
DecodeHook: func(
|
||||
f reflect.Type,
|
||||
t reflect.Type,
|
||||
data interface{}) (interface{}, error) {
|
||||
|
||||
if t.Kind() == reflect.Slice && t.Elem().Kind() == reflect.Uint8 {
|
||||
switch d := data.(type) {
|
||||
case string:
|
||||
return []byte(d), nil
|
||||
}
|
||||
} else {
|
||||
switch d := data.(type) {
|
||||
case *big.Int:
|
||||
return d.Uint64(), nil
|
||||
}
|
||||
}
|
||||
|
||||
return data, nil
|
||||
},
|
||||
Result: v,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := ms.Decode(m); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func camelCaseMap(m map[string]interface{}) map[string]interface{} {
|
||||
nm := map[string]interface{}{}
|
||||
for k, v := range m {
|
||||
sk := camelToSnake(k)
|
||||
if vm, ok := v.(map[string]interface{}); ok {
|
||||
v = camelCaseMap(vm)
|
||||
} else {
|
||||
// TODO: error
|
||||
v, _ = gojqextra.ToGoJQValue(v)
|
||||
}
|
||||
nm[sk] = v
|
||||
}
|
||||
|
||||
return nm
|
||||
}
|
||||
|
||||
func structToMap(v interface{}) (map[string]interface{}, error) {
|
||||
m := map[string]interface{}{}
|
||||
ms, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
|
||||
Result: &m,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ms.Decode(v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return camelCaseMap(m), nil
|
||||
}
|
||||
|
||||
@ -2,15 +2,16 @@ include "internal";
|
||||
include "options";
|
||||
include "binary";
|
||||
include "decode";
|
||||
include "funcs";
|
||||
include "format_decode";
|
||||
include "format_include";
|
||||
include "format_func";
|
||||
include "grep";
|
||||
include "args";
|
||||
include "eval";
|
||||
include "query";
|
||||
include "repl";
|
||||
include "help";
|
||||
# generate torepr, format decode helpers and include format specific functions
|
||||
include "formats";
|
||||
include "funcs";
|
||||
# optional user init
|
||||
include "@config/init?";
|
||||
|
||||
@ -184,7 +185,7 @@ def _main:
|
||||
| _options_stack([
|
||||
( ( _opt_build_default_fixed
|
||||
+ $parsed_args
|
||||
+ ($parsed_args.option | _opt_cli_arg_tooptions)
|
||||
+ ($parsed_args.option | if . then _opt_cli_arg_to_options end)
|
||||
)
|
||||
| . + _opt_eval($rest)
|
||||
)
|
||||
|
||||
@ -68,8 +68,62 @@ def _opt_build_default_fixed:
|
||||
}
|
||||
);
|
||||
|
||||
def _opt_options:
|
||||
{
|
||||
addrbase: "number",
|
||||
arg: "array_string_pair",
|
||||
argjson: "array_string_pair",
|
||||
array_truncate: "number",
|
||||
bits_format: "string",
|
||||
byte_colors: "csv_ranges_array",
|
||||
color: "boolean",
|
||||
colors: "csv_kv_obj",
|
||||
compact: "boolean",
|
||||
completion_timeout: "number",
|
||||
decode_file: "array_string_pair",
|
||||
decode_format: "string",
|
||||
decode_progress: "boolean",
|
||||
depth: "number",
|
||||
display_bytes: "number",
|
||||
expr: "string",
|
||||
expr_eval_path: "string",
|
||||
expr_file: "string",
|
||||
filenames: "array_string",
|
||||
force: "boolean",
|
||||
include_path: "string",
|
||||
join_string: "string",
|
||||
line_bytes: "number",
|
||||
null_input: "boolean",
|
||||
raw_file: "array_string_pair",
|
||||
raw_output: "boolean",
|
||||
raw_string: "boolean",
|
||||
repl: "boolean",
|
||||
sizebase: "number",
|
||||
show_formats: "boolean",
|
||||
show_help: "boolean",
|
||||
slurp: "boolean",
|
||||
string_input: "boolean",
|
||||
unicode: "boolean",
|
||||
verbose: "boolean",
|
||||
width: "number",
|
||||
};
|
||||
|
||||
def _opt_eval($rest):
|
||||
( { argjson: (
|
||||
( with_entries(
|
||||
( select(.value | type == "string" and startswith("@"))
|
||||
| .value |=
|
||||
( . as $v
|
||||
| try
|
||||
( .[1:]
|
||||
| open
|
||||
| tobytes
|
||||
| tostring
|
||||
)
|
||||
catch $v
|
||||
)
|
||||
)
|
||||
)
|
||||
+ { argjson: (
|
||||
( .argjson
|
||||
| if . then
|
||||
map(
|
||||
@ -155,7 +209,7 @@ def _opt_eval($rest):
|
||||
);
|
||||
|
||||
# these _to* function do a bit for fuzzy string to type conversions
|
||||
def _opt_toboolean:
|
||||
def _opt_to_boolean:
|
||||
try
|
||||
if . == "true" then true
|
||||
elif . == "false" then false
|
||||
@ -164,14 +218,14 @@ def _opt_toboolean:
|
||||
catch
|
||||
null;
|
||||
|
||||
def _opt_fromboolean: tostring;
|
||||
def _opt_from_boolean: tostring;
|
||||
|
||||
def _opt_tonumber:
|
||||
def _opt_to_number:
|
||||
try tonumber catch null;
|
||||
|
||||
def _opt_fromnumber: tostring;
|
||||
def _opt_from_number: tostring;
|
||||
|
||||
def _opt_tostring:
|
||||
def _opt_to_string:
|
||||
if . != null then
|
||||
( "\"\(.)\""
|
||||
| try
|
||||
@ -182,19 +236,22 @@ def _opt_tostring:
|
||||
)
|
||||
end;
|
||||
|
||||
def _opt_fromstring: if . then tojson[1:-1] else "" end;
|
||||
def _opt_from_string: if . then tojson[1:-1] else "" end;
|
||||
|
||||
def _opt_toarray(f):
|
||||
def _opt_is_string_pair:
|
||||
type == "array" and length == 2 and all(type == "string");
|
||||
|
||||
def _opt_to_array(f):
|
||||
try
|
||||
( fromjson
|
||||
| if type == "array" and (all(f) | not) then null end
|
||||
)
|
||||
catch null;
|
||||
|
||||
def _opt_fromarray: tojson;
|
||||
def _opt_to_array_string_pair: _opt_to_array(_opt_is_string_pair);
|
||||
def _opt_to_array_string: _opt_to_array(type == "string");
|
||||
|
||||
def _opt_is_string_pair:
|
||||
type == "array" and length == 2 and all(type == "string");
|
||||
def _opt_from_array: tojson;
|
||||
|
||||
# TODO: cleanup
|
||||
def _trim: capture("^\\s*(?<str>.*?)\\s*$"; "").str;
|
||||
@ -246,86 +303,53 @@ def _opt_from_csv_kv_obj:
|
||||
| join(",")
|
||||
);
|
||||
|
||||
def _opt_cli_arg_tooptions:
|
||||
( {
|
||||
addrbase: (.addrbase | _opt_tonumber),
|
||||
arg: (.arg | _opt_toarray(_opt_is_string_pair)),
|
||||
argjson: (.argjson | _opt_toarray(_opt_is_string_pair)),
|
||||
array_truncate: (.array_truncate | _opt_tonumber),
|
||||
bits_format: (.bits_format | _opt_tostring),
|
||||
byte_colors: (.byte_colors | _opt_to_csv_ranges_array),
|
||||
color: (.color | _opt_toboolean),
|
||||
colors: (.colors | _opt_to_csv_kv_obj),
|
||||
compact: (.compact | _opt_toboolean),
|
||||
completion_timeout: (.array_truncate | _opt_tonumber),
|
||||
decode_file: (.decode_file | _opt_toarray(_opt_is_string_pair)),
|
||||
decode_format: (.decode_format | _opt_tostring),
|
||||
decode_progress: (.decode_progress | _opt_toboolean),
|
||||
depth: (.depth | _opt_tonumber),
|
||||
display_bytes: (.display_bytes | _opt_tonumber),
|
||||
expr: (.expr | _opt_tostring),
|
||||
expr_file: (.expr_file | _opt_tostring),
|
||||
filenames: (.filenames | _opt_toarray(type == "string")),
|
||||
force: (.force | _opt_toboolean),
|
||||
include_path: (.include_path | _opt_tostring),
|
||||
join_string: (.join_string | _opt_tostring),
|
||||
line_bytes: (.line_bytes | _opt_tonumber),
|
||||
null_input: (.null_input | _opt_toboolean),
|
||||
raw_file: (.raw_file| _opt_toarray(_opt_is_string_pair)),
|
||||
raw_output: (.raw_output | _opt_toboolean),
|
||||
raw_string: (.raw_string | _opt_toboolean),
|
||||
repl: (.repl | _opt_toboolean),
|
||||
sizebase: (.sizebase | _opt_tonumber),
|
||||
show_formats: (.show_formats | _opt_toboolean),
|
||||
show_help: (.show_help | _opt_toboolean),
|
||||
slurp: (.slurp | _opt_toboolean),
|
||||
string_input: (.string_input | _opt_toboolean),
|
||||
unicode: (.unicode | _opt_toboolean),
|
||||
verbose: (.verbose | _opt_toboolean),
|
||||
width: (.width | _opt_tonumber),
|
||||
}
|
||||
| with_entries(select(.value != null))
|
||||
def _opt_to_fuzzy:
|
||||
( . as $s
|
||||
| try fromjson
|
||||
catch $s
|
||||
);
|
||||
|
||||
def _opt_cli_arg_fromoptions:
|
||||
( {
|
||||
addrbase: (.addrbase | _opt_fromnumber),
|
||||
arg: (.arg | _opt_fromarray),
|
||||
argjson: (.argjson | _opt_fromarray),
|
||||
array_truncate: (.array_truncate | _opt_fromnumber),
|
||||
bits_format: (.bits_format | _opt_fromstring),
|
||||
byte_colors: (.byte_colors | _opt_from_csv_ranges_array),
|
||||
color: (.color | _opt_fromboolean),
|
||||
colors: (.colors | _opt_from_csv_kv_obj),
|
||||
compact: (.compact | _opt_fromboolean),
|
||||
completion_timeout: (.array_truncate | _opt_fromnumber),
|
||||
decode_file: (.decode_file | _opt_fromarray),
|
||||
decode_format: (.decode_format | _opt_fromstring),
|
||||
decode_progress: (.decode_progress | _opt_fromboolean),
|
||||
depth: (.depth | _opt_fromnumber),
|
||||
display_bytes: (.display_bytes | _opt_fromnumber),
|
||||
expr: (.expr | _opt_fromstring),
|
||||
expr_file: (.expr_file | _opt_fromstring),
|
||||
filenames: (.filenames | _opt_fromarray),
|
||||
force: (.force | _opt_fromboolean),
|
||||
include_path: (.include_path | _opt_fromstring),
|
||||
join_string: (.join_string | _opt_fromstring),
|
||||
line_bytes: (.line_bytes | _opt_fromnumber),
|
||||
null_input: (.null_input | _opt_fromboolean),
|
||||
raw_file: (.raw_file| _opt_fromarray),
|
||||
raw_output: (.raw_output | _opt_fromboolean),
|
||||
raw_string: (.raw_string | _opt_fromboolean),
|
||||
repl: (.repl | _opt_fromboolean),
|
||||
sizebase: (.sizebase | _opt_fromnumber),
|
||||
show_formats: (.show_formats | _opt_fromboolean),
|
||||
show_help: (.show_help | _opt_fromboolean),
|
||||
slurp: (.slurp | _opt_fromboolean),
|
||||
string_input: (.string_input | _opt_fromboolean),
|
||||
unicode: (.unicode | _opt_fromboolean),
|
||||
verbose: (.verbose | _opt_fromboolean),
|
||||
width: (.width | _opt_tonumber),
|
||||
}
|
||||
| with_entries(select(.value != null))
|
||||
def _opt_to($type):
|
||||
if $type == "array_string" then _opt_to_array_string
|
||||
elif $type == "array_string_pair" then _opt_to_array_string_pair
|
||||
elif $type == "boolean" then _opt_to_boolean
|
||||
elif $type == "csv_kv_obj" then _opt_to_csv_kv_obj
|
||||
elif $type == "csv_ranges_array" then _opt_to_csv_ranges_array
|
||||
elif $type == "number" then _opt_to_number
|
||||
elif $type == "string" then _opt_to_string
|
||||
elif $type == "fuzzy" then _opt_to_fuzzy
|
||||
else error("unknown type \($type)")
|
||||
end;
|
||||
|
||||
def _opt_from($type):
|
||||
if $type == "array_string" then _opt_from_array
|
||||
elif $type == "array_string_pair" then _opt_from_array
|
||||
elif $type == "boolean" then _opt_from_boolean
|
||||
elif $type == "csv_kv_obj" then _opt_from_csv_kv_obj
|
||||
elif $type == "csv_ranges_array" then _opt_from_csv_ranges_array
|
||||
elif $type == "number" then _opt_from_number
|
||||
elif $type == "string" then _opt_from_string
|
||||
else error("unknown type \($type)")
|
||||
end;
|
||||
|
||||
def _opt_cli_arg_to_options:
|
||||
( _opt_options as $opts
|
||||
| with_entries(
|
||||
( .key as $k
|
||||
| .value |= _opt_to($opts[$k] // "fuzzy")
|
||||
| select(.value != null)
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
def _opt_cli_arg_from_options:
|
||||
( _opt_options as $opts
|
||||
| with_entries(
|
||||
( .key as $k
|
||||
| .value |= _opt_from($opts[$k] // "string")
|
||||
| select(.value != null)
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
def _opt_cli_opts:
|
||||
|
||||
@ -51,6 +51,11 @@ def _query_is_func:
|
||||
def _query_is_func($name):
|
||||
_query_is_func and _query_func_name == $name;
|
||||
|
||||
def _query_is_string:
|
||||
.term.type == "TermTypeString";
|
||||
def _query_string_str:
|
||||
.term.str.str;
|
||||
|
||||
def _query_empty:
|
||||
_query_func("empty");
|
||||
|
||||
|
||||
@ -119,7 +119,7 @@ def _prompt($opts):
|
||||
def _repl_level:
|
||||
(_options_stack | length | if . > 2 then ((.-2) * ">") else empty end);
|
||||
def _value_path:
|
||||
(._path? // []) | if . == [] then empty else path_to_expr($opts) end;
|
||||
(._path? // []) | if . == [] then empty else _path_to_expr($opts) end;
|
||||
def _value_preview($depth):
|
||||
if $depth == 0 and format == null and type == "array" then
|
||||
[ "["
|
||||
|
||||
5
pkg/interp/testdata/args.fqtest
vendored
5
pkg/interp/testdata/args.fqtest
vendored
@ -67,13 +67,14 @@ byte_colors 0-255=brightwhite,0=brightblack,32-126:9-13=white
|
||||
color false
|
||||
colors array=white,dumpaddr=yellow,dumpheader=yellow+underline,error=brightred,false=yellow,index=white,null=brightblack,number=cyan,object=white,objectkey=brightblue,prompt_repl_level=brightblack,prompt_value=white,string=green,true=yellow,value=white
|
||||
compact false
|
||||
completion_timeout 50
|
||||
completion_timeout 10
|
||||
decode_file []
|
||||
decode_format probe
|
||||
decode_progress false
|
||||
depth 0
|
||||
display_bytes 16
|
||||
expr .
|
||||
expr_eval_path arg
|
||||
expr_file
|
||||
filenames [null]
|
||||
force false
|
||||
@ -100,7 +101,7 @@ adts_frame Audio Data Transport Stream frame
|
||||
amf0 Action Message Format 0
|
||||
apev2 APEv2 metadata tag
|
||||
ar Unix archive
|
||||
asn1_ber ASN1 Basic Encoding Rules (also CER and DER)
|
||||
asn1_ber ASN1 BER (basic encoding rules, also CER and DER)
|
||||
av1_ccr AV1 Codec Configuration Record
|
||||
av1_frame AV1 frame
|
||||
av1_obu AV1 Open Bitstream Unit
|
||||
|
||||
1
pkg/interp/testdata/completion.fqtest
vendored
1
pkg/interp/testdata/completion.fqtest
vendored
@ -48,6 +48,7 @@ _format_root
|
||||
_index
|
||||
_len
|
||||
_name
|
||||
_out
|
||||
_parent
|
||||
_path
|
||||
_root
|
||||
|
||||
25
pkg/interp/testdata/help.fqtest
vendored
25
pkg/interp/testdata/help.fqtest
vendored
@ -1,10 +1,11 @@
|
||||
$ fq -ni
|
||||
null> help
|
||||
Type expression to evaluate
|
||||
help(...) Help for topic. Ex: help(mp4), help("mp4")
|
||||
\t Completion
|
||||
Up/Down History
|
||||
^C Interrupt execution
|
||||
... | repl Start a new REPL
|
||||
^C Interrupt execution
|
||||
^D Exit REPL
|
||||
null> help | abc
|
||||
error: expr: function not defined: abc/0
|
||||
@ -25,6 +26,28 @@ length: Length of string, array, object, etc
|
||||
- For number the number itself
|
||||
- For boolean is an error
|
||||
|
||||
Examples:
|
||||
> [1,2,3] | length
|
||||
3
|
||||
> "abc" | length
|
||||
3
|
||||
> {"a":1,"b":2} | length
|
||||
2
|
||||
> null | length
|
||||
0
|
||||
> 123 | length
|
||||
123
|
||||
> true | length
|
||||
error: length cannot be applied to: boolean (true)
|
||||
null> help("length")
|
||||
length: Length of string, array, object, etc
|
||||
- For string number of unicode codepoints
|
||||
- For array number of elements in array
|
||||
- For object number of key-value pairs
|
||||
- For null zero
|
||||
- For number the number itself
|
||||
- For boolean is an error
|
||||
|
||||
Examples:
|
||||
> [1,2,3] | length
|
||||
3
|
||||
|
||||
2
pkg/interp/testdata/torepr.fqtest
vendored
2
pkg/interp/testdata/torepr.fqtest
vendored
@ -2,5 +2,5 @@ $ fq -i
|
||||
null> torepr
|
||||
error: value is not a format root
|
||||
null> "{}" | json | torepr
|
||||
error: format has no torepr
|
||||
error: json has no torepr
|
||||
null> ^D
|
||||
|
||||
Loading…
Reference in New Issue
Block a user