From d6d3265b3473adcfc730893d8dcab0d4444a2414 Mon Sep 17 00:00:00 2001 From: Mattias Wadman Date: Mon, 22 Nov 2021 14:16:59 +0100 Subject: [PATCH] doc: Update README a bit --- README.md | 8 +- doc/TODO.md | 1 + doc/demo.sh | 8 +- doc/demo.svg | 70 +-- doc/formats.svg | 1130 +++++++++++++++++++++++------------------------ 5 files changed, 617 insertions(+), 600 deletions(-) diff --git a/README.md b/README.md index b48165d2..31dd429d 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,12 @@ Tool, language and decoders for exploring binary data. ![fq demo](doc/demo.svg) +In most cases it works the same way of jq but instead of reading JSON it reads binary formats into +a JSON compatbile structure. Also each value knows from which bit ranges it comes from, can have +symbolic interprestations and knows how to be presented in a useful way. + +**NOTE:** fq is early in development and many things are missing, broken or do not make sense. That also means there is a great opportunity to help out. + ## Goals - Make binary formats accessible and queryable. @@ -12,8 +18,6 @@ Tool, language and decoders for exploring binary data. - Bit and byte transformations and conversions. - Programmer's calculator. -**NOTE:** fq is early in development and many things are missing, broken or do not make sense. That also means there is a great opportunity to help out. - ## Usage Basic usage is: `fq . file`. diff --git a/doc/TODO.md b/doc/TODO.md index 5afec5d3..e0d95f0d 100644 --- a/doc/TODO.md +++ b/doc/TODO.md @@ -99,6 +99,7 @@ #### Formats +- Network protocols, pcap? - Pass argument to format - Value decoder in jq `u(32)`, `u32`? - Warnings and errors diff --git a/doc/demo.sh b/doc/demo.sh index 8e02a3d9..2cc9575f 100755 --- a/doc/demo.sh +++ b/doc/demo.sh @@ -11,16 +11,16 @@ s() { sh -c "${1/fq/$FQ -o unicode=true -C}" } -c "Overview of mp3" +c "Overview of mp3 file" s "fq . file.mp3" echo -c "Show ID3v2 tag in mp3 file" +c "Show ID3v2 tag inside mp3 file" s "fq '.headers[0]' file.mp3" echo -c "Resolution of ID3v2 cover art" +c "Resolution of embedded PNG cover art" s "fq '.headers[0].frames[] | select(.id == \"APIC\").picture.chunks[] | select(.type == \"IHDR\") | {width, height}' file.mp3" echo -c "Extract image file" +c "Extract PNG file" s "fq '.headers[].frames[] | select(.id == \"APIC\")?.picture | tobits' file.mp3 >file.png" s "file file.png" rm -f file.png diff --git a/doc/demo.svg b/doc/demo.svg index 274ecff1..93aad84e 100644 --- a/doc/demo.svg +++ b/doc/demo.svg @@ -24,6 +24,10 @@ m p 3 + f + i + l + e $ f q @@ -376,13 +380,17 @@ g i n - m - p - 3 - f - i - l - e + s + i + d + e + m + p + 3 + f + i + l + e $ f q @@ -815,19 +823,25 @@ n o f - I - D - 3 - v - 2 - c - o - v - e - r - a - r - t + e + m + b + e + d + d + e + d + P + N + G + c + o + v + e + r + a + r + t $ f q @@ -969,15 +983,13 @@ a c t - i - m - a - g - e - f - i - l - e + P + N + G + f + i + l + e $ f q diff --git a/doc/formats.svg b/doc/formats.svg index 09df7a54..86e067a1 100644 --- a/doc/formats.svg +++ b/doc/formats.svg @@ -4,1316 +4,1316 @@ - + formats - + adts - -adts - -adts_frame + +adts + +adts_frame adts_frame - -adts_frame - -aac_frame + +adts_frame + +aac_frame adts:adts_frame->adts_frame - - + + aac_frame - -aac_frame + +aac_frame adts_frame:aac_frame->aac_frame - - + + apev2 - -apev2 - -image + +apev2 + +image image - -image + +image apev2:image->image - - + + jpeg - -jpeg - -exif - -icc_profile + +jpeg + +exif + +icc_profile image->jpeg:jpeg - - + + mp4 - -mp4 - -aac_frame - -av1_ccr - -av1_frame - -flac_frame - -flac_metadatablocks - -id3v2 - -image - -jpeg - -mp3_frame - -avc_au - -avc_dcr - -mpeg_es - -hevc_au - -hevc_dcr - -mpeg_pes_packet - -opus_packet - -protobuf_widevine - -pssh_playready - -vorbis_packet - -vp9_frame - -vpx_ccr + +mp4 + +aac_frame + +av1_ccr + +av1_frame + +flac_frame + +flac_metadatablocks + +id3v2 + +image + +jpeg + +mp3_frame + +avc_au + +avc_dcr + +mpeg_es + +hevc_au + +hevc_dcr + +mpeg_pes_packet + +opus_packet + +protobuf_widevine + +pssh_playready + +vorbis_packet + +vp9_frame + +vpx_ccr image->mp4:mp4 - - + + png - -png - -icc_profile - -exif + +png + +icc_profile + +exif image->png:png - - + + tiff - -tiff - -icc_profile + +tiff + +icc_profile image->tiff:tiff - - + + webp - -webp - -vp8_frame + +webp + +vp8_frame image->webp:webp - - + + gif - -gif + +gif image->gif:gif - - + + av1_frame - -av1_frame - -av1_obu + +av1_frame + +av1_obu av1_obu - -av1_obu + +av1_obu av1_frame:av1_obu->av1_obu - - + + avc_annexb - -avc_annexb - -avc_nalu + +avc_annexb + +avc_nalu avc_nalu - -avc_nalu - -avc_sps - -avc_pps - -avc_sei + +avc_nalu + +avc_sps + +avc_pps + +avc_sei avc_annexb:avc_nalu->avc_nalu - - + + avc_sps - -avc_sps + +avc_sps avc_nalu:avc_sps->avc_sps - - + + avc_pps - -avc_pps + +avc_pps avc_nalu:avc_pps->avc_pps - - + + avc_sei - -avc_sei + +avc_sei avc_nalu:avc_sei->avc_sei - - + + avc_au - -avc_au - -avc_nalu + +avc_au + +avc_nalu avc_au:avc_nalu->avc_nalu - - + + avc_dcr - -avc_dcr - -avc_nalu + +avc_dcr + +avc_nalu avc_dcr:avc_nalu->avc_nalu - - + + bzip2 - -bzip2 - -probe + +bzip2 + +probe probe - -probe + +probe bzip2:probe->probe - - + + probe->adts:adts - - + + probe->bzip2:bzip2 - - + + flac - -flac - -flac_metadatablocks - -flac_frame + +flac + +flac_metadatablocks + +flac_frame probe->flac:flac - - + + gzip - -gzip - -probe + +gzip + +probe probe->gzip:gzip - - + + probe->jpeg:jpeg - - + + matroska - -matroska - -aac_frame - -av1_ccr - -av1_frame - -avc_au - -avc_dcr - -flac_frame - -flac_metadatablocks - -hevc_au - -hevc_dcr - -image - -mp3_frame - -mpeg_asc - -mpeg_pes_packet - -mpeg_spu - -opus_packet - -vorbis_packet - -vp8_frame - -vp9_cfm - -vp9_frame + +matroska + +aac_frame + +av1_ccr + +av1_frame + +avc_au + +avc_dcr + +flac_frame + +flac_metadatablocks + +hevc_au + +hevc_dcr + +image + +mp3_frame + +mpeg_asc + +mpeg_pes_packet + +mpeg_spu + +opus_packet + +vorbis_packet + +vp8_frame + +vp9_cfm + +vp9_frame probe->matroska:matroska - - - + + mp3 - -mp3 - -id3v2 - -id3v1 - -id3v11 - -apev2 - -mp3_frame + +mp3 + +id3v2 + +id3v1 + +id3v11 + +apev2 + +mp3_frame probe->mp3:mp3 - - + + probe->mp4:mp4 - - - + + ogg - -ogg - -ogg_page - -vorbis_packet - -opus_packet - -flac_metadatablock - -flac_frame + +ogg + +ogg_page + +vorbis_packet + +opus_packet + +flac_metadatablock + +flac_frame probe->ogg:ogg - - + + + probe->png:png - - + + + tar - -tar - -probe + +tar + +probe probe->tar:tar - - + + probe->tiff:tiff - - + + wav - -wav - -id3v2 - -id3v1 - -id3v11 + +wav + +id3v2 + +id3v1 + +id3v11 probe->wav:wav - - + + probe->webp:webp - - + + + zip - -zip - -probe + +zip + +probe probe->zip:zip - - + + elf - -elf + +elf probe->elf:elf - - + + probe->gif:gif - - - + + json - -json + +json probe->json:json - - + + mpeg_ts - -mpeg_ts + +mpeg_ts probe->mpeg_ts:mpeg_ts - - + + flac_metadatablocks - -flac_metadatablocks - -flac_metadatablock + +flac_metadatablocks + +flac_metadatablock flac:flac_metadatablocks->flac_metadatablocks - - + + flac_frame - -flac_frame + +flac_frame flac:flac_frame->flac_frame - - + + flac_metadatablock - -flac_metadatablock - -flac_streaminfo - -flac_picture - -vorbis_comment + +flac_metadatablock + +flac_streaminfo + +flac_picture + +vorbis_comment flac_metadatablocks:flac_metadatablock->flac_metadatablock - - + + flac_streaminfo - -flac_streaminfo + +flac_streaminfo flac_metadatablock:flac_streaminfo->flac_streaminfo - - + + flac_picture - -flac_picture - -image + +flac_picture + +image flac_metadatablock:flac_picture->flac_picture - - + + vorbis_comment - -vorbis_comment - -flac_picture + +vorbis_comment + +flac_picture flac_metadatablock:vorbis_comment->vorbis_comment - - + + flac_picture:image->image - - + + vorbis_comment:flac_picture->flac_picture - - + + gzip:probe->probe - - + + hevc_annexb - -hevc_annexb - -hevc_nalu + +hevc_annexb + +hevc_nalu hevc_nalu - -hevc_nalu + +hevc_nalu hevc_annexb:hevc_nalu->hevc_nalu - - + + hevc_au - -hevc_au - -hevc_nalu + +hevc_au + +hevc_nalu hevc_au:hevc_nalu->hevc_nalu - - + + hevc_dcr - -hevc_dcr - -hevc_nalu + +hevc_dcr + +hevc_nalu hevc_dcr:hevc_nalu->hevc_nalu - - + + id3v2 - -id3v2 - -image + +id3v2 + +image id3v2:image->image - - - + + + exif - -exif + +exif jpeg:exif->exif - - + + icc_profile - -icc_profile + +icc_profile jpeg:icc_profile->icc_profile - - + + matroska:aac_frame->aac_frame - - + + matroska:image->image - - + + matroska:av1_frame->av1_frame - - + + matroska:avc_au->avc_au - - + + matroska:avc_dcr->avc_dcr - - + + matroska:flac_metadatablocks->flac_metadatablocks - - + + matroska:flac_frame->flac_frame - - + + matroska:hevc_au->hevc_au - - + + matroska:hevc_dcr->hevc_dcr - - + + av1_ccr - -av1_ccr + +av1_ccr matroska:av1_ccr->av1_ccr - - + + mp3_frame - -mp3_frame - -xing + +mp3_frame + +xing matroska:mp3_frame->mp3_frame - - - + + mpeg_asc - -mpeg_asc + +mpeg_asc matroska:mpeg_asc->mpeg_asc - - + + mpeg_pes_packet - -mpeg_pes_packet + +mpeg_pes_packet matroska:mpeg_pes_packet->mpeg_pes_packet - - - + + + mpeg_spu - -mpeg_spu + +mpeg_spu matroska:mpeg_spu->mpeg_spu - - + + opus_packet - -opus_packet - -vorbis_comment + +opus_packet + +vorbis_comment matroska:opus_packet->opus_packet - - + + vorbis_packet - -vorbis_packet - -vorbis_comment + +vorbis_packet + +vorbis_comment matroska:vorbis_packet->vorbis_packet - - + + vp8_frame - -vp8_frame + +vp8_frame matroska:vp8_frame->vp8_frame - - + + vp9_cfm - -vp9_cfm + +vp9_cfm matroska:vp9_cfm->vp9_cfm - - + + vp9_frame - -vp9_frame + +vp9_frame matroska:vp9_frame->vp9_frame - - + + xing - -xing + +xing mp3_frame:xing->xing - - + + opus_packet:vorbis_comment->vorbis_comment - - + + vorbis_packet:vorbis_comment->vorbis_comment - - + + mp3:apev2->apev2 - - + + mp3:id3v2->id3v2 - + + mp3:mp3_frame->mp3_frame - + + id3v1 - -id3v1 + +id3v1 mp3:id3v1->id3v1 - - + + id3v11 - -id3v11 + +id3v11 mp3:id3v11->id3v11 - - + + mp4:aac_frame->aac_frame - - + + mp4:image->image - - + + mp4:av1_frame->av1_frame - - + + mp4:avc_au->avc_au - - + + mp4:avc_dcr->avc_dcr - - + + mp4:flac_metadatablocks->flac_metadatablocks - - + + mp4:flac_frame->flac_frame - - + + mp4:hevc_au->hevc_au - - + + mp4:hevc_dcr->hevc_dcr - - + + mp4:id3v2->id3v2 - - + + mp4:jpeg->jpeg - - + + mp4:av1_ccr->av1_ccr - - + + mp4:mp3_frame->mp3_frame - - + + mp4:mpeg_pes_packet->mpeg_pes_packet - - + + mp4:opus_packet->opus_packet - - + + mp4:vorbis_packet->vorbis_packet - - + + mp4:vp9_frame->vp9_frame - - + + mpeg_es - -mpeg_es - -mpeg_asc - -vorbis_packet + +mpeg_es + +mpeg_asc + +vorbis_packet mp4:mpeg_es->mpeg_es - - + + protobuf_widevine - -protobuf_widevine - -protobuf + +protobuf_widevine + +protobuf mp4:protobuf_widevine->protobuf_widevine - - + + pssh_playready - -pssh_playready + +pssh_playready mp4:pssh_playready->pssh_playready - - + + vpx_ccr - -vpx_ccr + +vpx_ccr mp4:vpx_ccr->vpx_ccr - - + + mpeg_es:mpeg_asc->mpeg_asc - - + + mpeg_es:vorbis_packet->vorbis_packet - - + + protobuf - -protobuf + +protobuf protobuf_widevine:protobuf->protobuf - - + + mpeg_pes - -mpeg_pes - -mpeg_pes_packet - -mpeg_spu + +mpeg_pes + +mpeg_pes_packet + +mpeg_spu mpeg_pes:mpeg_pes_packet->mpeg_pes_packet - + mpeg_pes:mpeg_spu->mpeg_spu - - + + ogg:flac_frame->flac_frame - - + + ogg:flac_metadatablock->flac_metadatablock - - + + ogg:opus_packet->opus_packet - - + + ogg:vorbis_packet->vorbis_packet - - + + ogg_page - -ogg_page + +ogg_page ogg:ogg_page->ogg_page - - + + png:exif->exif - - + + png:icc_profile->icc_profile - - - + + + tar:probe->probe - - + + tiff:icc_profile->icc_profile - + wav:id3v2->id3v2 - - - + + wav:id3v1->id3v1 - - + + wav:id3v11->id3v11 - - + + webp:vp8_frame->vp8_frame - - + + zip:probe->probe - - + + dns - -dns + +dns raw - -raw + +raw