diff --git a/README.md b/README.md index 42dabf28..0ae6aa8e 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ cp fq /usr/local/bin [./formats_list.jq]: sh-start -aac_frame, adts, adts_frame, apev2, av1_ccr, av1_frame, av1_obu, avc_annexb, avc_au, avc_dcr, avc_nalu, avc_pps, avc_sei, avc_sps, bzip2, dns, dns_tcp, elf, ether8023_frame, exif, flac, flac_frame, flac_metadatablock, flac_metadatablocks, flac_picture, flac_streaminfo, gif, gzip, hevc_annexb, hevc_au, hevc_dcr, hevc_nalu, icc_profile, icmp, id3v1, id3v11, id3v2, ipv4_packet, jpeg, json, matroska, mp3, mp3_frame, mp4, mpeg_asc, mpeg_es, mpeg_pes, mpeg_pes_packet, mpeg_spu, mpeg_ts, ogg, ogg_page, opus_packet, pcap, pcapng, png, protobuf, protobuf_widevine, pssh_playready, raw, sll2_packet, sll_packet, tar, tcp_segment, tiff, udp_datagram, vorbis_comment, vorbis_packet, vp8_frame, vp9_cfm, vp9_frame, vpx_ccr, wav, webp, xing, zip +aac_frame, adts, adts_frame, apev2, av1_ccr, av1_frame, av1_obu, avc_annexb, avc_au, avc_dcr, avc_nalu, avc_pps, avc_sei, avc_sps, bsd_loopback_frame, bzip2, dns, dns_tcp, elf, ether8023_frame, exif, flac, flac_frame, flac_metadatablock, flac_metadatablocks, flac_picture, flac_streaminfo, gif, gzip, hevc_annexb, hevc_au, hevc_dcr, hevc_nalu, icc_profile, icmp, id3v1, id3v11, id3v2, ipv4_packet, jpeg, json, matroska, mp3, mp3_frame, mp4, mpeg_asc, mpeg_es, mpeg_pes, mpeg_pes_packet, mpeg_spu, mpeg_ts, ogg, ogg_page, opus_packet, pcap, pcapng, png, protobuf, protobuf_widevine, pssh_playready, raw, sll2_packet, sll_packet, tar, tcp_segment, tiff, udp_datagram, vorbis_comment, vorbis_packet, vp8_frame, vp9_cfm, vp9_frame, vpx_ccr, wav, webp, xing, zip [#]: sh-end diff --git a/doc/formats.md b/doc/formats.md index 34072aef..3210bfff 100644 --- a/doc/formats.md +++ b/doc/formats.md @@ -18,6 +18,7 @@ |`avc_pps` |H.264/AVC Picture Parameter Set || |`avc_sei` |H.264/AVC Supplemental Enhancement Information || |`avc_sps` |H.264/AVC Sequence Parameter Set || +|`bsd_loopback_frame` |BSD loopback frame |`ipv4_packet`| |`bzip2` |bzip2 compression |`probe`| |`dns` |DNS packet || |`dns_tcp` |DNS packet (TCP) || @@ -57,8 +58,8 @@ |`ogg` |OGG file |`ogg_page` `vorbis_packet` `opus_packet` `flac_metadatablock` `flac_frame`| |`ogg_page` |OGG page || |`opus_packet` |Opus packet |`vorbis_comment`| -|`pcap` |PCAP packet capture |`ether8023_frame` `sll_packet` `sll2_packet` `tcp_stream` `ipv4_packet`| -|`pcapng` |PCAPNG packet capture |`ether8023_frame` `sll_packet` `sll2_packet` `tcp_stream` `ipv4_packet`| +|`pcap` |PCAP packet capture |`link_frame` `tcp_stream` `ipv4_packet`| +|`pcapng` |PCAPNG packet capture |`link_frame` `tcp_stream` `ipv4_packet`| |`png` |Portable Network Graphics file |`icc_profile` `exif`| |`protobuf` |Protobuf || |`protobuf_widevine` |Widevine protobuf |`protobuf`| @@ -81,6 +82,7 @@ |`xing` |Xing header || |`zip` |ZIP archive |`probe`| |`image` |Group |`gif` `jpeg` `mp4` `png` `tiff` `webp`| +|`link_frame` |Group |`bsd_loopback_frame` `ether8023_frame` `sll2_packet` `sll_packet`| |`probe` |Group |`adts` `bzip2` `elf` `flac` `gif` `gzip` `jpeg` `json` `matroska` `mp3` `mp4` `mpeg_ts` `ogg` `pcap` `pcapng` `png` `tar` `tiff` `wav` `webp` `zip`| |`tcp_stream` |Group |`dns`| |`udp_payload` |Group |`dns`| diff --git a/doc/formats.svg b/doc/formats.svg index 683f23ce..cbbed896 100644 --- a/doc/formats.svg +++ b/doc/formats.svg @@ -4,1551 +4,1562 @@ - - + + formats - + adts - -adts - -adts_frame + +adts + +adts_frame adts_frame - -adts_frame - -aac_frame + +adts_frame + +aac_frame adts:adts_frame->adts_frame - - + + aac_frame - -aac_frame + +aac_frame adts_frame:aac_frame->aac_frame - - - + + + apev2 - -apev2 - -image + +apev2 + +image image - -image + +image apev2:image->image - - + + - + jpeg - -jpeg - -exif - -icc_profile + +jpeg + +exif + +icc_profile - + image->jpeg:jpeg - - + + - + mp4 - -mp4 - -aac_frame - -av1_ccr - -av1_frame - -flac_frame - -flac_metadatablocks - -id3v2 - -image - -jpeg - -mp3_frame - -avc_au - -avc_dcr - -mpeg_es - -hevc_au - -hevc_dcr - -mpeg_pes_packet - -opus_packet - -protobuf_widevine - -pssh_playready - -vorbis_packet - -vp9_frame - -vpx_ccr + +mp4 + +aac_frame + +av1_ccr + +av1_frame + +flac_frame + +flac_metadatablocks + +id3v2 + +image + +jpeg + +mp3_frame + +avc_au + +avc_dcr + +mpeg_es + +hevc_au + +hevc_dcr + +mpeg_pes_packet + +opus_packet + +protobuf_widevine + +pssh_playready + +vorbis_packet + +vp9_frame + +vpx_ccr - + image->mp4:mp4 - - + + png - -png - -icc_profile - -exif + +png + +icc_profile + +exif - + image->png:png - - + + - + tiff - -tiff - -icc_profile + +tiff + +icc_profile - + image->tiff:tiff - - + + - + webp - -webp - -vp8_frame + +webp + +vp8_frame - + image->webp:webp - - + + - + gif - -gif + +gif - + image->gif:gif - - + + av1_frame - -av1_frame - -av1_obu + +av1_frame + +av1_obu av1_obu - -av1_obu + +av1_obu av1_frame:av1_obu->av1_obu - - + + avc_annexb - -avc_annexb - -avc_nalu + +avc_annexb + +avc_nalu avc_nalu - -avc_nalu - -avc_sps - -avc_pps - -avc_sei + +avc_nalu + +avc_sps + +avc_pps + +avc_sei avc_annexb:avc_nalu->avc_nalu - - + + avc_sps - -avc_sps + +avc_sps avc_nalu:avc_sps->avc_sps - - + + avc_pps - -avc_pps + +avc_pps avc_nalu:avc_pps->avc_pps - - + + avc_sei - -avc_sei + +avc_sei avc_nalu:avc_sei->avc_sei - - + + avc_au - -avc_au - -avc_nalu + +avc_au + +avc_nalu avc_au:avc_nalu->avc_nalu - - + + avc_dcr - -avc_dcr - -avc_nalu + +avc_dcr + +avc_nalu avc_dcr:avc_nalu->avc_nalu - - + + + + + +bsd_loopback_frame + +bsd_loopback_frame + +ipv4_packet + + + +ipv4_packet + +ipv4_packet + +udp_datagram + +tcp_segment + +icmp + + + +bsd_loopback_frame:ipv4_packet->ipv4_packet + + + + +udp_datagram + +udp_datagram + +udp_payload + + + +ipv4_packet:udp_datagram->udp_datagram + + + + + +tcp_segment + +tcp_segment + + + +ipv4_packet:tcp_segment->tcp_segment + + + + + +icmp + +icmp + + + +ipv4_packet:icmp->icmp + + - + bzip2 - -bzip2 - -probe + +bzip2 + +probe - + probe - -probe + +probe - + bzip2:probe->probe - - + + - + probe->adts:adts - - + + - + probe->bzip2:bzip2 - - + + - + flac - -flac - -flac_metadatablocks - -flac_frame + +flac + +flac_metadatablocks + +flac_frame - + probe->flac:flac - - - + + + - + gzip - -gzip - -probe + +gzip + +probe - + probe->gzip:gzip - - + + - + probe->jpeg:jpeg - - - + + - + matroska - -matroska - -aac_frame - -av1_ccr - -av1_frame - -avc_au - -avc_dcr - -flac_frame - -flac_metadatablocks - -hevc_au - -hevc_dcr - -image - -mp3_frame - -mpeg_asc - -mpeg_pes_packet - -mpeg_spu - -opus_packet - -vorbis_packet - -vp8_frame - -vp9_cfm - -vp9_frame + +matroska + +aac_frame + +av1_ccr + +av1_frame + +avc_au + +avc_dcr + +flac_frame + +flac_metadatablocks + +hevc_au + +hevc_dcr + +image + +mp3_frame + +mpeg_asc + +mpeg_pes_packet + +mpeg_spu + +opus_packet + +vorbis_packet + +vp8_frame + +vp9_cfm + +vp9_frame - + probe->matroska:matroska - - + + + - + mp3 - -mp3 - -id3v2 - -id3v1 - -id3v11 - -apev2 - -mp3_frame + +mp3 + +id3v2 + +id3v1 + +id3v11 + +apev2 + +mp3_frame - + probe->mp3:mp3 - - + + - + probe->mp4:mp4 - - - + + - + ogg - -ogg - -ogg_page - -vorbis_packet - -opus_packet - -flac_metadatablock - -flac_frame + +ogg + +ogg_page + +vorbis_packet + +opus_packet + +flac_metadatablock + +flac_frame - + probe->ogg:ogg - - + + - + pcap - -pcap - -ether8023_frame - -sll_packet - -sll2_packet - -tcp_stream - -ipv4_packet + +pcap + +link_frame + +tcp_stream + +ipv4_packet - + probe->pcap:pcap - - + + pcapng - -pcapng - -ether8023_frame - -sll_packet - -sll2_packet - -tcp_stream - -ipv4_packet + +pcapng + +link_frame + +tcp_stream + +ipv4_packet - + probe->pcapng:pcapng - - + + - + probe->png:png - - + + + - + tar - -tar - -probe + +tar + +probe - + probe->tar:tar - - + + - + probe->tiff:tiff - - - - + + - + wav - -wav - -id3v2 - -id3v1 - -id3v11 + +wav + +id3v2 + +id3v1 + +id3v11 - + probe->wav:wav - - + + - + probe->webp:webp - - - + + + - + zip - -zip - -probe + +zip + +probe - + probe->zip:zip - - + + - + elf - -elf + +elf - + probe->elf:elf - - + + - + probe->gif:gif - - + + + - + json - -json + +json - + probe->json:json - - + + - + mpeg_ts - -mpeg_ts + +mpeg_ts - + probe->mpeg_ts:mpeg_ts - - + + - + ether8023_frame - -ether8023_frame - -ipv4_packet - - - -ipv4_packet - -ipv4_packet - -udp_datagram - -tcp_segment - -icmp + +ether8023_frame + +ipv4_packet - + ether8023_frame:ipv4_packet->ipv4_packet - - - - - -udp_datagram - -udp_datagram - -udp_payload - - - -ipv4_packet:udp_datagram->udp_datagram - - - - - -tcp_segment - -tcp_segment - - - -ipv4_packet:tcp_segment->tcp_segment - - - - - -icmp - -icmp - - - -ipv4_packet:icmp->icmp - - + + - + flac_metadatablocks - -flac_metadatablocks - -flac_metadatablock + +flac_metadatablocks + +flac_metadatablock - + flac:flac_metadatablocks->flac_metadatablocks - - + + - + flac_frame - -flac_frame + +flac_frame - + flac:flac_frame->flac_frame - - + + - + flac_metadatablock - -flac_metadatablock - -flac_streaminfo - -flac_picture - -vorbis_comment + +flac_metadatablock + +flac_streaminfo + +flac_picture + +vorbis_comment - + flac_metadatablocks:flac_metadatablock->flac_metadatablock - - + + - + flac_streaminfo - -flac_streaminfo + +flac_streaminfo - + flac_metadatablock:flac_streaminfo->flac_streaminfo - - + + - + flac_picture - -flac_picture - -image + +flac_picture + +image - + flac_metadatablock:flac_picture->flac_picture - - + + - + vorbis_comment - -vorbis_comment - -flac_picture + +vorbis_comment + +flac_picture - + flac_metadatablock:vorbis_comment->vorbis_comment - - + + - + flac_picture:image->image - - - + + - + vorbis_comment:flac_picture->flac_picture - - + + - + gzip:probe->probe - - + + - + hevc_annexb - -hevc_annexb - -hevc_nalu + +hevc_annexb + +hevc_nalu - + hevc_nalu - -hevc_nalu + +hevc_nalu - + hevc_annexb:hevc_nalu->hevc_nalu - - + + - + hevc_au - -hevc_au - -hevc_nalu + +hevc_au + +hevc_nalu - + hevc_au:hevc_nalu->hevc_nalu - - + + - + hevc_dcr - -hevc_dcr - -hevc_nalu + +hevc_dcr + +hevc_nalu - + hevc_dcr:hevc_nalu->hevc_nalu - - + + - + id3v2 - -id3v2 - -image + +id3v2 + +image - + id3v2:image->image - - + + + - + udp_payload - -udp_payload + +udp_payload - + udp_datagram:udp_payload->udp_payload - - + + - + exif - -exif + +exif - + jpeg:exif->exif - - + + - + icc_profile - -icc_profile + +icc_profile - + jpeg:icc_profile->icc_profile - - + + - + matroska:aac_frame->aac_frame - + - + matroska:image->image - - + + - + matroska:av1_frame->av1_frame - - + + - + matroska:avc_au->avc_au - - + + - + matroska:avc_dcr->avc_dcr - - + + - + matroska:flac_metadatablocks->flac_metadatablocks - - + + - + matroska:flac_frame->flac_frame - - + + - + matroska:hevc_au->hevc_au - - + + - + matroska:hevc_dcr->hevc_dcr - - + + - + av1_ccr - -av1_ccr + +av1_ccr - + matroska:av1_ccr->av1_ccr - - + + - + mp3_frame - -mp3_frame - -xing + +mp3_frame + +xing - + matroska:mp3_frame->mp3_frame - - + + - + mpeg_asc - -mpeg_asc + +mpeg_asc - + matroska:mpeg_asc->mpeg_asc - - + + - + mpeg_pes_packet - -mpeg_pes_packet + +mpeg_pes_packet - + matroska:mpeg_pes_packet->mpeg_pes_packet - - + + + - + mpeg_spu - -mpeg_spu + +mpeg_spu - + matroska:mpeg_spu->mpeg_spu - - + + - + opus_packet - -opus_packet - -vorbis_comment + +opus_packet + +vorbis_comment - + matroska:opus_packet->opus_packet - - + + - + vorbis_packet - -vorbis_packet - -vorbis_comment + +vorbis_packet + +vorbis_comment - + matroska:vorbis_packet->vorbis_packet - - + + - + vp8_frame - -vp8_frame + +vp8_frame - + matroska:vp8_frame->vp8_frame - - + + - + vp9_cfm - -vp9_cfm + +vp9_cfm - + matroska:vp9_cfm->vp9_cfm - - + + - + vp9_frame - -vp9_frame + +vp9_frame - + matroska:vp9_frame->vp9_frame - - + + - + xing - -xing + +xing - + mp3_frame:xing->xing - - + + - + opus_packet:vorbis_comment->vorbis_comment - - + + - + vorbis_packet:vorbis_comment->vorbis_comment - - + + - + mp3:apev2->apev2 - - + + - + mp3:id3v2->id3v2 - - - + + - + mp3:mp3_frame->mp3_frame - - + + - + id3v1 - -id3v1 + +id3v1 - + mp3:id3v1->id3v1 - - + + - + id3v11 - -id3v11 + +id3v11 - + mp3:id3v11->id3v11 - - + + - + mp4:aac_frame->aac_frame - - + + - + mp4:image->image - - + + - + mp4:av1_frame->av1_frame - - + + - + mp4:avc_au->avc_au - - + + - + mp4:avc_dcr->avc_dcr - - + + - + mp4:flac_metadatablocks->flac_metadatablocks - - + + - + mp4:flac_frame->flac_frame - - + + - + mp4:hevc_au->hevc_au - - + + - + mp4:hevc_dcr->hevc_dcr - - + + - + mp4:id3v2->id3v2 - - + + - + mp4:jpeg->jpeg - - + + - + mp4:av1_ccr->av1_ccr - - + + - + mp4:mp3_frame->mp3_frame - - + + - + mp4:mpeg_pes_packet->mpeg_pes_packet - - + + - + mp4:opus_packet->opus_packet - - + + - + mp4:vorbis_packet->vorbis_packet - - + + - + mp4:vp9_frame->vp9_frame - - + + - + mpeg_es - -mpeg_es - -mpeg_asc - -vorbis_packet + +mpeg_es + +mpeg_asc + +vorbis_packet - + mp4:mpeg_es->mpeg_es - - + + - + protobuf_widevine - -protobuf_widevine - -protobuf + +protobuf_widevine + +protobuf - + mp4:protobuf_widevine->protobuf_widevine - - + + - + pssh_playready - -pssh_playready + +pssh_playready - + mp4:pssh_playready->pssh_playready - - + + - + vpx_ccr - -vpx_ccr + +vpx_ccr - + mp4:vpx_ccr->vpx_ccr - - + + - + mpeg_es:mpeg_asc->mpeg_asc - - + + - + mpeg_es:vorbis_packet->vorbis_packet - - + + protobuf - -protobuf + +protobuf - + protobuf_widevine:protobuf->protobuf - - + + - + mpeg_pes - -mpeg_pes - -mpeg_pes_packet - -mpeg_spu + +mpeg_pes + +mpeg_pes_packet + +mpeg_spu - + mpeg_pes:mpeg_pes_packet->mpeg_pes_packet - - + - + mpeg_pes:mpeg_spu->mpeg_spu - - + + - + ogg:flac_frame->flac_frame - - + + - + ogg:flac_metadatablock->flac_metadatablock - - + + - + ogg:opus_packet->opus_packet - - + + - + ogg:vorbis_packet->vorbis_packet - - + + - + ogg_page - -ogg_page + +ogg_page - + ogg:ogg_page->ogg_page - - - - - -pcap:ether8023_frame->ether8023_frame - - + + - + pcap:ipv4_packet->ipv4_packet - - + + + - - -sll_packet - -sll_packet - -ether8023_frame - - - -pcap:sll_packet->sll_packet - - - - + -sll2_packet - -sll2_packet - -ether8023_frame +link_frame + +link_frame - - -pcap:sll2_packet->sll2_packet - - + + +pcap:link_frame->link_frame + + tcp_stream - -tcp_stream + +tcp_stream - + pcap:tcp_stream->tcp_stream - - + + - - -sll_packet:ether8023_frame->ether8023_frame - - + + +link_frame->bsd_loopback_frame:bsd_loopback_frame + + - - -sll2_packet:ether8023_frame->ether8023_frame - - + + +link_frame->ether8023_frame:ether8023_frame + + + + + +sll2_packet + +sll2_packet + +ether8023_frame + + + +link_frame->sll2_packet:sll2_packet + + + + + +sll_packet + +sll_packet + +ether8023_frame + + + +link_frame->sll_packet:sll_packet + + - + dns - -dns + +dns - + tcp_stream->dns:dns - - - - - -pcapng:ether8023_frame->ether8023_frame - - + + - -pcapng:ipv4_packet->ipv4_packet - - - - -pcapng:sll_packet->sll_packet - - +pcapng:ipv4_packet->ipv4_packet + + - - -pcapng:sll2_packet->sll2_packet - - + + +pcapng:link_frame->link_frame + + - + pcapng:tcp_stream->tcp_stream - - + + - + png:exif->exif - - + + - + png:icc_profile->icc_profile - + + + + + + +sll2_packet:ether8023_frame->ether8023_frame + + + + + +sll_packet:ether8023_frame->ether8023_frame + + - + tar:probe->probe - - + + - + tiff:icc_profile->icc_profile - - - + - + udp_payload->dns:dns - - + + - + wav:id3v2->id3v2 - + + - + wav:id3v1->id3v1 - - + + - + wav:id3v11->id3v11 - - + + - + webp:vp8_frame->vp8_frame - - + + - + zip:probe->probe - - + + - + dns_tcp - -dns_tcp + +dns_tcp - + raw - -raw + +raw diff --git a/format/dns/dns.go b/format/dns/dns.go index d7311f98..59a64e18 100644 --- a/format/dns/dns.go +++ b/format/dns/dns.go @@ -271,9 +271,9 @@ func dnsUDPDecode(d *decode.D, in interface{}) interface{} { } d.Fatalf("wrong port") } - if udi, ok := in.(format.UDPDatagramIn); ok { - if udi.DestinationPort == format.UDPPortDomain || udi.SourcePort == format.UDPPortDomain || - udi.DestinationPort == format.UDPPortMDNS || udi.SourcePort == format.UDPPortMDNS { + if upi, ok := in.(format.UDPPayloadIn); ok { + if upi.DestinationPort == format.UDPPortDomain || upi.SourcePort == format.UDPPortDomain || + upi.DestinationPort == format.UDPPortMDNS || upi.SourcePort == format.UDPPortMDNS { return dnsDecode(d, false) } d.Fatalf("wrong port") diff --git a/format/format.go b/format/format.go index 0874fe9f..fd7a93a4 100644 --- a/format/format.go +++ b/format/format.go @@ -8,19 +8,21 @@ const ( IMAGE = "image" TCP_STREAM = "tcp_stream" UDP_PAYLOAD = "udp_payload" + LINK_FRAME = "link_frame" RAW = "raw" JSON = "json" - DNS = "dns" - DNS_TCP = "dns_tcp" - ETHER8023_FRAME = "ether8023_frame" - SLL_PACKET = "sll_packet" - SLL2_PACKET = "sll2_packet" - IPV4_PACKET = "ipv4_packet" - UDP_DATAGRAM = "udp_datagram" - TCP_SEGMENT = "tcp_segment" - ICMP = "icmp" + DNS = "dns" + DNS_TCP = "dns_tcp" + ETHER8023_FRAME = "ether8023_frame" + BSD_LOOPBACK_FRAME = "bsd_loopback_frame" + SLL_PACKET = "sll_packet" + SLL2_PACKET = "sll2_packet" + IPV4_PACKET = "ipv4_packet" + UDP_DATAGRAM = "udp_datagram" + TCP_SEGMENT = "tcp_segment" + ICMP = "icmp" AAC_FRAME = "aac_frame" ADTS = "adts" @@ -187,7 +189,17 @@ type MP3FrameOut struct { ChannelModeIndex int } -type UDPDatagramIn struct { +type In struct { + SourcePort int + DestinationPort int +} + +type LinkFrameIn struct { + Type int + LittleEndian bool // pcap endian etc +} + +type UDPPayloadIn struct { SourcePort int DestinationPort int } diff --git a/format/inet/bsd_loopback_frame.go b/format/inet/bsd_loopback_frame.go new file mode 100644 index 00000000..8611a7e8 --- /dev/null +++ b/format/inet/bsd_loopback_frame.go @@ -0,0 +1,58 @@ +package inet + +// TODO: rename NetworkLayer? wireshark calls it "Family", pcap-linktype(7) calls it "network-layer protocol" + +import ( + "github.com/wader/fq/format" + "github.com/wader/fq/format/registry" + "github.com/wader/fq/pkg/decode" + "github.com/wader/fq/pkg/scalar" +) + +var bsdLoopbackFrameIPv4Format decode.Group + +func init() { + registry.MustRegister(decode.Format{ + Name: format.BSD_LOOPBACK_FRAME, + Description: "BSD loopback frame", + Groups: []string{format.LINK_FRAME}, + Dependencies: []decode.Dependency{ + {Names: []string{format.IPV4_PACKET}, Group: &bsdLoopbackFrameIPv4Format}, + }, + DecodeFn: decodeLoopbackFrame, + }) +} + +const ( + bsdLoopbackNetworkLayerIPv4 = 2 +) + +var bsdLoopbackFrameNetworkLayerFormat = map[uint64]*decode.Group{ + bsdLoopbackNetworkLayerIPv4: &bsdLoopbackFrameIPv4Format, +} + +var bsdLookbackNetworkLayerMap = scalar.UToScalar{ + bsdLoopbackNetworkLayerIPv4: {Sym: "ipv4", Description: `Internet protocol v4`}, +} + +func decodeLoopbackFrame(d *decode.D, in interface{}) interface{} { + lsi, ok := in.(format.LinkFrameIn) + if ok { + if lsi.Type != format.LinkTypeNULL { + d.Fatalf("wrong link type") + } + if lsi.LittleEndian { + d.Endian = decode.LittleEndian + } + } + // if no LinkFrameIn assume big endian for now + + networkLayer := d.FieldU32("network_layer", bsdLookbackNetworkLayerMap, scalar.Hex) + if g, ok := bsdLoopbackFrameNetworkLayerFormat[networkLayer]; ok { + d.FieldFormatLen("packet", d.BitsLeft(), *g, nil) + } else { + d.FieldRawLen("data", d.BitsLeft()) + } + + return nil +} diff --git a/format/inet/ether8023_frame.go b/format/inet/ether8023_frame.go index b9265ad9..07e8a7bb 100644 --- a/format/inet/ether8023_frame.go +++ b/format/inet/ether8023_frame.go @@ -18,10 +18,11 @@ func init() { registry.MustRegister(decode.Format{ Name: format.ETHER8023_FRAME, Description: "Ethernet 802.3 frame", + Groups: []string{format.LINK_FRAME}, Dependencies: []decode.Dependency{ {Names: []string{format.IPV4_PACKET}, Group: ðer8023FrameIPv4Format}, }, - DecodeFn: decodeEthernet, + DecodeFn: decodeEthernetFrame, }) } @@ -37,7 +38,13 @@ var mapUToEtherSym = scalar.Fn(func(s scalar.S) (scalar.S, error) { return s, nil }) -func decodeEthernet(d *decode.D, in interface{}) interface{} { +func decodeEthernetFrame(d *decode.D, in interface{}) interface{} { + if lsi, ok := in.(format.LinkFrameIn); ok { + if lsi.Type != format.LinkTypeETHERNET { + d.Fatalf("wrong link type") + } + } + d.FieldU("destination", 48, mapUToEtherSym, scalar.Hex) d.FieldU("source", 48, mapUToEtherSym, scalar.Hex) etherType := d.FieldU16("ether_type", format.EtherTypeMap, scalar.Hex) diff --git a/format/inet/flowsdecoder/flowsdecoder.go b/format/inet/flowsdecoder/flowsdecoder.go index b6719a6d..ca036ff9 100644 --- a/format/inet/flowsdecoder/flowsdecoder.go +++ b/format/inet/flowsdecoder/flowsdecoder.go @@ -141,6 +141,10 @@ func (fd *Decoder) EthernetFrame(bs []byte) error { return fd.packet(gopacket.NewPacket(bs, layers.LayerTypeEthernet, gopacket.Lazy)) } +func (fd *Decoder) LoopbackFrame(bs []byte) error { + return fd.packet(gopacket.NewPacket(bs, layers.LayerTypeLoopback, gopacket.Lazy)) +} + func (fd *Decoder) packet(p gopacket.Packet) error { // TODO: linkType ip4Layer := p.Layer(layers.LayerTypeIPv4) diff --git a/format/inet/sll2_packet.go b/format/inet/sll2_packet.go index 64f5ad4a..ab1670bd 100644 --- a/format/inet/sll2_packet.go +++ b/format/inet/sll2_packet.go @@ -16,6 +16,7 @@ func init() { registry.MustRegister(decode.Format{ Name: format.SLL2_PACKET, Description: "Linux cooked capture encapsulation v2", + Groups: []string{format.LINK_FRAME}, Dependencies: []decode.Dependency{ {Names: []string{format.ETHER8023_FRAME}, Group: &sllPacket2Ether8023Format}, }, @@ -28,6 +29,12 @@ var sllPacket2FrameTypeFormat = map[uint64]*decode.Group{ } func decodeSLL2(d *decode.D, in interface{}) interface{} { + if lsi, ok := in.(format.LinkFrameIn); ok { + if lsi.Type != format.LinkTypeLINUX_SLL2 { + d.Fatalf("wrong link type") + } + } + protcolType := d.FieldU16("protocol_type", format.EtherTypeMap, scalar.Hex) d.FieldU16("reserved") d.FieldU32("interface_index") diff --git a/format/inet/sll_packet.go b/format/inet/sll_packet.go index 2adc9207..b208c49e 100644 --- a/format/inet/sll_packet.go +++ b/format/inet/sll_packet.go @@ -16,6 +16,7 @@ func init() { registry.MustRegister(decode.Format{ Name: format.SLL_PACKET, Description: "Linux cooked capture encapsulation", + Groups: []string{format.LINK_FRAME}, Dependencies: []decode.Dependency{ {Names: []string{format.ETHER8023_FRAME}, Group: &sllPacketEther8023Format}, }, @@ -112,6 +113,12 @@ var arpHdrTypeMAp = scalar.UToScalar{ } func decodeSLL(d *decode.D, in interface{}) interface{} { + if lsi, ok := in.(format.LinkFrameIn); ok { + if lsi.Type != format.LinkTypeLINUX_SLL { + d.Fatalf("wrong link type") + } + } + d.FieldU16("packet_type", sllPacketTypeMap) arpHdrType := d.FieldU16("arphdr_type", arpHdrTypeMAp) addressLength := d.FieldU16("link_address_length") diff --git a/format/inet/udp_datagram.go b/format/inet/udp_datagram.go index 5f5cd345..cf25e1bf 100644 --- a/format/inet/udp_datagram.go +++ b/format/inet/udp_datagram.go @@ -27,7 +27,7 @@ func decodeUDP(d *decode.D, in interface{}) interface{} { d.FieldU16("checksum", scalar.Hex) dataLen := int64(length-8) * 8 - if dv, _, _ := d.TryFieldFormatLen("data", dataLen, udpPayloadFormat, format.UDPDatagramIn{ + if dv, _, _ := d.TryFieldFormatLen("data", dataLen, udpPayloadFormat, format.UDPPayloadIn{ SourcePort: int(soucePort), DestinationPort: int(destPort), }); dv == nil { diff --git a/format/pcap/pcap.go b/format/pcap/pcap.go index 0a11ad17..aad3a152 100644 --- a/format/pcap/pcap.go +++ b/format/pcap/pcap.go @@ -11,9 +11,7 @@ import ( "github.com/wader/fq/pkg/scalar" ) -var pcapEther8023Format decode.Group -var pcapSLLPacket decode.Group -var pcapSLL2Packet decode.Group +var pcapLinkFrameFormat decode.Group var pcapTCPStreamFormat decode.Group var pcapIPv4PacketFormat decode.Group @@ -33,9 +31,7 @@ func init() { Description: "PCAP packet capture", Groups: []string{format.PROBE}, Dependencies: []decode.Dependency{ - {Names: []string{format.ETHER8023_FRAME}, Group: &pcapEther8023Format}, - {Names: []string{format.SLL_PACKET}, Group: &pcapSLLPacket}, - {Names: []string{format.SLL2_PACKET}, Group: &pcapSLL2Packet}, + {Names: []string{format.LINK_FRAME}, Group: &pcapLinkFrameFormat}, {Names: []string{format.TCP_STREAM}, Group: &pcapTCPStreamFormat}, {Names: []string{format.IPV4_PACKET}, Group: &pcapIPv4PacketFormat}, }, @@ -89,9 +85,10 @@ func decodePcap(d *decode.D, in interface{}) interface{} { _ = fn(fd, bs) } - if g, ok := linkToFormat[linkType]; ok { - d.FieldFormatLen("packet", int64(inclLen)*8, *g, nil) - } else { + if dv, _, _ := d.TryFieldFormatLen("packet", int64(inclLen)*8, pcapLinkFrameFormat, format.LinkFrameIn{ + Type: linkType, + LittleEndian: d.Endian == decode.LittleEndian, + }); dv == nil { d.FieldRawLen("packet", int64(inclLen)*8) } }) diff --git a/format/pcap/pcapng.go b/format/pcap/pcapng.go index 39321854..e23da781 100644 --- a/format/pcap/pcapng.go +++ b/format/pcap/pcapng.go @@ -13,9 +13,7 @@ import ( "github.com/wader/fq/pkg/scalar" ) -var pcapngEther8023Format decode.Group -var pcapngSLLPacketFormat decode.Group -var pcapngSLL2PacketFormat decode.Group +var pcapngLinkFrameFormat decode.Group var pcapngTCPStreamFormat decode.Group var pcapngIPvPacket4Format decode.Group @@ -26,9 +24,7 @@ func init() { RootArray: true, Groups: []string{format.PROBE}, Dependencies: []decode.Dependency{ - {Names: []string{format.ETHER8023_FRAME}, Group: &pcapngEther8023Format}, - {Names: []string{format.SLL_PACKET}, Group: &pcapngSLLPacketFormat}, - {Names: []string{format.SLL2_PACKET}, Group: &pcapngSLL2PacketFormat}, + {Names: []string{format.LINK_FRAME}, Group: &pcapngLinkFrameFormat}, {Names: []string{format.TCP_STREAM}, Group: &pcapngTCPStreamFormat}, {Names: []string{format.IPV4_PACKET}, Group: &pcapngIPvPacket4Format}, }, @@ -244,12 +240,12 @@ var blockFns = map[uint64]func(d *decode.D, dc *decodeContext){ if fn, ok := linkToDecodeFn[linkType]; ok { // TODO: report decode errors _ = fn(dc.flowDecoder, bs) - _ = fn(dc.flowDecoder, bs) } - if g, ok := linkToFormat[linkType]; ok { - d.FieldFormatLen("packet", int64(capturedLength)*8, *g, nil) - } else { + if dv, _, _ := d.TryFieldFormatLen("packet", int64(capturedLength)*8, pcapngLinkFrameFormat, format.LinkFrameIn{ + Type: linkType, + LittleEndian: d.Endian == decode.LittleEndian, + }); dv == nil { d.FieldRawLen("packet", int64(capturedLength)*8) } diff --git a/format/pcap/shared.go b/format/pcap/shared.go index 716d3ef4..d4092954 100644 --- a/format/pcap/shared.go +++ b/format/pcap/shared.go @@ -9,14 +9,8 @@ import ( "github.com/wader/fq/pkg/decode" ) -// TODO: is shared between pcap and pcapng -var linkToFormat = map[int]*decode.Group{ - format.LinkTypeETHERNET: &pcapngEther8023Format, - format.LinkTypeLINUX_SLL: &pcapngSLLPacketFormat, - format.LinkTypeLINUX_SLL2: &pcapngSLL2PacketFormat, -} - var linkToDecodeFn = map[int]func(fd *flowsdecoder.Decoder, bs []byte) error{ + format.LinkTypeNULL: (*flowsdecoder.Decoder).LoopbackFrame, format.LinkTypeETHERNET: (*flowsdecoder.Decoder).EthernetFrame, format.LinkTypeLINUX_SLL: (*flowsdecoder.Decoder).SLLPacket, format.LinkTypeLINUX_SLL2: func(fd *flowsdecoder.Decoder, bs []byte) error { diff --git a/format/pcap/testdata/many_interfaces.fqtest b/format/pcap/testdata/many_interfaces.fqtest index 83140aea..6fc9ef75 100644 --- a/format/pcap/testdata/many_interfaces.fqtest +++ b/format/pcap/testdata/many_interfaces.fqtest @@ -490,9 +490,32 @@ $ fq -d pcapng verbose /many_interfaces.pcapng 0x0750| c0 6d 62 c9 | .mb. | timestamp_low: 3378671040 0x758-0x75b.7 (4) 0x0750| a8 00 00 00| ....| capture_packet_length: 168 0x75c-0x75f.7 (4) 0x0760|a8 00 00 00 |.... | original_packet_length: 168 0x760-0x763.7 (4) -0x0760| 02 00 00 00 45 00 00 a4 c6 ce 00 00| ....E.......| packet: raw bits 0x764-0x80b.7 (168) -0x0770|40 11 f1 47 c0 a8 01 8b ff ff ff ff 44 5c 44 5c|@..G........D\D\| -* |until 0x80b.7 (168) | | + | | | packet{}: (bsd_loopback_frame) 0x764-0x80b.7 (168) +0x0760| 02 00 00 00 | .... | network_layer: "ipv4" (0x2) (Internet protocol v4) 0x764-0x767.7 (4) + | | | packet{}: (ipv4_packet) 0x768-0x80b.7 (164) +0x0760| 45 | E | version: 4 0x768-0x768.3 (0.4) +0x0760| 45 | E | ihl: 5 0x768.4-0x768.7 (0.4) +0x0760| 00 | . | dscp: 0 0x769-0x769.5 (0.6) +0x0760| 00 | . | ecn: 0 0x769.6-0x769.7 (0.2) +0x0760| 00 a4 | .. | total_length: 164 0x76a-0x76b.7 (2) +0x0760| c6 ce | .. | identification: 50894 0x76c-0x76d.7 (2) +0x0760| 00 | . | reserved: 0 0x76e-0x76e (0.1) +0x0760| 00 | . | dont_fragment: false 0x76e.1-0x76e.1 (0.1) +0x0760| 00 | . | more_fragments: false 0x76e.2-0x76e.2 (0.1) +0x0760| 00 00| ..| fragment_offset: 0 0x76e.3-0x76f.7 (1.5) +0x0770|40 |@ | ttl: 64 0x770-0x770.7 (1) +0x0770| 11 | . | protocol: "udp" (17) (User datagram protocol) 0x771-0x771.7 (1) +0x0770| f1 47 | .G | header_checksum: 0xf147 (valid) 0x772-0x773.7 (2) +0x0770| c0 a8 01 8b | .... | source_ip: "192.168.1.139" (0xc0a8018b) 0x774-0x777.7 (4) +0x0770| ff ff ff ff | .... | destination_ip: "255.255.255.255" (0xffffffff) 0x778-0x77b.7 (4) + | | | data{}: (udp_datagram) 0x77c-0x80b.7 (144) +0x0770| 44 5c | D\ | source_port: 17500 0x77c-0x77d.7 (2) +0x0770| 44 5c| D\| destination_port: 17500 0x77e-0x77f.7 (2) +0x0780|00 90 |.. | length: 144 0x780-0x781.7 (2) +0x0780| ba 03 | .. | checksum: 0xba03 0x782-0x783.7 (2) +0x0780| 7b 22 68 6f 73 74 5f 69 6e 74 22 3a| {"host_int":| data: raw bits 0x784-0x80b.7 (136) +0x0790|20 34 30 39 34 35 31 34 34 38 33 2c 20 22 76 65| 4094514483, "ve| +* |until 0x80b.7 (136) | | | | | padding: raw bits 0x80c-NA (0) | | | options[0:0]: 0x80c-NA (0) 0x0800| c8 00 00 00| ....| footer_length: 200 0x80c-0x80f.7 (4) @@ -504,9 +527,32 @@ $ fq -d pcapng verbose /many_interfaces.pcapng 0x0820|be 6e 62 c9 |.nb. | timestamp_low: 3378671294 0x820-0x823.7 (4) 0x0820| a8 00 00 00 | .... | capture_packet_length: 168 0x824-0x827.7 (4) 0x0820| a8 00 00 00 | .... | original_packet_length: 168 0x828-0x82b.7 (4) -0x0820| 02 00 00 00| ....| packet: raw bits 0x82c-0x8d3.7 (168) -0x0830|45 00 00 a4 60 b4 00 00 40 11 94 ba c0 a8 01 8b|E...`...@.......| -* |until 0x8d3.7 (168) | | + | | | packet{}: (bsd_loopback_frame) 0x82c-0x8d3.7 (168) +0x0820| 02 00 00 00| ....| network_layer: "ipv4" (0x2) (Internet protocol v4) 0x82c-0x82f.7 (4) + | | | packet{}: (ipv4_packet) 0x830-0x8d3.7 (164) +0x0830|45 |E | version: 4 0x830-0x830.3 (0.4) +0x0830|45 |E | ihl: 5 0x830.4-0x830.7 (0.4) +0x0830| 00 | . | dscp: 0 0x831-0x831.5 (0.6) +0x0830| 00 | . | ecn: 0 0x831.6-0x831.7 (0.2) +0x0830| 00 a4 | .. | total_length: 164 0x832-0x833.7 (2) +0x0830| 60 b4 | `. | identification: 24756 0x834-0x835.7 (2) +0x0830| 00 | . | reserved: 0 0x836-0x836 (0.1) +0x0830| 00 | . | dont_fragment: false 0x836.1-0x836.1 (0.1) +0x0830| 00 | . | more_fragments: false 0x836.2-0x836.2 (0.1) +0x0830| 00 00 | .. | fragment_offset: 0 0x836.3-0x837.7 (1.5) +0x0830| 40 | @ | ttl: 64 0x838-0x838.7 (1) +0x0830| 11 | . | protocol: "udp" (17) (User datagram protocol) 0x839-0x839.7 (1) +0x0830| 94 ba | .. | header_checksum: 0x94ba (valid) 0x83a-0x83b.7 (2) +0x0830| c0 a8 01 8b| ....| source_ip: "192.168.1.139" (0xc0a8018b) 0x83c-0x83f.7 (4) +0x0840|c0 a8 01 ff |.... | destination_ip: "192.168.1.255" (0xc0a801ff) 0x840-0x843.7 (4) + | | | data{}: (udp_datagram) 0x844-0x8d3.7 (144) +0x0840| 44 5c | D\ | source_port: 17500 0x844-0x845.7 (2) +0x0840| 44 5c | D\ | destination_port: 17500 0x846-0x847.7 (2) +0x0840| 00 90 | .. | length: 144 0x848-0x849.7 (2) +0x0840| f7 5b | .[ | checksum: 0xf75b 0x84a-0x84b.7 (2) +0x0840| 7b 22 68 6f| {"ho| data: raw bits 0x84c-0x8d3.7 (136) +0x0850|73 74 5f 69 6e 74 22 3a 20 34 30 39 34 35 31 34|st_int": 4094514| +* |until 0x8d3.7 (136) | | | | | padding: raw bits 0x8d4-NA (0) | | | options[0:0]: 0x8d4-NA (0) 0x08d0| c8 00 00 00 | .... | footer_length: 200 0x8d4-0x8d7.7 (4) diff --git a/pkg/interp/testdata/args.fqtest b/pkg/interp/testdata/args.fqtest index cb47f234..2cb1ba76 100644 --- a/pkg/interp/testdata/args.fqtest +++ b/pkg/interp/testdata/args.fqtest @@ -65,6 +65,7 @@ avc_nalu H.264/AVC Network Access Layer Unit avc_pps H.264/AVC Picture Parameter Set avc_sei H.264/AVC Supplemental Enhancement Information avc_sps H.264/AVC Sequence Parameter Set +bsd_loopback_frame BSD loopback frame bzip2 bzip2 compression dns DNS packet dns_tcp DNS packet (TCP)