wader/fq
1
1
Fork 0
mirror of https://github.com/wader/fq.git synced 2024-12-23 21:31:33 +03:00
Code Issues Projects Releases Wiki Activity

pcap: Add ns support and add header field

Browse Source
This commit is contained in:
Mattias Wadman 2022-08-18 14:09:42 +02:00
parent da6b9413fd
commit 74c7dc4eaf
7 changed files with 128 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
format/pcap/pcap.go
View File

@ -15,14 +15,22 @@ var pcapLinkFrameFormat decode.Group
var pcapTCPStreamFormat decode.Group
var pcapIPv4PacketFormat decode.Group
// writing application writes 0xa1b2c3d4 in native endian
const (
// timestamp is seconds + microseconds
bigEndian = 0xa1b2c3d4
littleEndian = 0xd4c3b2a1
// timestamp is seconds + nanoseconds
bigEndianNS = 0xa1b23c4d
littleEndianNS = 0x4d3cb2a1
)
var endianMap = scalar.UToSymStr{
bigEndian: "big_endian",
littleEndian: "little_endian",
bigEndianNS: "big_endian_ns",
littleEndianNS: "little_endian_ns",
}
func init() {
@ -40,29 +48,49 @@ func init() {
}
func decodePcap(d *decode.D, _ any) any {
endian := d.FieldU32("magic", d.AssertU(bigEndian, littleEndian), endianMap, scalar.ActualHex)
switch endian {
var endian decode.Endian
linkType := 0
timestampUNSStr := "ts_usec"
d.FieldStruct("header", func(d *decode.D) {
magic := d.FieldU32("magic", d.AssertU(
bigEndian,
littleEndian,
bigEndianNS,
littleEndianNS,
), endianMap, scalar.ActualHex)
switch magic {
case bigEndian:
d.Endian = decode.BigEndian
endian = decode.BigEndian
case littleEndian:
d.Endian = decode.LittleEndian
default:
d.Fatalf("unknown endian %d", endian)
endian = decode.LittleEndian
case bigEndianNS:
endian = decode.BigEndian
timestampUNSStr = "ts_nsec"
case littleEndianNS:
endian = decode.LittleEndian
timestampUNSStr = "ts_nsec"
}
d.Endian = endian
d.FieldU16("version_major")
d.FieldU16("version_minor")
d.FieldS32("thiszone")
d.FieldU32("sigfigs")
d.FieldU32("snaplen")
linkType := int(d.FieldU32("network", format.LinkTypeMap))
linkType = int(d.FieldU32("network", format.LinkTypeMap))
})
d.Endian = endian
fd := flowsdecoder.New()
d.FieldArray("packets", func(d *decode.D) {
for !d.End() {
d.FieldStruct("packet", func(d *decode.D) {
d.FieldU32("ts_sec")
d.FieldU32("ts_usec")
d.FieldU32(timestampUNSStr)
inclLen := d.FieldU32("incl_len")
origLen := d.FieldU32("orig_len")

1
format/pcap/testdata/http_gzip.fqtest vendored
View File

@ -1,6 +1,7 @@
# from https://wiki.wireshark.org/SampleCaptures
$ fq -d pcap dv http_gzip.cap
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: http_gzip.cap (pcap) 0x0-0x6aa.7 (1707)
| | | header{}: 0x0-0x17.7 (24)
0x00000|d4 c3 b2 a1 |.... | magic: "little_endian" (0xd4c3b2a1) (valid) 0x0-0x3.7 (4)
0x00000| 02 00 | .. | version_major: 2 0x4-0x5.7 (2)
0x00000| 04 00 | .. | version_minor: 4 0x6-0x7.7 (2)

1
format/pcap/testdata/ipv4frags.fqtest vendored
View File

@ -1,6 +1,7 @@
# from https://wiki.wireshark.org/SampleCaptures
$ fq -d pcap dv ipv4frags.pcap
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: ipv4frags.pcap (pcap) 0x0-0xbad.7 (2990)
| | | header{}: 0x0-0x17.7 (24)
0x00000|d4 c3 b2 a1 |.... | magic: "little_endian" (0xd4c3b2a1) (valid) 0x0-0x3.7 (4)
0x00000| 02 00 | .. | version_major: 2 0x4-0x5.7 (2)
0x00000| 04 00 | .. | version_minor: 4 0x6-0x7.7 (2)

1
format/pcap/testdata/ipv6_http.fqtest vendored
View File

@ -1,6 +1,7 @@
# from https://wiki.wireshark.org/SampleCaptures
$ fq -d pcap dv ipv6_http.pcap
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: ipv6_http.pcap (pcap) 0x0-0x23c6.7 (9159)
| | | header{}: 0x0-0x17.7 (24)
0x00000|d4 c3 b2 a1 |.... | magic: "little_endian" (0xd4c3b2a1) (valid) 0x0-0x3.7 (4)
0x00000| 02 00 | .. | version_major: 2 0x4-0x5.7 (2)
0x00000| 04 00 | .. | version_minor: 4 0x6-0x7.7 (2)

50
format/pcap/testdata/ns.fqtest vendored Normal file
View File

@ -0,0 +1,50 @@
# netflowv9.pcap from https://github.com/secdev/scapy/tree/master/test/pcaps
# fq '(.header,.packets[0]) | tobytes' netflowv9.pcap > ns.pcap
$ fq dv ns.pcap
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: ns.pcap (pcap) 0x0-0xc5.7 (198)
| | | header{}: 0x0-0x17.7 (24)
0x00|4d 3c b2 a1 |M<.. | magic: "little_endian_ns" (0x4d3cb2a1) (valid) 0x0-0x3.7 (4)
0x00| 02 00 | .. | version_major: 2 0x4-0x5.7 (2)
0x00| 04 00 | .. | version_minor: 4 0x6-0x7.7 (2)
0x00| 00 00 00 00 | .... | thiszone: 0 0x8-0xb.7 (4)
0x00| 00 00 00 00| ....| sigfigs: 0 0xc-0xf.7 (4)
0x10|ff ff 00 00 |.... | snaplen: 65535 0x10-0x13.7 (4)
0x10| 01 00 00 00 | .... | network: "ethernet" (1) (IEEE 802.3 Ethernet) 0x14-0x17.7 (4)
| | | packets[0:1]: 0x18-0xc5.7 (174)
| | | [0]{}: packet 0x18-0xc5.7 (174)
0x10| 0d 82 e8 59 | ...Y | ts_sec: 1508409869 0x18-0x1b.7 (4)
0x10| 53 c6 50 22| S.P"| ts_nsec: 575718995 0x1c-0x1f.7 (4)
0x20|9e 00 00 00 |.... | incl_len: 158 0x20-0x23.7 (4)
0x20| 9e 00 00 00 | .... | orig_len: 158 0x24-0x27.7 (4)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef| packet{}: (ether8023_frame) 0x28-0xc5.7 (158)
0x20| 00 10 94 00 00 01 | ...... | destination: "00:10:94:00:00:01" (0x1094000001) 0x28-0x2d.7 (6)
0x20| 00 1d| ..| source: "00:1d:b5:cb:28:ce" (0x1db5cb28ce) 0x2e-0x33.7 (6)
0x30|b5 cb 28 ce |..(. |
0x30| 08 00 | .. | ether_type: "ipv4" (0x800) (Internet Protocol version 4) 0x34-0x35.7 (2)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef| payload{}: (ipv4_packet) 0x36-0xc5.7 (144)
0x30| 45 | E | version: 4 0x36-0x36.3 (0.4)
0x30| 45 | E | ihl: 5 0x36.4-0x36.7 (0.4)
0x30| 00 | . | dscp: 0 0x37-0x37.5 (0.6)
0x30| 00 | . | ecn: 0 0x37.6-0x37.7 (0.2)
0x30| 00 8c | .. | total_length: 140 0x38-0x39.7 (2)
0x30| 00 00 | .. | identification: 0 0x3a-0x3b.7 (2)
0x30| 40 | @ | reserved: 0 0x3c-0x3c (0.1)
0x30| 40 | @ | dont_fragment: true 0x3c.1-0x3c.1 (0.1)
0x30| 40 | @ | more_fragments: false 0x3c.2-0x3c.2 (0.1)
0x30| 40 00 | @. | fragment_offset: 0 0x3c.3-0x3d.7 (1.5)
0x30| 3f | ? | ttl: 63 0x3e-0x3e.7 (1)
0x30| 11| .| protocol: "udp" (17) (User datagram protocol) 0x3f-0x3f.7 (1)
0x40|a7 52 |.R | header_checksum: 0xa752 (valid) 0x40-0x41.7 (2)
0x40| c0 a8 64 01 | ..d. | source_ip: "192.168.100.1" (0xc0a86401) 0x42-0x45.7 (4)
0x40| 0a 64 65 01 | .de. | destination_ip: "10.100.101.1" (0xa646501) 0x46-0x49.7 (4)
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef| payload{}: (udp_datagram) 0x4a-0xc1.7 (120)
0x40| 81 44 | .D | source_port: 33092 0x4a-0x4b.7 (2)
0x40| 08 07 | .. | destination_port: 2055 0x4c-0x4d.7 (2)
0x40| 00 78| .x| length: 120 0x4e-0x4f.7 (2)
0x50|1f 03 |.. | checksum: 0x1f03 0x50-0x51.7 (2)
0x50| 00 09 00 01 24 3c ba a0 59 e8 82 21 00 00| ....$<..Y..!..| payload: raw bits 0x52-0xc1.7 (112)
0x60|04 24 00 00 00 08 00 00 00 5c 01 a8 00 15 00 08|.$.......\......|
* |until 0xc1.7 (112) | |
0xc0| 74 be 47 c0| | t.G.| | unknown0: raw bits 0xc2-0xc5.7 (4)
| | | ipv4_reassembled[0:0]: 0xc6-NA (0)
| | | tcp_connections[0:0]: 0xc6-NA (0)

BIN
format/pcap/testdata/ns.pcap vendored Normal file
View File

Binary file not shown.

1
format/pcap/testdata/sll2_tcp.fqtest vendored
View File

@ -1,5 +1,6 @@
$ fq -d pcap dv sll2_tcp.pcap
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: sll2_tcp.pcap (pcap) 0x0-0x1e4.7 (485)
| | | header{}: 0x0-0x17.7 (24)
0x000|d4 c3 b2 a1 |.... | magic: "little_endian" (0xd4c3b2a1) (valid) 0x0-0x3.7 (4)
0x000| 02 00 | .. | version_major: 2 0x4-0x5.7 (2)
0x000| 04 00 | .. | version_minor: 4 0x6-0x7.7 (2)