mirror of
https://github.com/wader/fq.git
synced 2024-12-23 13:22:58 +03:00
pcap: Add ns support and add header field
This commit is contained in:
Mattias Wadman
parent
da6b9413fd
commit
74c7dc4eaf
@ -15,14 +15,22 @@ var pcapLinkFrameFormat decode.Group
|
||||
var pcapTCPStreamFormat decode.Group
|
||||
var pcapIPv4PacketFormat decode.Group
|
||||
|
||||
// writing application writes 0xa1b2c3d4 in native endian
|
||||
const (
|
||||
// timestamp is seconds + microseconds
|
||||
bigEndian = 0xa1b2c3d4
|
||||
littleEndian = 0xd4c3b2a1
|
||||
|
||||
// timestamp is seconds + nanoseconds
|
||||
bigEndianNS = 0xa1b23c4d
|
||||
littleEndianNS = 0x4d3cb2a1
|
||||
)
|
||||
|
||||
var endianMap = scalar.UToSymStr{
|
||||
bigEndian: "big_endian",
|
||||
littleEndian: "little_endian",
|
||||
bigEndian: "big_endian",
|
||||
littleEndian: "little_endian",
|
||||
bigEndianNS: "big_endian_ns",
|
||||
littleEndianNS: "little_endian_ns",
|
||||
}
|
||||
|
||||
func init() {
|
||||
@ -40,29 +48,49 @@ func init() {
|
||||
}
|
||||
|
||||
func decodePcap(d *decode.D, _ any) any {
|
||||
endian := d.FieldU32("magic", d.AssertU(bigEndian, littleEndian), endianMap, scalar.ActualHex)
|
||||
switch endian {
|
||||
case bigEndian:
|
||||
d.Endian = decode.BigEndian
|
||||
case littleEndian:
|
||||
d.Endian = decode.LittleEndian
|
||||
default:
|
||||
d.Fatalf("unknown endian %d", endian)
|
||||
}
|
||||
d.FieldU16("version_major")
|
||||
d.FieldU16("version_minor")
|
||||
d.FieldS32("thiszone")
|
||||
d.FieldU32("sigfigs")
|
||||
d.FieldU32("snaplen")
|
||||
linkType := int(d.FieldU32("network", format.LinkTypeMap))
|
||||
var endian decode.Endian
|
||||
linkType := 0
|
||||
timestampUNSStr := "ts_usec"
|
||||
|
||||
d.FieldStruct("header", func(d *decode.D) {
|
||||
magic := d.FieldU32("magic", d.AssertU(
|
||||
bigEndian,
|
||||
littleEndian,
|
||||
bigEndianNS,
|
||||
littleEndianNS,
|
||||
), endianMap, scalar.ActualHex)
|
||||
|
||||
switch magic {
|
||||
case bigEndian:
|
||||
endian = decode.BigEndian
|
||||
case littleEndian:
|
||||
endian = decode.LittleEndian
|
||||
case bigEndianNS:
|
||||
endian = decode.BigEndian
|
||||
timestampUNSStr = "ts_nsec"
|
||||
case littleEndianNS:
|
||||
endian = decode.LittleEndian
|
||||
timestampUNSStr = "ts_nsec"
|
||||
}
|
||||
|
||||
d.Endian = endian
|
||||
|
||||
d.FieldU16("version_major")
|
||||
d.FieldU16("version_minor")
|
||||
d.FieldS32("thiszone")
|
||||
d.FieldU32("sigfigs")
|
||||
d.FieldU32("snaplen")
|
||||
linkType = int(d.FieldU32("network", format.LinkTypeMap))
|
||||
})
|
||||
|
||||
d.Endian = endian
|
||||
fd := flowsdecoder.New()
|
||||
|
||||
d.FieldArray("packets", func(d *decode.D) {
|
||||
for !d.End() {
|
||||
d.FieldStruct("packet", func(d *decode.D) {
|
||||
d.FieldU32("ts_sec")
|
||||
d.FieldU32("ts_usec")
|
||||
d.FieldU32(timestampUNSStr)
|
||||
inclLen := d.FieldU32("incl_len")
|
||||
origLen := d.FieldU32("orig_len")
|
||||
|
||||
|
||||
15
format/pcap/testdata/http_gzip.fqtest
vendored
15
format/pcap/testdata/http_gzip.fqtest
vendored
@ -1,13 +1,14 @@
|
||||
# from https://wiki.wireshark.org/SampleCaptures
|
||||
$ fq -d pcap dv http_gzip.cap
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: http_gzip.cap (pcap) 0x0-0x6aa.7 (1707)
|
||||
0x00000|d4 c3 b2 a1 |.... | magic: "little_endian" (0xd4c3b2a1) (valid) 0x0-0x3.7 (4)
|
||||
0x00000| 02 00 | .. | version_major: 2 0x4-0x5.7 (2)
|
||||
0x00000| 04 00 | .. | version_minor: 4 0x6-0x7.7 (2)
|
||||
0x00000| 00 00 00 00 | .... | thiszone: 0 0x8-0xb.7 (4)
|
||||
0x00000| 00 00 00 00| ....| sigfigs: 0 0xc-0xf.7 (4)
|
||||
0x00010|ff ff 00 00 |.... | snaplen: 65535 0x10-0x13.7 (4)
|
||||
0x00010| 01 00 00 00 | .... | network: "ethernet" (1) (IEEE 802.3 Ethernet) 0x14-0x17.7 (4)
|
||||
| | | header{}: 0x0-0x17.7 (24)
|
||||
0x00000|d4 c3 b2 a1 |.... | magic: "little_endian" (0xd4c3b2a1) (valid) 0x0-0x3.7 (4)
|
||||
0x00000| 02 00 | .. | version_major: 2 0x4-0x5.7 (2)
|
||||
0x00000| 04 00 | .. | version_minor: 4 0x6-0x7.7 (2)
|
||||
0x00000| 00 00 00 00 | .... | thiszone: 0 0x8-0xb.7 (4)
|
||||
0x00000| 00 00 00 00| ....| sigfigs: 0 0xc-0xf.7 (4)
|
||||
0x00010|ff ff 00 00 |.... | snaplen: 65535 0x10-0x13.7 (4)
|
||||
0x00010| 01 00 00 00 | .... | network: "ethernet" (1) (IEEE 802.3 Ethernet) 0x14-0x17.7 (4)
|
||||
| | | packets[0:10]: 0x18-0x6aa.7 (1683)
|
||||
| | | [0]{}: packet 0x18-0x71.7 (90)
|
||||
0x00010| 3c d3 81 41 | <..A | ts_sec: 1099027260 0x18-0x1b.7 (4)
|
||||
|
||||
15
format/pcap/testdata/ipv4frags.fqtest
vendored
15
format/pcap/testdata/ipv4frags.fqtest
vendored
@ -1,13 +1,14 @@
|
||||
# from https://wiki.wireshark.org/SampleCaptures
|
||||
$ fq -d pcap dv ipv4frags.pcap
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: ipv4frags.pcap (pcap) 0x0-0xbad.7 (2990)
|
||||
0x00000|d4 c3 b2 a1 |.... | magic: "little_endian" (0xd4c3b2a1) (valid) 0x0-0x3.7 (4)
|
||||
0x00000| 02 00 | .. | version_major: 2 0x4-0x5.7 (2)
|
||||
0x00000| 04 00 | .. | version_minor: 4 0x6-0x7.7 (2)
|
||||
0x00000| 00 00 00 00 | .... | thiszone: 0 0x8-0xb.7 (4)
|
||||
0x00000| 00 00 00 00| ....| sigfigs: 0 0xc-0xf.7 (4)
|
||||
0x00010|d0 07 00 00 |.... | snaplen: 2000 0x10-0x13.7 (4)
|
||||
0x00010| 01 00 00 00 | .... | network: "ethernet" (1) (IEEE 802.3 Ethernet) 0x14-0x17.7 (4)
|
||||
| | | header{}: 0x0-0x17.7 (24)
|
||||
0x00000|d4 c3 b2 a1 |.... | magic: "little_endian" (0xd4c3b2a1) (valid) 0x0-0x3.7 (4)
|
||||
0x00000| 02 00 | .. | version_major: 2 0x4-0x5.7 (2)
|
||||
0x00000| 04 00 | .. | version_minor: 4 0x6-0x7.7 (2)
|
||||
0x00000| 00 00 00 00 | .... | thiszone: 0 0x8-0xb.7 (4)
|
||||
0x00000| 00 00 00 00| ....| sigfigs: 0 0xc-0xf.7 (4)
|
||||
0x00010|d0 07 00 00 |.... | snaplen: 2000 0x10-0x13.7 (4)
|
||||
0x00010| 01 00 00 00 | .... | network: "ethernet" (1) (IEEE 802.3 Ethernet) 0x14-0x17.7 (4)
|
||||
| | | packets[0:3]: 0x18-0xbad.7 (2966)
|
||||
| | | [0]{}: packet 0x18-0x419.7 (1026)
|
||||
0x00010| 14 2b d2 59 | .+.Y | ts_sec: 1506945812 0x18-0x1b.7 (4)
|
||||
|
||||
15
format/pcap/testdata/ipv6_http.fqtest
vendored
15
format/pcap/testdata/ipv6_http.fqtest
vendored
@ -1,13 +1,14 @@
|
||||
# from https://wiki.wireshark.org/SampleCaptures
|
||||
$ fq -d pcap dv ipv6_http.pcap
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: ipv6_http.pcap (pcap) 0x0-0x23c6.7 (9159)
|
||||
0x00000|d4 c3 b2 a1 |.... | magic: "little_endian" (0xd4c3b2a1) (valid) 0x0-0x3.7 (4)
|
||||
0x00000| 02 00 | .. | version_major: 2 0x4-0x5.7 (2)
|
||||
0x00000| 04 00 | .. | version_minor: 4 0x6-0x7.7 (2)
|
||||
0x00000| 00 00 00 00 | .... | thiszone: 0 0x8-0xb.7 (4)
|
||||
0x00000| 00 00 00 00| ....| sigfigs: 0 0xc-0xf.7 (4)
|
||||
0x00010|ff ff 00 00 |.... | snaplen: 65535 0x10-0x13.7 (4)
|
||||
0x00010| 01 00 00 00 | .... | network: "ethernet" (1) (IEEE 802.3 Ethernet) 0x14-0x17.7 (4)
|
||||
| | | header{}: 0x0-0x17.7 (24)
|
||||
0x00000|d4 c3 b2 a1 |.... | magic: "little_endian" (0xd4c3b2a1) (valid) 0x0-0x3.7 (4)
|
||||
0x00000| 02 00 | .. | version_major: 2 0x4-0x5.7 (2)
|
||||
0x00000| 04 00 | .. | version_minor: 4 0x6-0x7.7 (2)
|
||||
0x00000| 00 00 00 00 | .... | thiszone: 0 0x8-0xb.7 (4)
|
||||
0x00000| 00 00 00 00| ....| sigfigs: 0 0xc-0xf.7 (4)
|
||||
0x00010|ff ff 00 00 |.... | snaplen: 65535 0x10-0x13.7 (4)
|
||||
0x00010| 01 00 00 00 | .... | network: "ethernet" (1) (IEEE 802.3 Ethernet) 0x14-0x17.7 (4)
|
||||
| | | packets[0:55]: 0x18-0x23c6.7 (9135)
|
||||
| | | [0]{}: packet 0x18-0x7d.7 (102)
|
||||
0x00010| d7 20 b6 46 | . .F | ts_sec: 1186341079 0x18-0x1b.7 (4)
|
||||
|
||||
50
format/pcap/testdata/ns.fqtest
vendored
Normal file
50
format/pcap/testdata/ns.fqtest
vendored
Normal file
@ -0,0 +1,50 @@
|
||||
# netflowv9.pcap from https://github.com/secdev/scapy/tree/master/test/pcaps
|
||||
# fq '(.header,.packets[0]) | tobytes' netflowv9.pcap > ns.pcap
|
||||
$ fq dv ns.pcap
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: ns.pcap (pcap) 0x0-0xc5.7 (198)
|
||||
| | | header{}: 0x0-0x17.7 (24)
|
||||
0x00|4d 3c b2 a1 |M<.. | magic: "little_endian_ns" (0x4d3cb2a1) (valid) 0x0-0x3.7 (4)
|
||||
0x00| 02 00 | .. | version_major: 2 0x4-0x5.7 (2)
|
||||
0x00| 04 00 | .. | version_minor: 4 0x6-0x7.7 (2)
|
||||
0x00| 00 00 00 00 | .... | thiszone: 0 0x8-0xb.7 (4)
|
||||
0x00| 00 00 00 00| ....| sigfigs: 0 0xc-0xf.7 (4)
|
||||
0x10|ff ff 00 00 |.... | snaplen: 65535 0x10-0x13.7 (4)
|
||||
0x10| 01 00 00 00 | .... | network: "ethernet" (1) (IEEE 802.3 Ethernet) 0x14-0x17.7 (4)
|
||||
| | | packets[0:1]: 0x18-0xc5.7 (174)
|
||||
| | | [0]{}: packet 0x18-0xc5.7 (174)
|
||||
0x10| 0d 82 e8 59 | ...Y | ts_sec: 1508409869 0x18-0x1b.7 (4)
|
||||
0x10| 53 c6 50 22| S.P"| ts_nsec: 575718995 0x1c-0x1f.7 (4)
|
||||
0x20|9e 00 00 00 |.... | incl_len: 158 0x20-0x23.7 (4)
|
||||
0x20| 9e 00 00 00 | .... | orig_len: 158 0x24-0x27.7 (4)
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef| packet{}: (ether8023_frame) 0x28-0xc5.7 (158)
|
||||
0x20| 00 10 94 00 00 01 | ...... | destination: "00:10:94:00:00:01" (0x1094000001) 0x28-0x2d.7 (6)
|
||||
0x20| 00 1d| ..| source: "00:1d:b5:cb:28:ce" (0x1db5cb28ce) 0x2e-0x33.7 (6)
|
||||
0x30|b5 cb 28 ce |..(. |
|
||||
0x30| 08 00 | .. | ether_type: "ipv4" (0x800) (Internet Protocol version 4) 0x34-0x35.7 (2)
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef| payload{}: (ipv4_packet) 0x36-0xc5.7 (144)
|
||||
0x30| 45 | E | version: 4 0x36-0x36.3 (0.4)
|
||||
0x30| 45 | E | ihl: 5 0x36.4-0x36.7 (0.4)
|
||||
0x30| 00 | . | dscp: 0 0x37-0x37.5 (0.6)
|
||||
0x30| 00 | . | ecn: 0 0x37.6-0x37.7 (0.2)
|
||||
0x30| 00 8c | .. | total_length: 140 0x38-0x39.7 (2)
|
||||
0x30| 00 00 | .. | identification: 0 0x3a-0x3b.7 (2)
|
||||
0x30| 40 | @ | reserved: 0 0x3c-0x3c (0.1)
|
||||
0x30| 40 | @ | dont_fragment: true 0x3c.1-0x3c.1 (0.1)
|
||||
0x30| 40 | @ | more_fragments: false 0x3c.2-0x3c.2 (0.1)
|
||||
0x30| 40 00 | @. | fragment_offset: 0 0x3c.3-0x3d.7 (1.5)
|
||||
0x30| 3f | ? | ttl: 63 0x3e-0x3e.7 (1)
|
||||
0x30| 11| .| protocol: "udp" (17) (User datagram protocol) 0x3f-0x3f.7 (1)
|
||||
0x40|a7 52 |.R | header_checksum: 0xa752 (valid) 0x40-0x41.7 (2)
|
||||
0x40| c0 a8 64 01 | ..d. | source_ip: "192.168.100.1" (0xc0a86401) 0x42-0x45.7 (4)
|
||||
0x40| 0a 64 65 01 | .de. | destination_ip: "10.100.101.1" (0xa646501) 0x46-0x49.7 (4)
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef| payload{}: (udp_datagram) 0x4a-0xc1.7 (120)
|
||||
0x40| 81 44 | .D | source_port: 33092 0x4a-0x4b.7 (2)
|
||||
0x40| 08 07 | .. | destination_port: 2055 0x4c-0x4d.7 (2)
|
||||
0x40| 00 78| .x| length: 120 0x4e-0x4f.7 (2)
|
||||
0x50|1f 03 |.. | checksum: 0x1f03 0x50-0x51.7 (2)
|
||||
0x50| 00 09 00 01 24 3c ba a0 59 e8 82 21 00 00| ....$<..Y..!..| payload: raw bits 0x52-0xc1.7 (112)
|
||||
0x60|04 24 00 00 00 08 00 00 00 5c 01 a8 00 15 00 08|.$.......\......|
|
||||
* |until 0xc1.7 (112) | |
|
||||
0xc0| 74 be 47 c0| | t.G.| | unknown0: raw bits 0xc2-0xc5.7 (4)
|
||||
| | | ipv4_reassembled[0:0]: 0xc6-NA (0)
|
||||
| | | tcp_connections[0:0]: 0xc6-NA (0)
|
||||
BIN
format/pcap/testdata/ns.pcap
vendored
Normal file
BIN
format/pcap/testdata/ns.pcap
vendored
Normal file
Binary file not shown.
15
format/pcap/testdata/sll2_tcp.fqtest
vendored
15
format/pcap/testdata/sll2_tcp.fqtest
vendored
@ -1,12 +1,13 @@
|
||||
$ fq -d pcap dv sll2_tcp.pcap
|
||||
|00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: sll2_tcp.pcap (pcap) 0x0-0x1e4.7 (485)
|
||||
0x000|d4 c3 b2 a1 |.... | magic: "little_endian" (0xd4c3b2a1) (valid) 0x0-0x3.7 (4)
|
||||
0x000| 02 00 | .. | version_major: 2 0x4-0x5.7 (2)
|
||||
0x000| 04 00 | .. | version_minor: 4 0x6-0x7.7 (2)
|
||||
0x000| 00 00 00 00 | .... | thiszone: 0 0x8-0xb.7 (4)
|
||||
0x000| 00 00 00 00| ....| sigfigs: 0 0xc-0xf.7 (4)
|
||||
0x010|00 00 04 00 |.... | snaplen: 262144 0x10-0x13.7 (4)
|
||||
0x010| 14 01 00 00 | .... | network: "linux_sll2" (276) (Linux "cooked" capture encapsulation v2) 0x14-0x17.7 (4)
|
||||
| | | header{}: 0x0-0x17.7 (24)
|
||||
0x000|d4 c3 b2 a1 |.... | magic: "little_endian" (0xd4c3b2a1) (valid) 0x0-0x3.7 (4)
|
||||
0x000| 02 00 | .. | version_major: 2 0x4-0x5.7 (2)
|
||||
0x000| 04 00 | .. | version_minor: 4 0x6-0x7.7 (2)
|
||||
0x000| 00 00 00 00 | .... | thiszone: 0 0x8-0xb.7 (4)
|
||||
0x000| 00 00 00 00| ....| sigfigs: 0 0xc-0xf.7 (4)
|
||||
0x010|00 00 04 00 |.... | snaplen: 262144 0x10-0x13.7 (4)
|
||||
0x010| 14 01 00 00 | .... | network: "linux_sll2" (276) (Linux "cooked" capture encapsulation v2) 0x14-0x17.7 (4)
|
||||
| | | packets[0:5]: 0x18-0x1e4.7 (461)
|
||||
| | | [0]{}: packet 0x18-0x77.7 (96)
|
||||
0x010| 44 08 a5 61 | D..a | ts_sec: 1638205508 0x18-0x1b.7 (4)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user