wader/fq
1
1
Fork 0
mirror of https://github.com/wader/fq.git synced 2024-11-26 10:33:53 +03:00
Code Issues Projects Releases Wiki Activity

tcp,udp: Refactor and make port matching better

Browse Source 
rtmp: check port based on tcp direction
dns,rtmp: refactor to use tcp and udp port match helpers
flow: add has start/end to tcp to know if stream has missing bytes
This commit is contained in:
Mattias Wadman 2022-04-08 15:03:47 +02:00
parent a763840833
commit 5d25bbc2e1
12 changed files with 80 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
format/dns/dns.go
View File

@ -265,18 +265,8 @@ func dnsDecode(d *decode.D, isTCP bool) interface{} {
}
func dnsUDPDecode(d *decode.D, in interface{}) interface{} {
if tsi, ok := in.(format.TCPStreamIn); ok {
if tsi.DestinationPort == format.TCPPortDomain || tsi.SourcePort == format.TCPPortDomain {
return dnsDecode(d, true)
}
d.Fatalf("wrong port")
}
if upi, ok := in.(format.UDPPayloadIn); ok {
if upi.DestinationPort == format.UDPPortDomain || upi.SourcePort == format.UDPPortDomain ||
upi.DestinationPort == format.UDPPortMDNS || upi.SourcePort == format.UDPPortMDNS {
return dnsDecode(d, false)
}
d.Fatalf("wrong port")
upi.MustIsPort(d.Fatalf, format.UDPPortDomain, format.UDPPortMDNS)
}
return dnsDecode(d, false)
}

3
format/dns/dns_tcp.go
View File

@ -15,5 +15,8 @@ func init() {
}
func dnsTCPDecode(d *decode.D, in interface{}) interface{} {
if tsi, ok := in.(format.TCPStreamIn); ok {
tsi.MustIsPort(d.Fatalf, format.TCPPortDomain, format.TCPPortDomain)
}
return dnsDecode(d, true)
}

33
format/format.go
View File

@ -222,12 +222,45 @@ type UDPPayloadIn struct {
DestinationPort int
}
func (u UDPPayloadIn) IsPort(ports ...int) bool {
for _, p := range ports {
if u.DestinationPort == p || u.SourcePort == p {
return true
}
}
return false
}
func (u UDPPayloadIn) MustIsPort(fn func(format string, a ...interface{}), ports ...int) {
if !u.IsPort(ports...) {
fn("incorrect udp port %t src:%d dst:%d", u.DestinationPort, u.SourcePort)
}
}
type TCPStreamIn struct {
IsClient bool
HasStart bool
HasEnd bool
SourcePort int
DestinationPort int
}
func (t TCPStreamIn) IsPort(ports ...int) bool {
for _, p := range ports {
if (t.IsClient && t.DestinationPort == p) ||
(!t.IsClient && t.SourcePort == p) {
return true
}
}
return false
}
func (t TCPStreamIn) MustIsPort(fn func(format string, a ...interface{}), ports ...int) {
if !t.IsPort(ports...) {
fn("incorrect tcp port client %t src:%d dst:%d", t.IsClient, t.DestinationPort, t.SourcePort)
}
}
type X86_64In struct {
Base int64
SymLookup func(uint64) (string, uint64)

7
format/inet/flowsdecoder/flowsdecoder.go
View File

@ -21,6 +21,8 @@ type IPEndpoint struct {
type TCPConnection struct {
ClientEndpoint IPEndpoint
ServerEndpoint IPEndpoint
HasStart bool
HasEnd bool
ClientToServer *bytes.Buffer
ServerToClient *bytes.Buffer
@ -48,7 +50,7 @@ func (t *TCPConnection) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir rea
}
func (t *TCPConnection) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.AssemblerContext) {
dir, _, _, skip := sg.Info()
dir, start, end, skip := sg.Info()
length, _ := sg.Lengths()
if skip == -1 {
@ -59,6 +61,9 @@ func (t *TCPConnection) ReassembledSG(sg reassembly.ScatterGather, ac reassembly
return
}
t.HasStart = t.HasStart || start
t.HasEnd = t.HasEnd || end
data := sg.Fetch(length)
switch dir {

16
format/inet/testdata/flow_missing_synack.fqtest vendored
View File

@ -6,6 +6,8 @@ $ fq '.tcp_connections | d' flow_missing_synack.pcap
| | | source_port: 2061
| | | destination_ip: "192.168.1.3"
| | | destination_port: "https" (443) (http protocol over TLS/SSL)
| | | has_start: false
| | | has_end: false
0x0000|16 03 01 00 9e 01 00 00 9a 03 01 50 83 9c fa fe|...........P....| client_stream: raw bits
* |until 0x177.7 (end) (376) | |
0x0000|16 03 01 00 35 02 00 00 31 03 01 50 83 9c 9f e3|....5...1..P....| server_stream: raw bits
@ -15,6 +17,8 @@ $ fq '.tcp_connections | d' flow_missing_synack.pcap
| | | source_port: 2068
| | | destination_ip: "192.168.1.3"
| | | destination_port: "https" (443) (http protocol over TLS/SSL)
| | | has_start: false
| | | has_end: false
0x0000|16 03 01 00 9e 01 00 00 9a 03 01 50 83 9d 00 a1|...........P....| client_stream: raw bits
* |until 0x177.7 (end) (376) | |
0x0000|16 03 01 00 35 02 00 00 31 03 01 50 83 9c a5 e5|....5...1..P....| server_stream: raw bits
@ -24,6 +28,8 @@ $ fq '.tcp_connections | d' flow_missing_synack.pcap
| | | source_port: 2070
| | | destination_ip: "192.168.1.3"
| | | destination_port: "https" (443) (http protocol over TLS/SSL)
| | | has_start: false
| | | has_end: false
0x0000|16 03 01 00 9e 01 00 00 9a 03 01 50 83 9d 03 f3|...........P....| client_stream: raw bits
* |until 0x2ad.7 (end) (686) | |
0x0000|16 03 01 00 35 02 00 00 31 03 01 50 83 9c a8 b2|....5...1..P....| server_stream: raw bits
@ -33,6 +39,8 @@ $ fq '.tcp_connections | d' flow_missing_synack.pcap
| | | source_port: 2071
| | | destination_ip: "192.168.1.3"
| | | destination_port: "https" (443) (http protocol over TLS/SSL)
| | | has_start: false
| | | has_end: false
0x0000|16 03 01 01 6e 01 00 01 6a 03 01 50 83 9d 03 d8|....n...j..P....| client_stream: raw bits
* |until 0x2df.7 (end) (736) | |
0x0000|16 03 01 00 51 02 00 00 4d 03 01 50 83 9c a8 fc|....Q...M..P....| server_stream: raw bits
@ -42,6 +50,8 @@ $ fq '.tcp_connections | d' flow_missing_synack.pcap
| | | source_port: 2072
| | | destination_ip: "192.168.1.3"
| | | destination_port: "https" (443) (http protocol over TLS/SSL)
| | | has_start: false
| | | has_end: false
0x0000|16 03 01 01 6e 01 00 01 6a 03 01 50 83 9d 03 94|....n...j..P....| client_stream: raw bits
* |until 0x2fd.7 (end) (766) | |
0x0000|16 03 01 00 51 02 00 00 4d 03 01 50 83 9c a8 d8|....Q...M..P....| server_stream: raw bits
@ -51,6 +61,8 @@ $ fq '.tcp_connections | d' flow_missing_synack.pcap
| | | source_port: 2073
| | | destination_ip: "192.168.1.3"
| | | destination_port: "https" (443) (http protocol over TLS/SSL)
| | | has_start: false
| | | has_end: true
0x0000|16 03 01 01 6e 01 00 01 6a 03 01 50 83 9d 0d 96|....n...j..P....| client_stream: raw bits
* |until 0x2fd.7 (end) (766) | |
0x0000|16 03 01 00 51 02 00 00 4d 03 01 50 83 9c b2 45|....Q...M..P...E| server_stream: raw bits
@ -60,6 +72,8 @@ $ fq '.tcp_connections | d' flow_missing_synack.pcap
| | | source_port: 2078
| | | destination_ip: "192.168.1.3"
| | | destination_port: "https" (443) (http protocol over TLS/SSL)
| | | has_start: false
| | | has_end: false
0x0000|16 03 01 01 6e 01 00 01 6a 03 01 50 83 9d d7 3a|....n...j..P...:| client_stream: raw bits
* |until 0x38c.7 (end) (909) | |
0x0000|16 03 01 00 51 02 00 00 4d 03 01 50 83 9d 7c ac|....Q...M..P..|.| server_stream: raw bits
@ -69,6 +83,8 @@ $ fq '.tcp_connections | d' flow_missing_synack.pcap
| | | source_port: 2085
| | | destination_ip: "192.168.1.3"
| | | destination_port: "https" (443) (http protocol over TLS/SSL)
| | | has_start: false
| | | has_end: false
0x0000|16 03 01 01 6e 01 00 01 6a 03 01 50 83 9e 02 2b|....n...j..P...+| client_stream: raw bits
* |until 0x4a0.7 (end) (1185) | |
0x0000|16 03 01 00 51 02 00 00 4d 03 01 50 83 9d a7 8b|....Q...M..P....| server_stream: raw bits

10
format/pcap/shared.go
View File

@ -55,6 +55,8 @@ func fieldFlows(d *decode.D, fd *flowsdecoder.Decoder, tcpStreamFormat decode.Gr
d.FieldValueU("source_port", uint64(s.ClientEndpoint.Port), format.TCPPortMap)
d.FieldValueStr("destination_ip", s.ServerEndpoint.IP.String())
d.FieldValueU("destination_port", uint64(s.ServerEndpoint.Port), format.TCPPortMap)
d.FieldValueBool("has_start", s.HasStart)
d.FieldValueBool("has_end", s.HasEnd)
csBR := bitio.NewBitReader(s.ClientToServer.Bytes(), -1)
if dv, _, _ := d.TryFieldFormatBitBuf(
"client_stream",
@ -62,6 +64,8 @@ func fieldFlows(d *decode.D, fd *flowsdecoder.Decoder, tcpStreamFormat decode.Gr
tcpStreamFormat,
format.TCPStreamIn{
IsClient: true,
HasStart: s.HasStart,
HasEnd: s.HasEnd,
SourcePort: s.ClientEndpoint.Port,
DestinationPort: s.ServerEndpoint.Port,
},
@ -76,8 +80,10 @@ func fieldFlows(d *decode.D, fd *flowsdecoder.Decoder, tcpStreamFormat decode.Gr
tcpStreamFormat,
format.TCPStreamIn{
IsClient: false,
SourcePort: s.ClientEndpoint.Port,
DestinationPort: s.ServerEndpoint.Port,
HasStart: s.HasStart,
HasEnd: s.HasEnd,
SourcePort: s.ServerEndpoint.Port,
DestinationPort: s.ClientEndpoint.Port,
},
); dv == nil {
d.FieldRootBitBuf("server_stream", scBR)

2
format/pcap/testdata/http_gzip.fqtest vendored
View File

@ -613,6 +613,8 @@ $ fq -d pcap dv /http_gzip.cap
| | | source_port: 34059 0x6ab-NA (0)
| | | destination_ip: "192.168.69.1" 0x6ab-NA (0)
| | | destination_port: "http" (80) (World Wide Web HTTP) 0x6ab-NA (0)
| | | has_start: true 0x6ab-NA (0)
| | | has_end: true 0x6ab-NA (0)
0x000|47 45 54 20 2f 74 65 73 74 2f 65 74 68 65 72 65|GET /test/ethere| client_stream: raw bits 0x0-0x1bc.7 (445)
* |until 0x1bc.7 (end) (445) | |
0x000|48 54 54 50 2f 31 2e 31 20 32 30 30 20 4f 4b 0d|HTTP/1.1 200 OK.| server_stream: raw bits 0x0-0x191.7 (402)

2
format/pcap/testdata/ipv6_http.fqtest vendored
View File

@ -3487,6 +3487,8 @@ $ fq -d pcap dv ipv6_http.pcap
| | | source_port: 59201 0x23c7-NA (0)
| | | destination_ip: "2001:6f8:900:7c0::2" 0x23c7-NA (0)
| | | destination_port: "http" (80) (World Wide Web HTTP) 0x23c7-NA (0)
| | | has_start: true 0x23c7-NA (0)
| | | has_end: true 0x23c7-NA (0)
0x000|47 45 54 20 2f 20 48 54 54 50 2f 31 2e 30 0d 0a|GET / HTTP/1.0..| client_stream: raw bits 0x0-0xef.7 (240)
* |until 0xef.7 (end) (240) | |
0x000|48 54 54 50 2f 31 2e 31 20 32 30 30 20 4f 4b 0d|HTTP/1.1 200 OK.| server_stream: raw bits 0x0-0x8d2.7 (2259)

4
format/pcap/testdata/many_interfaces.fqtest vendored
View File

@ -5491,6 +5491,8 @@ $ fq -d pcapng dv /many_interfaces.pcapng
| | | source_port: 50981 0x51b8-NA (0)
| | | destination_ip: "74.125.228.227" 0x51b8-NA (0)
| | | destination_port: "https" (443) (http protocol over TLS/SSL) 0x51b8-NA (0)
| | | has_start: true 0x51b8-NA (0)
| | | has_end: false 0x51b8-NA (0)
0x000|16 03 01 02 00 01 00 01 fc 03 03 f0 91 bc 87 3e|...............>| client_stream: raw bits 0x0-0x7b0.7 (1969)
* |until 0x7b0.7 (end) (1969) | |
0x000|16 03 03 00 5a 02 00 00 56 03 03 55 d0 e5 ff ab|....Z...V..U....| server_stream: raw bits 0x0-0x35b.7 (860)
@ -5500,6 +5502,8 @@ $ fq -d pcapng dv /many_interfaces.pcapng
| | | source_port: 50982 0x51b8-NA (0)
| | | destination_ip: "74.125.228.227" 0x51b8-NA (0)
| | | destination_port: "https" (443) (http protocol over TLS/SSL) 0x51b8-NA (0)
| | | has_start: true 0x51b8-NA (0)
| | | has_end: false 0x51b8-NA (0)
0x000|16 03 01 00 d3 01 00 00 cf 03 03 c0 a6 33 83 e1|.............3..| client_stream: raw bits 0x0-0xd7.7 (216)
* |until 0xd7.7 (end) (216) | |
| | | server_stream: raw bits 0x0-NA (0)

2
format/pcap/testdata/sll2_tcp.fqtest vendored
View File

@ -339,5 +339,7 @@ $ fq -d pcap dv /sll2_tcp.pcap
| | | source_port: 47174 0x1e5-NA (0)
| | | destination_ip: "127.0.0.1" 0x1e5-NA (0)
| | | destination_port: 1234 0x1e5-NA (0)
| | | has_start: true 0x1e5-NA (0)
| | | has_end: false 0x1e5-NA (0)
0x00|74 65 73 74 0a| |test.| | client_stream: raw bits 0x0-0x4.7 (5)
| | | server_stream: raw bits 0x0-NA (0)

5
format/rtmp/rtmp.go
View File

@ -303,11 +303,8 @@ func rtmpDecodeMessageType(d *decode.D, typ int, chunkSize *int) {
func rtmpDecode(d *decode.D, in interface{}) interface{} {
var isClient bool
if tsi, ok := in.(format.TCPStreamIn); ok {
if tsi.DestinationPort != format.TCPPortRTMP {
d.Fatalf("wrong port")
}
tsi.MustIsPort(d.Fatalf, format.TCPPortRTMP)
isClient = tsi.IsClient
}

2
format/rtmp/testdata/rtmp_sample.cap.fqtest vendored
View File

@ -5,6 +5,8 @@ $ fq '.tcp_connections | dv' rtmp_sample.cap
| | | source_port: 1177 0x2268-NA (0)
| | | destination_ip: "192.168.43.128" 0x2268-NA (0)
| | | destination_port: "rtmp" (1935) (Real-Time Messaging Protocol) 0x2268-NA (0)
| | | has_start: true 0x2268-NA (0)
| | | has_end: false 0x2268-NA (0)
| | | client_stream{}: (rtmp) 0x0-0xd7b.7 (3452)
| | | handshake{}: 0x0-0xc00.7 (3073)
| | | c0{}: 0x0-0x0.7 (1)