Merge 3f2aa75ec8 into 2bc6c768c7

Same as other formats that use timestamps and makes it easier to use them in query,
number instead of a formatted string. Can still use todescription/0 to get
formatted timestamp.

README.md

@@ -138,7 +138,7 @@ prores_frame,
 [protobuf](doc/formats.md#protobuf),
 protobuf_widevine,
 pssh_playready,
-pyrdp,
+[pyrdp](doc/formats.md#pyrdp),
 [rtmp](doc/formats.md#rtmp),
 sll2_packet,
 sll_packet,

doc/formats.md

@@ -110,6 +110,7 @@
 |[`protobuf`](#protobuf)                                         |Protobuf                                                                                                     |<sub></sub>|
 |`protobuf_widevine`                                             |Widevine&nbsp;protobuf                                                                                       |<sub>`protobuf`</sub>|
 |`pssh_playready`                                                |PlayReady&nbsp;PSSH                                                                                          |<sub></sub>|
+|[`pyrdp`](#pyrdp)                                               |PyRDP&nbsp;Replay&nbsp;Files                                                                                 |<sub></sub>|
 |[`rtmp`](#rtmp)                                                 |Real-Time&nbsp;Messaging&nbsp;Protocol                                                                       |<sub>`amf0` `mpeg_asc`</sub>|
 |`sll2_packet`                                                   |Linux&nbsp;cooked&nbsp;capture&nbsp;encapsulation&nbsp;v2                                                    |<sub>`inet_packet`</sub>|
 |`sll_packet`                                                    |Linux&nbsp;cooked&nbsp;capture&nbsp;encapsulation                                                            |<sub>`inet_packet`</sub>|
@@ -1195,6 +1196,16 @@ $ fq -d protobuf '.fields[6].wire_value | protobuf | d' file
 ### References
 - https://developers.google.com/protocol-buffers/docs/encoding
 
+## pyrdp
+PyRDP Replay Files.
+
+### Authors
+- Olivier Bilodeau <olivier.bilodeau@flare.io>, Maintainer
+- Lisandro Ubiedo, Author
+
+### References
+- https://github.com/GoSecure/pyrdp
+
 ## rtmp
 Real-Time Messaging Protocol.
 

format/markdown/markdown.jq

@@ -57,7 +57,11 @@ def _markdown_children_to_text($width):
         | join("")
         ) as $text
       | if $text == .destination then $text
-        else "\($text) (\(.destination))"
+        else
+          if .destination | startswith("mailto:") then
+            "<\(.destination[7:])>"
+          else "\($text) (\(.destination))"
+          end
         end
       )
     elif .type == "code_block" then .literal | rtrimstr("\n") | split("\n") | "  " + join("\n  ")

format/pyrdp/pdu/client_data.go

@@ -16,61 +16,61 @@ const (
 	RDP10_2  = 0x80007
 	RDP10_3  = 0x80008
 	RDP10_4  = 0x80009
-	RDP10_5  = 0x8000A
-	RDP10_6  = 0x8000B
-	RDP10_7  = 0x8000C
+	RDP10_5  = 0x8000a
+	RDP10_6  = 0x8000b
+	RDP10_7  = 0x8000c
 	RDP10_8  = 0x8000d
 	RDP10_9  = 0x8000e
 	RDP10_10 = 0x8000f
 )
 
 var RDPVersionMap = scalar.UintMapSymStr{
-	RDP4:     "rdp4",
-	RDP5:     "rdp5",
-	RDP10:    "rdp10",
-	RDP10_1:  "rdp10_1",
-	RDP10_2:  "rdp10_2",
-	RDP10_3:  "rdp10_3",
-	RDP10_4:  "rdp10_4",
-	RDP10_5:  "rdp10_5",
-	RDP10_6:  "rdp10_6",
-	RDP10_7:  "rdp10_7",
-	RDP10_8:  "rdp10_8",
-	RDP10_9:  "rdp10_9",
-	RDP10_10: "rdp10_10",
+	RDP4:     "4",
+	RDP5:     "5",
+	RDP10:    "10",
+	RDP10_1:  "10_1",
+	RDP10_2:  "10_2",
+	RDP10_3:  "10_3",
+	RDP10_4:  "10_4",
+	RDP10_5:  "10_5",
+	RDP10_6:  "10_6",
+	RDP10_7:  "10_7",
+	RDP10_8:  "10_8",
+	RDP10_9:  "10_9",
+	RDP10_10: "10_10",
 }
 
 const (
-	CLIENT_CORE     = 0xC001
-	CLIENT_SECURITY = 0xC002
-	CLIENT_NETWORK  = 0xC003
-	CLIENT_CLUSTER  = 0xC004
+	CLIENT_CORE     = 0xc001
+	CLIENT_SECURITY = 0xc002
+	CLIENT_NETWORK  = 0xc003
+	CLIENT_CLUSTER  = 0xc004
 )
 
 var clientDataMap = scalar.UintMapSymStr{
-	CLIENT_CORE:     "client_core",
-	CLIENT_SECURITY: "client_security",
-	CLIENT_NETWORK:  "client_network",
-	CLIENT_CLUSTER:  "client_cluster",
+	CLIENT_CORE:     "core",
+	CLIENT_SECURITY: "security",
+	CLIENT_NETWORK:  "network",
+	CLIENT_CLUSTER:  "cluster",
 }
 
 func ParseClientData(d *decode.D, length int64) {
 	d.FieldStruct("client_data", func(d *decode.D) {
 		header := d.FieldU16("header", clientDataMap)
-		data_len := int64(d.FieldU16("length") - 4)
+		dataLen := int64(d.FieldU16("length") - 4)
 
 		switch header {
 		case CLIENT_CORE:
-			ParseClientDataCore(d, data_len)
+			ParseClientDataCore(d, dataLen)
 		case CLIENT_SECURITY:
-			ParseClientDataSecurity(d, data_len)
+			ParseClientDataSecurity(d, dataLen)
 		case CLIENT_NETWORK:
-			ParseClientDataNetwork(d, data_len)
+			ParseClientDataNetwork(d, dataLen)
 		case CLIENT_CLUSTER:
-			ParseClientDataCluster(d, data_len)
+			ParseClientDataCluster(d, dataLen)
 		default:
 			// Assert() once all functions are implemented and tested.
-			d.FieldRawLen("data", data_len*8)
+			d.FieldRawLen("data", dataLen*8)
 			return
 		}
 	})

format/pyrdp/pdu/client_info.go

@@ -12,46 +12,46 @@ func ParseClientInfo(d *decode.D, length int64) {
 	d.FieldStruct("client_info", func(d *decode.D) {
 		pos := d.Pos()
 		var (
-			is_unicode bool
-			has_null   bool
-			null_n     uint64 = 0
-			unicode_n  uint64 = 0
+			isUnicode bool
+			hasNull   bool
+			nullN     uint64 = 0
+			unicodeN  uint64 = 0
 		)
-		code_page := d.FieldU32("code_page")
+		codePage := d.FieldU32("code_page")
 		flags := d.U32()
 		d.SeekRel(-4 * 8)
 		d.FieldStruct("flags", decodeFlagsFn)
 
-		is_unicode = ((flags & INFO_UNICODE) != 0)
-		has_null = (code_page == 1252 || is_unicode)
+		isUnicode = ((flags & INFO_UNICODE) != 0)
+		hasNull = (codePage == 1252 || isUnicode)
 
-		if has_null {
-			null_n = 1
+		if hasNull {
+			nullN = 1
 		}
-		if is_unicode {
-			unicode_n = 2
+		if isUnicode {
+			unicodeN = 2
 		}
 
-		domain_length := int(d.FieldU16("domain_length") + null_n*unicode_n)
-		username_length := int(d.FieldU16("username_length") + null_n*unicode_n)
-		password_length := int(d.FieldU16("password_length") + null_n*unicode_n)
-		alternate_shell_length := int(d.FieldU16("alternate_shell_length") + null_n*unicode_n)
-		working_dir_length := int(d.FieldU16("working_dir_length") + null_n*unicode_n)
+		domainLength := int(d.FieldU16("domain_length") + nullN*unicodeN)
+		usernameLength := int(d.FieldU16("username_length") + nullN*unicodeN)
+		passwordLength := int(d.FieldU16("password_length") + nullN*unicodeN)
+		alternateShellLength := int(d.FieldU16("alternate_shell_length") + nullN*unicodeN)
+		workingDirLength := int(d.FieldU16("working_dir_length") + nullN*unicodeN)
 
-		d.FieldUTF16LE("domain", domain_length, scalar.StrActualTrim("\x00"))
-		d.FieldUTF16LE("username", username_length, scalar.StrActualTrim("\x00"))
-		d.FieldUTF16LE("password", password_length, scalar.StrActualTrim("\x00"))
-		d.FieldUTF16LE("alternate_shell", alternate_shell_length, scalar.StrActualTrim("\x00"))
-		d.FieldUTF16LE("working_dir", working_dir_length, scalar.StrActualTrim("\x00"))
+		d.FieldUTF16LE("domain", domainLength, scalar.StrActualTrim("\x00"))
+		d.FieldUTF16LE("username", usernameLength, scalar.StrActualTrim("\x00"))
+		d.FieldUTF16LE("password", passwordLength, scalar.StrActualTrim("\x00"))
+		d.FieldUTF16LE("alternate_shell", alternateShellLength, scalar.StrActualTrim("\x00"))
+		d.FieldUTF16LE("working_dir", workingDirLength, scalar.StrActualTrim("\x00"))
 
-		extra_length := length - ((d.Pos() - pos) / 8)
-		if extra_length > 0 {
+		extraLength := length - ((d.Pos() - pos) / 8)
+		if extraLength > 0 {
 			d.FieldStruct("extra_info", func(d *decode.D) {
 				d.FieldU16("address_family", scalar.UintHex)
-				address_length := int(d.FieldU16("address_length"))
-				d.FieldUTF16LE("address", address_length, scalar.StrActualTrim("\x00"))
-				client_dir_length := int(d.FieldU16("client_dir_length"))
-				d.FieldUTF16LE("client_dir", client_dir_length, scalar.StrActualTrim("\x00"))
+				addressLength := int(d.FieldU16("address_length"))
+				d.FieldUTF16LE("address", addressLength, scalar.StrActualTrim("\x00"))
+				clientDirLength := int(d.FieldU16("client_dir_length"))
+				d.FieldUTF16LE("client_dir", clientDirLength, scalar.StrActualTrim("\x00"))
 				// TS_TIME_ZONE_INFORMATION structure
 				// https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-rdpbcgr/526ed635-d7a9-4d3c-bbe1-4e3fb17585f4
 				d.FieldU32("timezone_bias")
@@ -92,14 +92,14 @@ const (
 func decodeFlagsFn(d *decode.D) {
 	d.FieldBool("mouse")
 	d.FieldBool("disabledctrlaltdel")
-	d.SeekRel(1)
+	d.FieldRawLen("unused0", 1)
 	d.FieldBool("autologon")
 	d.FieldBool("unicode")
 	d.FieldBool("maximizeshell")
 	d.FieldBool("logonnotify")
 	d.FieldBool("compression")
 	d.FieldBool("enablewindowskey")
-	d.SeekRel(4)
+	d.FieldRawLen("unused1", 4)
 	d.FieldBool("remoteconsoleaudio")
 	d.FieldBool("force_encrypted_cs_pdu")
 	d.FieldBool("rail")
@@ -113,6 +113,5 @@ func decodeFlagsFn(d *decode.D) {
 	d.FieldBool("reserved1")
 	d.FieldBool("reserved2")
 	d.FieldBool("hidef_rail_supported")
-
-	d.SeekRel(d.Pos() % 31)
+	d.FieldRawLen("unused2", 6)
 }

format/pyrdp/pdu/clipboard_data.go

@@ -30,24 +30,24 @@ const (
 )
 
 var cbTypesMap = scalar.UintMapSymStr{
-	CB_MONITOR_READY:         "cb_monitor_ready",
-	CB_FORMAT_LIST:           "cb_format_list",
-	CB_FORMAT_LIST_RESPONSE:  "cb_format_list_response",
-	CB_FORMAT_DATA_REQUEST:   "cb_format_data_request",
-	CB_FORMAT_DATA_RESPONSE:  "cb_format_data_response",
-	CB_TEMP_DIRECTORY:        "cb_temp_directory",
-	CB_CLIP_CAPS:             "cb_clip_caps",
-	CB_FILECONTENTS_REQUEST:  "cb_filecontents_request",
-	CB_FILECONTENTS_RESPONSE: "cb_filecontents_response",
-	CB_LOCK_CLIPDATA:         "cb_lock_clipdata",
-	CB_UNLOCK_CLIPDATA:       "cb_unlock_clipdata",
+	CB_MONITOR_READY:         "monitor_ready",
+	CB_FORMAT_LIST:           "format_list",
+	CB_FORMAT_LIST_RESPONSE:  "format_list_response",
+	CB_FORMAT_DATA_REQUEST:   "format_data_request",
+	CB_FORMAT_DATA_RESPONSE:  "format_data_response",
+	CB_TEMP_DIRECTORY:        "temp_directory",
+	CB_CLIP_CAPS:             "clip_caps",
+	CB_FILECONTENTS_REQUEST:  "filecontents_request",
+	CB_FILECONTENTS_RESPONSE: "filecontents_response",
+	CB_LOCK_CLIPDATA:         "lock_clipdata",
+	CB_UNLOCK_CLIPDATA:       "unlock_clipdata",
 }
 
 var cbFlagsMap = scalar.UintMapSymStr{
 	NONE:             "none",
-	CB_RESPONSE_OK:   "cb_response_ok",
-	CB_RESPONSE_FAIL: "cb_response_fail",
-	CB_ASCII_NAMES:   "cb_ascii_names",
+	CB_RESPONSE_OK:   "response_ok",
+	CB_RESPONSE_FAIL: "response_fail",
+	CB_ASCII_NAMES:   "ascii_names",
 }
 
 var cbParseFnMap = map[uint16]interface{}{
@@ -56,20 +56,20 @@ var cbParseFnMap = map[uint16]interface{}{
 
 func ParseClipboardData(d *decode.D, length int64) {
 	d.FieldStruct("clipboard_data", func(d *decode.D) {
-		msg_type := uint16(d.FieldU16("msg_type", cbTypesMap))
+		msgType := uint16(d.FieldU16("msg_type", cbTypesMap))
 		d.FieldU16("msg_flags", cbFlagsMap)
-		data_length := d.FieldU32("data_len")
+		dataLength := d.FieldU32("data_len")
 
-		cbParser, ok := cbParseFnMap[msg_type]
+		cbParser, ok := cbParseFnMap[msgType]
 		if ok {
 			parseFn, ok := cbParser.(func(d *decode.D, length uint64))
 			if ok {
-				parseFn(d, data_length)
+				parseFn(d, dataLength)
 				return
 			}
 		}
 		// Assert() once all functions are implemented.
-		d.FieldRawLen("data", int64(data_length*8))
+		d.FieldRawLen("data", int64(dataLength*8))
 	})
 }
 

format/pyrdp/pdu/fastpath_input.go

@@ -67,9 +67,9 @@ func ParseFastPathInput(d *decode.D, length int64) {
 			}
 		})
 
-		input_length := d.FieldU8("input_length1", scalar.UintHex)
-		if input_length&0x80 != 0 {
-			input_length = ((input_length & 0x7f) << 8) | d.FieldU8("input_length2", scalar.UintHex)
+		inputLength := d.FieldU8("input_length1", scalar.UintHex)
+		if inputLength&0x80 != 0 {
+			inputLength = ((inputLength & 0x7f) << 8) | d.FieldU8("input_length2", scalar.UintHex)
 		}
 
 		// d.FieldU64("data_signature", scalar.Hex)
@@ -92,9 +92,9 @@ func ParseFastPathInput(d *decode.D, length int64) {
 		// 	}
 		// })
 
-		input_length -= uint64(d.Pos()-pos) / 8
-		if input_length > 0 {
-			d.FieldRawLen("data", int64(input_length*8))
+		inputLength -= uint64(d.Pos()-pos) / 8
+		if inputLength > 0 {
+			d.FieldRawLen("data", int64(inputLength*8))
 		}
 	})
 }

format/pyrdp/pyrdp.go

@@ -10,6 +10,7 @@
 package pyrdp
 
 import (
+	"embed"
 	"time"
 
 	"github.com/wader/fq/format"
@@ -19,6 +20,19 @@ import (
 	"github.com/wader/fq/pkg/scalar"
 )
 
+//go:embed pyrdp.md
+var pyrdpFS embed.FS
+
+func init() {
+	interp.RegisterFormat(
+		format.PYRDP,
+		&decode.Format{
+			Description: "PyRDP Replay Files",
+			DecodeFn:    decodePYRDP,
+		})
+	interp.RegisterFS(pyrdpFS)
+}
+
 const (
 	READ_EXTRA = true
 
@@ -46,26 +60,26 @@ const (
 )
 
 var pduTypesMap = scalar.UintMapSymStr{
-	PDU_FAST_PATH_INPUT:            "pdu_fastpath_input",
-	PDU_FAST_PATH_OUTPUT:           "pdu_fastpath_output",
-	PDU_CLIENT_INFO:                "pdu_client_info",
-	PDU_SLOW_PATH_PDU:              "pdu_slow_path_pdu",
-	PDU_CONNECTION_CLOSE:           "pdu_connection_close",
-	PDU_CLIPBOARD_DATA:             "pdu_clipboard_data",
-	PDU_CLIENT_DATA:                "pdu_client_data",
-	PDU_MOUSE_MOVE:                 "pdu_mouse_move",
-	PDU_MOUSE_BUTTON:               "pdu_mouse_button",
-	PDU_MOUSE_WHEEL:                "pdu_mouse_wheel",
-	PDU_KEYBOARD:                   "pdu_keyboard",
-	PDU_TEXT:                       "pdu_text",
-	PDU_FORWARDING_STATE:           "pdu_forwarding_state",
-	PDU_BITMAP:                     "pdu_bitmap",
-	PDU_DEVICE_MAPPING:             "pdu_device_mapping",
-	PDU_DIRECTORY_LISTING_REQUEST:  "pdu_directory_listing_request",
-	PDU_DIRECTORY_LISTING_RESPONSE: "pdu_directory_listing_response",
-	PDU_FILE_DOWNLOAD_REQUEST:      "pdu_file_download_request",
-	PDU_FILE_DOWNLOAD_RESPONSE:     "pdu_file_download_response",
-	PDU_FILE_DOWNLOAD_COMPLETE:     "pdu_file_download_complete",
+	PDU_FAST_PATH_INPUT:            "fastpath_input",
+	PDU_FAST_PATH_OUTPUT:           "fastpath_output",
+	PDU_CLIENT_INFO:                "client_info",
+	PDU_SLOW_PATH_PDU:              "slow_path_pdu",
+	PDU_CONNECTION_CLOSE:           "connection_close",
+	PDU_CLIPBOARD_DATA:             "clipboard_data",
+	PDU_CLIENT_DATA:                "client_data",
+	PDU_MOUSE_MOVE:                 "mouse_move",
+	PDU_MOUSE_BUTTON:               "mouse_button",
+	PDU_MOUSE_WHEEL:                "mouse_wheel",
+	PDU_KEYBOARD:                   "keyboard",
+	PDU_TEXT:                       "text",
+	PDU_FORWARDING_STATE:           "forwarding_state",
+	PDU_BITMAP:                     "bitmap",
+	PDU_DEVICE_MAPPING:             "device_mapping",
+	PDU_DIRECTORY_LISTING_REQUEST:  "directory_listing_request",
+	PDU_DIRECTORY_LISTING_RESPONSE: "directory_listing_response",
+	PDU_FILE_DOWNLOAD_REQUEST:      "file_download_request",
+	PDU_FILE_DOWNLOAD_RESPONSE:     "file_download_response",
+	PDU_FILE_DOWNLOAD_COMPLETE:     "file_download_complete",
 }
 
 var pduParsersMap = map[uint16]interface{}{
@@ -91,15 +105,6 @@ var pduParsersMap = map[uint16]interface{}{
 	// PDU_FILE_DOWNLOAD_COMPLETE:     pyrdp_pdu.ParseFileDownloadComplete,
 }
 
-func init() {
-	interp.RegisterFormat(
-		format.PYRDP,
-		&decode.Format{
-			Description: "PyRDP Replay Files",
-			DecodeFn:    decodePYRDP,
-		})
-}
-
 func decodePYRDP(d *decode.D) any {
 	d.Endian = decode.LittleEndian
 
@@ -109,14 +114,14 @@ func decodePYRDP(d *decode.D) any {
 				pos := d.Pos()
 
 				size := d.FieldU64("size") // minus the length
-				pdu_type := uint16(d.FieldU16("pdu_type", pduTypesMap))
-				d.FieldU64("timestamp", timestampMapper)
-				pdu_size := int64(size - 18)
+				pduType := uint16(d.FieldU16("pdu_type", pduTypesMap))
+				d.FieldU64("timestamp", scalar.UintActualUnixTimeDescription(time.Millisecond, time.RFC3339Nano))
+				pduSize := int64(size - 18)
 
-				pduParser, ok := pduParsersMap[pdu_type]
+				pduParser, ok := pduParsersMap[pduType]
 				if !ok { // catch undeclared parsers
-					if pdu_size > 0 {
-						d.FieldRawLen("data", pdu_size*8)
+					if pduSize > 0 {
+						d.FieldRawLen("data", pduSize*8)
 					}
 					return
 				}
@@ -124,7 +129,7 @@ func decodePYRDP(d *decode.D) any {
 				if !ok {
 					return
 				}
-				parseFn(d, pdu_size)
+				parseFn(d, pduSize)
 
 				curr := d.Pos() - pos
 				if READ_EXTRA {
@@ -139,8 +144,3 @@ func decodePYRDP(d *decode.D) any {
 }
 
 func noParse(d *decode.D, length int64) {}
-
-var timestampMapper = scalar.UintFn(func(s scalar.Uint) (scalar.Uint, error) {
-	s.Sym = time.UnixMilli(int64(s.Actual)).UTC().String()
-	return s, nil
-})

format/pyrdp/pyrdp.md

@@ -0,0 +1,6 @@
+### Authors
+- Olivier Bilodeau <olivier.bilodeau@flare.io>, Maintainer
+- Lisandro Ubiedo, Author
+
+### References
+- https://github.com/GoSecure/pyrdp