wader/fq
1
1
Fork 0
mirror of https://github.com/wader/fq.git synced 2024-10-04 07:27:08 +03:00
Code Issues Projects Releases Wiki Activity

postgres: fix linter

Browse Source
This commit is contained in:
Pavel Safonov 2022-09-06 12:55:15 +03:00
parent f224ed000e
commit 85c042289d
9 changed files with 211 additions and 218 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
format/postgres/common/pg_control.go
View File

@ -2,7 +2,9 @@ package common
import (
"fmt"
"github.com/wader/fq/pkg/scalar"
"time"
)
@ -91,6 +93,7 @@ var TimeMapper = timeMapper{}
// PGPRO_STANDARD = ('P'<<8|'P'),
// PGPRO_ENTERPRISE = ('P'<<8|'E'),
//} PgEdition;
//nolint:revive
const (
PG_UNKNOWN = 0xFFFF
PG_ORIGINAL = 0

7
format/postgres/common/pgheap.go
View File

@ -4,9 +4,10 @@ import (
"github.com/wader/fq/pkg/scalar"
)
//nolint:revive
const (
HeapPageSize = 8192
FirstNormalTransactionId = 3
FirstNormalTransactionID = 3
LP_UNUSED = 0 /* unused (should always have lp_len=0) */
LP_NORMAL = 1 /* used (should always have lp_len>0) */
@ -14,8 +15,8 @@ const (
LP_DEAD = 3
)
func TransactionIdIsNormal(xid uint64) bool {
return xid >= FirstNormalTransactionId
func TransactionIDIsNormal(xid uint64) bool {
return xid >= FirstNormalTransactionID
}
type lpOffMapper struct{}

3
format/postgres/common/utils_test.go
View File

@ -1,8 +1,9 @@
package common_test
import (
"github.com/wader/fq/format/postgres/common"
"testing"
"github.com/wader/fq/format/postgres/common"
)
func TestTypeAlign8(t *testing.T) {

31
format/postgres/flavours/postgres14/common14/pgheap.go
View File

@ -2,11 +2,14 @@ package common14
import (
"fmt"
"github.com/wader/fq/format/postgres/common"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/scalar"
)
//nolint:revive
const (
HEAP_HASNULL = 0x0001 /* has null attribute(s) */
HEAP_HASVARWIDTH = 0x0002 /* has variable-width attribute(s) */
@ -37,6 +40,7 @@ const (
HEAP_MOVED = HEAP_MOVED_OFF | HEAP_MOVED_IN
)
//nolint:revive
const (
HEAP_KEYS_UPDATED = 0x2000 /* tuple was updated and key cols modified, or tuple deleted */
HEAP_HOT_UPDATED = 0x4000 /* tuple was HOT-updated */
@ -131,7 +135,7 @@ type HeapPageD struct {
PdSpecial uint16
PdPagesizeVersion uint16
ItemIds []itemIdDataD
ItemIds []itemIDDataD
PagePosBegin uint64
ItemsEnd int64
@ -146,16 +150,7 @@ type TupleD struct {
IsMulti uint64
}
func (hp *HeapPageD) getItemId(offset uint32) (bool, itemIdDataD) {
for _, id := range hp.ItemIds {
if id.lpOff == offset {
return true, id
}
}
return false, itemIdDataD{}
}
type itemIdDataD struct {
type itemIDDataD struct {
/* 0: 0 | 4 */ // unsigned int lp_off: 15
/* 1: 7 | 4 */ // unsigned int lp_flags: 2
/* 2: 1 | 4 */ // unsigned int lp_len: 15
@ -265,16 +260,16 @@ func DecodeItemIds(heap *HeapD, d *decode.D) {
/* 1: 7 | 4 */ // unsigned int lp_flags: 2
/* 2: 1 | 4 */ // unsigned int lp_len: 15
d.FieldStruct("ItemIdData", func(d *decode.D) {
itemId := itemIdDataD{}
itemID := itemIDDataD{}
itemPos := d.Pos()
itemId.lpOff = uint32(d.FieldU32("lp_off", common.LpOffMapper))
itemID.lpOff = uint32(d.FieldU32("lp_off", common.LpOffMapper))
d.SeekAbs(itemPos)
itemId.lpFlags = uint32(d.FieldU32("lp_flags", common.LpFlagsMapper))
itemID.lpFlags = uint32(d.FieldU32("lp_flags", common.LpFlagsMapper))
d.SeekAbs(itemPos)
itemId.lpLen = uint32(d.FieldU32("lp_len", common.LpLenMapper))
itemID.lpLen = uint32(d.FieldU32("lp_len", common.LpLenMapper))
page.ItemIds = append(page.ItemIds, itemId)
page.ItemIds = append(page.ItemIds, itemID)
})
} // for pd_linp
}
@ -489,12 +484,12 @@ type TransactionMapper struct {
func (m TransactionMapper) MapScalar(s scalar.S) (scalar.S, error) {
xid := s.ActualU()
if m.Page.PdXidBase != 0 && m.Tuple.IsMulti == 0 && common.TransactionIdIsNormal(xid) {
if m.Page.PdXidBase != 0 && m.Tuple.IsMulti == 0 && common.TransactionIDIsNormal(xid) {
xid64 := xid + m.Page.PdXidBase
s.Sym = fmt.Sprintf("%d", xid64)
}
if m.Page.PdMultiBase != 0 && m.Tuple.IsMulti != 0 && common.TransactionIdIsNormal(xid) {
if m.Page.PdMultiBase != 0 && m.Tuple.IsMulti != 0 && common.TransactionIDIsNormal(xid) {
xid64 := xid + m.Page.PdMultiBase
s.Sym = fmt.Sprintf("%d", xid64)
}

89
format/postgres/flavours/postgres14/pgwal.go
View File

@ -1,8 +1,8 @@
package postgres14
import (
"context"
"fmt"
"github.com/wader/fq/format/postgres/common"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/scalar"
@ -10,16 +10,19 @@ import (
//func init() {
// interp.RegisterFormat(decode.Format{
// Name: format.PGWAL,
// Name: format.PG_WAL,
// Description: "PostgreSQL write-ahead log file",
// DecodeFn: pgwalDecode,
// DecodeFn: DecodePgwal,
// })
//}
const XLOG_BLCKSZ = 8192
const XLP_LONG_HEADER = 2
//nolint:revive
const (
XLOG_BLCKSZ = 8192
XLP_LONG_HEADER = 2
)
//nolint:revive
const (
BKPBLOCK_FORK_MASK = 0x0F
/* block data is an XLogRecordBlockImage */
@ -32,6 +35,7 @@ const (
)
/* Information stored in bimg_info */
//nolint:revive
const (
/* page image has "hole" */
BKPIMAGE_HAS_HOLE = 0x01
@ -41,8 +45,6 @@ const (
BKPIMAGE_APPLY = 0x04
)
var expected_rem_len uint64 = 0
var rmgrIds = scalar.UToScalar{
0: {Sym: "XLOG", Description: "RM_XLOG_ID"},
1: {Sym: "Transaction", Description: "RM_XACT_ID"},
@ -132,30 +134,21 @@ type walD struct {
record *decode.D
}
func getWalD(d *decode.D) *walD {
val := d.Ctx.Value("wald")
return val.(*walD)
}
func DecodePgwal(d *decode.D, in any) any {
walD := &walD{
pages: d.FieldArrayValue("pages"),
records: d.FieldArrayValue("records"),
}
parentCtx := d.Ctx
ctx := context.WithValue(parentCtx, "wald", walD)
d.Ctx = ctx
d.SeekAbs(0)
d.FieldArray("XLogPages", decodeXLogPage)
d.FieldArray("XLogPages", func(d *decode.D) {
decodeXLogPage(walD, d)
})
return nil
}
func decodeXLogPage(d *decode.D) {
wal := getWalD(d)
func decodeXLogPage(wal *walD, d *decode.D) {
// type = struct XLogPageHeaderData {
/* 0 | 2 */ // uint16 xlp_magic;
/* 2 | 2 */ // uint16 xlp_info;
@ -169,7 +162,7 @@ func decodeXLogPage(d *decode.D) {
xlpInfo := page.FieldU16("xlp_info")
page.FieldU32("xlp_tli")
page.FieldU64("xlp_pageaddr")
remLen := page.FieldU32("xlp_rem_len")
page.FieldU32("xlp_rem_len")
page.U32()
if xlpInfo&XLP_LONG_HEADER != 0 {
@ -181,19 +174,19 @@ func decodeXLogPage(d *decode.D) {
})
}
remLen = 40
remLen := 40
wal.remLen = uint32(remLen)
record := wal.record
if record == nil {
rawLen := int64(common.TypeAlign8(remLen))
rawLen := int64(common.TypeAlign8(uint64(remLen)))
page.FieldRawLen("prev_file_rec", rawLen*8)
}
pageRecords := page.FieldArrayValue("records")
wal.pageRecords = pageRecords
decodeXLogRecords(d)
decodeXLogRecords(wal, d)
//page.Pos()
//for {
@ -202,8 +195,7 @@ func decodeXLogPage(d *decode.D) {
//fmt.Printf("d pos = %d\n", d.Pos())
}
func decodeXLogRecords(d *decode.D) {
wal := getWalD(d)
func decodeXLogRecords(wal *walD, d *decode.D) {
pageRecords := wal.pageRecords
pos := d.Pos() / 8
@ -279,16 +271,21 @@ func DecodePgwalOri(d *decode.D, in any) any {
pageHeaders := d.FieldArrayValue("XLogPageHeaders")
header := pageHeaders.FieldStruct("XLogPageHeaderData", decodeXLogPageHeaderData)
d.FieldRawLen("prev_file_rec", int64(header.FieldGet("xlp_rem_len").V.(uint32)*8))
xlpRemLen, ok := header.FieldGet("xlp_rem_len").V.(uint32)
if !ok {
d.Fatalf("can't get xlp_rem_len\n")
}
d.FieldRawLen("prev_file_rec", int64(xlpRemLen*8))
d.FieldRawLen("prev_file_rec_padding", int64(d.AlignBits(64)))
d.FieldArray("XLogRecords", func(d *decode.D) {
for {
d.FieldStruct("XLogRecord", func(d *decode.D) {
record_pos := uint64(d.Pos()) >> 3
record_len := d.FieldU32("xl_tot_len")
record_end := record_pos + record_len
header_pos := record_end - record_end%XLOG_BLCKSZ
recordPos := uint64(d.Pos()) >> 3
recordLen := d.FieldU32("xl_tot_len")
recordEnd := recordPos + recordLen
headerPos := recordEnd - recordEnd%XLOG_BLCKSZ
d.FieldU32("xl_xid")
d.FieldU64("xl_prev", scalar.ActualHex)
d.FieldU8("xl_info")
@ -296,7 +293,7 @@ func DecodePgwalOri(d *decode.D, in any) any {
d.FieldRawLen("padding", int64(d.AlignBits(32)))
d.FieldU32("xl_crc", scalar.ActualHex)
var lenghts []uint64 = []uint64{}
var lengths []uint64
d.FieldArray("XLogRecordBlockHeader", func(d *decode.D) {
for blkheaderid := uint64(0); d.PeekBits(8) == blkheaderid; blkheaderid++ {
@ -304,25 +301,25 @@ func DecodePgwalOri(d *decode.D, in any) any {
/* block reference ID */
d.FieldU8("id", d.AssertU(blkheaderid))
/* fork within the relation, and flags */
fork_flags := d.FieldU8("fork_flags")
forkFlags := d.FieldU8("fork_flags")
/* number of payload bytes (not including page image) */
lenghts = append(lenghts, d.FieldU16("data_length"))
if fork_flags&BKPBLOCK_HAS_IMAGE != 0 {
lengths = append(lengths, d.FieldU16("data_length"))
if forkFlags&BKPBLOCK_HAS_IMAGE != 0 {
d.FieldStruct("XLogRecordBlockImageHeader", func(d *decode.D) {
/* number of page image bytes */
d.FieldU16("length")
/* number of bytes before "hole" */
d.FieldU16("hole_offset")
/* flag bits, see below */
bimg_info := d.FieldU8("bimg_info")
bimgInfo := d.FieldU8("bimg_info")
d.FieldRawLen("padding", int64(d.AlignBits(16)))
if bimg_info&BKPIMAGE_HAS_HOLE != 0 &&
bimg_info&BKPIMAGE_IS_COMPRESSED != 0 {
if bimgInfo&BKPIMAGE_HAS_HOLE != 0 &&
bimgInfo&BKPIMAGE_IS_COMPRESSED != 0 {
d.FieldU16("hole_length")
}
})
}
if fork_flags&BKPBLOCK_SAME_REL == 0 {
if forkFlags&BKPBLOCK_SAME_REL == 0 {
d.FieldStruct("RelFileNode", func(d *decode.D) {
/* tablespace */
d.FieldU32("spcNode")
@ -339,18 +336,18 @@ func DecodePgwalOri(d *decode.D, in any) any {
if d.PeekBits(8) == 0xff {
d.FieldStruct("XLogRecordDataHeaderShort", func(d *decode.D) {
d.FieldU8("id", d.AssertU(0xff))
lenghts = append(lenghts, d.FieldU8("data_length"))
lengths = append(lengths, d.FieldU8("data_length"))
})
}
d.FieldArray("data", func(d *decode.D) {
for _, x := range lenghts {
for _, x := range lengths {
pos := uint64(d.Pos()) >> 3
if pos < header_pos && (header_pos < pos+x) {
d.FieldRawLen("data", int64((header_pos-pos)*8))
if pos < headerPos && (headerPos < pos+x) {
d.FieldRawLen("data", int64((headerPos-pos)*8))
header := pageHeaders.FieldStruct("XLogPageHeaderData", decodeXLogPageHeaderData)
header.FieldGet("xlp_rem_len").TryScalarFn(d.ValidateU(record_end - header_pos))
d.FieldRawLen("data", int64((x+pos-header_pos)*8))
_ = header.FieldGet("xlp_rem_len").TryScalarFn(d.ValidateU(recordEnd - headerPos))
d.FieldRawLen("data", int64((x+pos-headerPos)*8))
} else {
d.FieldRawLen("data", int64(x*8))
}

10
format/postgres/pg_control.go
View File

@ -31,6 +31,7 @@ func init() {
})
}
//nolint:revive
const (
PG_CONTROL_VERSION_10 = 1002
PG_CONTROL_VERSION_11 = 1100
@ -40,6 +41,7 @@ const (
PG_CONTROL_VERSION_14 = 1300
)
//nolint:revive
const (
PG_FLAVOUR_POSTGRES = "postgres"
PG_FLAVOUR_POSTGRES11 = "postgres11"
@ -60,8 +62,12 @@ const (
func decodePgControl(d *decode.D, in any) any {
d.Endian = decode.LittleEndian
flavour := in.(format.PostgresIn).Flavour
switch flavour {
pgIn, ok := in.(format.PostgresIn)
if !ok {
d.Fatalf("DecodeInArg must be PostgresIn!\n")
}
switch pgIn.Flavour {
case PG_FLAVOUR_POSTGRES11:
return postgres11.DecodePgControl(d, in)
case PG_FLAVOUR_POSTGRES12:

8
format/postgres/pg_heap.go
View File

@ -38,8 +38,12 @@ func init() {
func decodePgheap(d *decode.D, in any) any {
d.Endian = decode.LittleEndian
flavour := in.(format.PostgresIn).Flavour
switch flavour {
pgIn, ok := in.(format.PostgresIn)
if !ok {
d.Fatalf("DecodeInArg must be PostgresIn!\n")
}
switch pgIn.Flavour {
case PG_FLAVOUR_POSTGRES11:
return postgres11.DecodeHeap(d)
case PG_FLAVOUR_POSTGRES12:

161
format/postgres/pg_multixact.go
View File

@ -1,89 +1,82 @@
package postgres
import (
"github.com/wader/fq/format"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
"github.com/wader/fq/pkg/scalar"
)
// TO DO
// actualize
const BLCKSZ = 8192
func init() {
interp.RegisterFormat(decode.Format{
Name: format.PG_MULTIXACTOFF,
Description: "PostgreSQL multixact offset file",
DecodeFn: mxOffsetDecode,
})
interp.RegisterFormat(decode.Format{
Name: format.PG_MULTIXACTMEM,
Description: "PostgreSQL multixact members file",
DecodeFn: mxMembersDecode,
})
}
func mxOffsetDecode(d *decode.D, in interface{}) interface{} {
d.Endian = decode.LittleEndian
d.FieldArray("offsets", func(d *decode.D) {
for {
if d.End() {
break
}
d.FieldU32("offset", scalar.ActualHex)
}
})
return nil
}
var flags = scalar.UToScalar{
0: {Sym: "ForKeyShare", Description: "For Key Share"},
1: {Sym: "ForShare", Description: "For Share"},
2: {Sym: "ForNoKeyUpdate", Description: "For No Key Update"},
3: {Sym: "ForUpdate", Description: "For Update"},
4: {Sym: "NoKeyUpdate", Description: "No Key Update"},
5: {Sym: "Update", Description: "Update"},
}
func mxMembersDecode(d *decode.D, in interface{}) interface{} {
var xidLen uint = 4
var groupLen uint = 4 * (1 + xidLen)
d.Endian = decode.LittleEndian
m := d.FieldArrayValue("members")
p := d.FieldArrayValue("paddings")
for {
var xacts []*decode.D = make([]*decode.D, 4)
for i := 0; i < 4; i++ {
xacts[i] = m.FieldStructValue("xact")
xacts[i].FieldU8("status", flags)
}
for i := 0; i < 4; i++ {
xacts[i].FieldU32("xid")
}
// Check if rest of bytes are padding before EOF
if d.BitsLeft() < int64(groupLen*8) && d.BitsLeft() > 0 {
p.FieldRawLen("padding", d.BitsLeft())
break
}
// Check on EOF
if d.End() {
break
}
// Not EOF, let's check on block boundary
if blkLeft := BLCKSZ - (uint(d.Pos())>>3)%BLCKSZ; blkLeft < groupLen {
p.FieldRawLen("padding", int64(blkLeft*8))
}
}
return nil
}
//const BLCKSZ = 8192
//
//func init() {
// interp.RegisterFormat(decode.Format{
// Name: format.PG_MULTIXACTOFF,
// Description: "PostgreSQL multixact offset file",
// DecodeFn: mxOffsetDecode,
// })
// interp.RegisterFormat(decode.Format{
// Name: format.PG_MULTIXACTMEM,
// Description: "PostgreSQL multixact members file",
// DecodeFn: mxMembersDecode,
// })
//}
//
//func mxOffsetDecode(d *decode.D, in interface{}) interface{} {
// d.Endian = decode.LittleEndian
//
// d.FieldArray("offsets", func(d *decode.D) {
// for {
// if d.End() {
// break
// }
// d.FieldU32("offset", scalar.ActualHex)
//
// }
// })
// return nil
//}
//
//var flags = scalar.UToScalar{
// 0: {Sym: "ForKeyShare", Description: "For Key Share"},
// 1: {Sym: "ForShare", Description: "For Share"},
// 2: {Sym: "ForNoKeyUpdate", Description: "For No Key Update"},
// 3: {Sym: "ForUpdate", Description: "For Update"},
// 4: {Sym: "NoKeyUpdate", Description: "No Key Update"},
// 5: {Sym: "Update", Description: "Update"},
//}
//
//func mxMembersDecode(d *decode.D, in interface{}) interface{} {
// var xidLen uint = 4
// var groupLen uint = 4 * (1 + xidLen)
// d.Endian = decode.LittleEndian
//
// m := d.FieldArrayValue("members")
// p := d.FieldArrayValue("paddings")
//
// for {
// var xacts []*decode.D = make([]*decode.D, 4)
//
// for i := 0; i < 4; i++ {
// xacts[i] = m.FieldStructValue("xact")
// xacts[i].FieldU8("status", flags)
// }
//
// for i := 0; i < 4; i++ {
// xacts[i].FieldU32("xid")
// }
//
// // Check if rest of bytes are padding before EOF
// if d.BitsLeft() < int64(groupLen*8) && d.BitsLeft() > 0 {
// p.FieldRawLen("padding", d.BitsLeft())
// break
// }
//
// // Check on EOF
// if d.End() {
// break
// }
//
// // Not EOF, let's check on block boundary
// if blkLeft := BLCKSZ - (uint(d.Pos())>>3)%BLCKSZ; blkLeft < groupLen {
// p.FieldRawLen("padding", int64(blkLeft*8))
// }
// }
// return nil
//}

117
format/postgres/pg_wal.go
View File

@ -1,67 +1,60 @@
package postgres
import (
"github.com/wader/fq/format"
"github.com/wader/fq/format/postgres/flavours/postgres14"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
)
// TO DO
// not ready yet
func init() {
interp.RegisterFormat(decode.Format{
Name: format.PG_WAL,
Description: "PostgreSQL write-ahead log file",
DecodeFn: decodePgwal,
DecodeInArg: format.PostgresIn{
Flavour: "default",
},
})
}
// https://pgpedia.info/x/XLOG_PAGE_MAGIC.html
const (
XLOG_PAGE_MAGIC_15 = uint16(0xD10F)
XLOG_PAGE_MAGIC_14 = uint16(0xD10D)
XLOG_PAGE_MAGIC_13 = uint16(0xD106)
XLOG_PAGE_MAGIC_12 = uint16(0xD101)
XLOG_PAGE_MAGIC_11 = uint16(0xD098)
XLOG_PAGE_MAGIC_10 = uint16(0xD097)
XLOG_PAGE_MAGIC_96 = uint16(0xD093)
)
func decodePgwal(d *decode.D, in any) any {
d.Endian = decode.LittleEndian
flavour := in.(format.PostgresIn).Flavour
switch flavour {
//case PG_FLAVOUR_POSTGRES11:
// return postgres11.DecodePgControl(d, in)
case PG_FLAVOUR_POSTGRES14, PG_FLAVOUR_POSTGRES:
return postgres14.DecodePgwal(d, in)
//case PG_FLAVOUR_PGPROEE14:
// return pgproee14.DecodePgControl(d, in)
default:
break
}
return probePgwal(d, in)
}
func probePgwal(d *decode.D, in any) any {
// read version
xlp_magic := uint16(d.U16())
// restore position
d.SeekAbs(0)
switch xlp_magic {
case XLOG_PAGE_MAGIC_14:
return postgres14.DecodePgwal(d, in)
default:
d.Fatalf("unsupported xlp_magic = %X\n", xlp_magic)
}
return nil
}
//func init() {
// interp.RegisterFormat(decode.Format{
// Name: format.PG_WAL,
// Description: "PostgreSQL write-ahead log file",
// DecodeFn: decodePgwal,
// DecodeInArg: format.PostgresIn{
// Flavour: "default",
// },
// })
//}
//
//// https://pgpedia.info/x/XLOG_PAGE_MAGIC.html
//const (
// XLOG_PAGE_MAGIC_15 = uint16(0xD10F)
// XLOG_PAGE_MAGIC_14 = uint16(0xD10D)
// XLOG_PAGE_MAGIC_13 = uint16(0xD106)
// XLOG_PAGE_MAGIC_12 = uint16(0xD101)
// XLOG_PAGE_MAGIC_11 = uint16(0xD098)
// XLOG_PAGE_MAGIC_10 = uint16(0xD097)
// XLOG_PAGE_MAGIC_96 = uint16(0xD093)
//)
//
//func decodePgwal(d *decode.D, in any) any {
// d.Endian = decode.LittleEndian
//
// flavour := in.(format.PostgresIn).Flavour
// switch flavour {
// //case PG_FLAVOUR_POSTGRES11:
// // return postgres11.DecodePgControl(d, in)
// case PG_FLAVOUR_POSTGRES14, PG_FLAVOUR_POSTGRES:
// return postgres14.DecodePgwal(d, in)
// //case PG_FLAVOUR_PGPROEE14:
// // return pgproee14.DecodePgControl(d, in)
// default:
// break
// }
//
// return probePgwal(d, in)
//}
//
//func probePgwal(d *decode.D, in any) any {
// // read version
// xlp_magic := uint16(d.U16())
//
// // restore position
// d.SeekAbs(0)
//
// switch xlp_magic {
// case XLOG_PAGE_MAGIC_14:
// return postgres14.DecodePgwal(d, in)
// default:
// d.Fatalf("unsupported xlp_magic = %X\n", xlp_magic)
// }
// return nil
//}