wader/fq
1
1
Fork 0
mirror of https://github.com/wader/fq.git synced 2024-09-11 12:05:39 +03:00
Code Issues Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Mattias Wadman 2022-12-16 15:51:03 +01:00
parent cfd18e7657
commit 35f8379e08
10 changed files with 836 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
format/format.go
View File

@ -142,6 +142,10 @@ var (
MPEG_PES_Packet = &decode.Group{Name: "mpeg_pes_packet"}
MPEG_SPU = &decode.Group{Name: "mpeg_spu"}
MPEG_TS = &decode.Group{Name: "mpeg_ts"}
MPEG_TS_Packet = &decode.Group{Name: "mpeg_ts_packet"}
MPEG_TS_PAT = &decode.Group{Name: "mpeg_ts_pat"}
MPEG_TS_PMT = &decode.Group{Name: "mpeg_ts_pmt"}
MPEG_TS_SDT = &decode.Group{Name: "mpeg_ts_sdt"}
MsgPack = &decode.Group{Name: "msgpack"}
Ogg = &decode.Group{Name: "ogg"}
Ogg_Page = &decode.Group{Name: "ogg_page"}
@ -398,3 +402,35 @@ type Pg_Heap_In struct {
type Pg_BTree_In struct {
Page int `doc:"First page number in file, default is 0"`
}
type MpegTsStream struct {
ProgramPid int
Type int
}
type MpegTsProgram struct {
Number int
Pid int
StreamPids []int
}
type MpegTsPacketIn struct {
ProgramMap map[int]MpegTsProgram
StreamMap map[int]MpegTsStream
}
type MpegTsPacketOut struct {
Pid int
ContinuityCounter int
TransportScramblingControl int
PayloadUnitStart bool
Payload []byte
}
type MpegTsPatOut struct {
PidMap map[int]int // pid to program number that has pmt
}
type MpegTsPmtOut struct {
Streams map[int]MpegTsStream
}

67
format/mpeg/mpeg_pes.go
View File

@ -5,6 +5,8 @@ package mpeg
// http://dvdnav.mplayerhq.hu/dvdinfo/mpeghdrs.html
import (
"log"
"github.com/wader/fq/format"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/decode"
@ -20,8 +22,8 @@ func init() {
&decode.Format{
Description: "MPEG Packetized elementary stream",
DecodeFn: pesDecode,
RootArray: true,
RootName: "packets",
// RootArray: true,
RootName: "packets",
Dependencies: []decode.Dependency{
{Groups: []*decode.Group{format.MPEG_PES_Packet}, Out: &pesPacketGroup},
{Groups: []*decode.Group{format.MPEG_SPU}, Out: &mpegSpuGroup},
@ -46,36 +48,39 @@ func pesDecode(d *decode.D) any {
spuD := d.FieldArrayValue("spus")
for d.NotEnd() {
dv, v, err := d.TryFieldFormat("packet", &pesPacketGroup, nil)
if dv == nil || err != nil {
break
d.FieldArray("packets", func(d *decode.D) {
for d.NotEnd() {
dv, v, err := d.TryFieldFormat("packet", &pesPacketGroup, nil)
if dv == nil || err != nil {
log.Printf("err: %#+v\n", err)
break
}
switch dvv := v.(type) {
case subStreamPacket:
s, ok := substreams[dvv.number]
if !ok {
s = &subStream{}
substreams[dvv.number] = s
}
s.b = append(s.b, dvv.buf...)
if s.l == 0 && len(s.b) >= 2 {
s.l = int(s.b[0])<<8 | int(s.b[1])
// TODO: zero l?
}
// TODO: is this how spu end is signalled?
if s.l == len(s.b) {
spuD.FieldFormatBitBuf("spu", bitio.NewBitReader(s.b, -1), &mpegSpuGroup, nil)
s.b = nil
s.l = 0
}
}
i++
}
switch dvv := v.(type) {
case subStreamPacket:
s, ok := substreams[dvv.number]
if !ok {
s = &subStream{}
substreams[dvv.number] = s
}
s.b = append(s.b, dvv.buf...)
if s.l == 0 && len(s.b) >= 2 {
s.l = int(s.b[0])<<8 | int(s.b[1])
// TODO: zero l?
}
// TODO: is this how spu end is signalled?
if s.l == len(s.b) {
spuD.FieldFormatBitBuf("spu", bitio.NewBitReader(s.b, -1), &mpegSpuGroup, nil)
s.b = nil
s.l = 0
}
}
i++
}
})
return nil
}

6
format/mpeg/mpeg_pes_packet.go
View File

@ -202,9 +202,9 @@ func pesPacketDecode(d *decode.D) any {
// nop
}
if d.BitsLeft() > 0 {
d.FieldRawLen("data", d.BitsLeft())
}
// if d.BitsLeft() > 0 {
// d.FieldRawLen("data", d.BitsLeft())
// }
return v
}

286
format/mpeg/mpeg_ts.go
View File

@ -1,12 +1,36 @@
package mpeg
// TODO: dump bug: array with only sub buffers show wrong summary bytes
// TODO: dump idea: array with only value scalars, collapse?
// TODO: split into generic table decoder or helper function? and use table_id to select? pass on args and return out value? switch on return type?
// TODO: probe, count?
// TODO: check crc
// TODO: mpeg_pes, share code?
// TODO: mpeg_pes_packet, length 0 for video?
// TODO: dup start?
// ffmpeg $(for i in $(seq 0 50); do echo "-f lavfi -i sine"; done) -t 100ms $(for i in $(seq 0 50); do echo "-map $i:0"; done) test2.ts
// ISO/IEC 13818-1 - Generic coding of moving pictures and associated audio information: Systems
// https://tsduck.io/download/docs/mpegts-introduction.pdf
import (
"bytes"
"encoding/binary"
"fmt"
"github.com/wader/fq/format"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
"github.com/wader/fq/pkg/scalar"
)
var mpegTsMpegTsPacketGroup decode.Group
var mpegTsMpegTsPatGroup decode.Group
var mpegTsMpegTsPmtGroup decode.Group
var mpegTsMpegPesPacketGroup decode.Group
func init() {
interp.RegisterFormat(
format.MPEG_TS,
@ -15,20 +39,264 @@ func init() {
Description: "MPEG Transport Stream",
Groups: []*decode.Group{format.Probe},
DecodeFn: tsDecode,
Dependencies: []decode.Dependency{
{Groups: []*decode.Group{format.MPEG_TS_Packet}, Out: &mpegTsMpegTsPacketGroup},
{Groups: []*decode.Group{format.MPEG_TS_PAT}, Out: &mpegTsMpegTsPatGroup},
{Groups: []*decode.Group{format.MPEG_TS_PMT}, Out: &mpegTsMpegTsPmtGroup},
{Groups: []*decode.Group{format.MPEG_PES_Packet}, Out: &mpegTsMpegPesPacketGroup},
},
})
}
// TODO: ts_packet
const (
adaptationFieldControlPayloadOnly = 0b01
adaptationFieldControlAdaptationFieldOnly = 0b10
adaptationFieldControlAdaptationFieldAndPayload = 0b11
)
type tsBuffer struct {
length int
buf bytes.Buffer
packetIndexes []int
}
func (tb *tsBuffer) Reset() {
tb.length = -1
// new bytes buffer to not share byte slice
tb.buf = bytes.Buffer{}
tb.packetIndexes = nil
}
type tsContinuityMap map[int]int
func (tcm tsContinuityMap) Update(pid int, n int) bool {
current, currentOk := tcm[pid]
tcm[pid] = n
if currentOk {
return (current+1)&0xf == n
}
return n == 0
}
func tsPesDecode(d *decode.D, pid int, programPid int, streamType int, pesBuf *tsBuffer) {
d.FieldValueUint("pid", uint64(pid), tsPidMap, scalar.UintHex) // TODO: more things? or less?
d.FieldValueUint("program", uint64(programPid), scalar.UintHex) // TODO: more things? or less?
d.FieldValueUint("stream_type", uint64(streamType), tsStreamTypeMap, scalar.UintHex) // TODO: more things? or less?
d.FieldArray("indexes", func(d *decode.D) {
for _, i := range pesBuf.packetIndexes {
d.FieldValueUint("index", uint64(i))
}
})
d.FieldRawLen("payload", d.BitsLeft())
// d.TryFieldFormatBitBuf("payload", bitio.NewBitReader(b.Bytes(), -1), mpegTsFormatMpegPesPacket, nil)
}
func tsDecode(d *decode.D) any {
d.FieldU8("sync", d.UintAssert(0x47), scalar.UintHex)
d.FieldBool("transport_error_indicator")
d.FieldBool("payload_unit_start")
d.FieldBool("transport_priority")
d.FieldU13("pid")
d.FieldU2("transport_scrambling_control")
d.FieldU2("adaptation_field_control")
d.FieldU4("continuity_counter")
var tableReassemble = map[int]*tsBuffer{}
var pesReassemble = map[int]*tsBuffer{}
pidProgramMap := map[int]format.MpegTsProgram{}
pidStreamMap := map[int]format.MpegTsStream{}
continuityMap := tsContinuityMap(map[int]int{})
packetIndex := 0
tablesD := d.FieldArrayValue("tables")
pesD := d.FieldArrayValue("pes")
d.FieldArray("packets", func(d *decode.D) {
for !d.End() {
_, v, err := d.TryFieldFormat(
"packet",
&mpegTsMpegTsPacketGroup,
format.MpegTsPacketIn{
ProgramMap: pidProgramMap,
StreamMap: pidStreamMap,
},
)
if err != nil {
// TODO: malformted packet, how?
d.FieldRawLen("packet", tsPacketLength)
continue
}
mtpo, mtpoOk := v.(format.MpegTsPacketOut)
if !mtpoOk {
panic("packet is not a MpegTsPacketOut")
}
isContinous := continuityMap.Update(mtpo.Pid, mtpo.ContinuityCounter)
isTable := tsPidIsTable(mtpo.Pid, pidProgramMap)
stream, isStream := pidStreamMap[mtpo.Pid]
// log.Printf("mtpo.Pid: %x isContinous=%t isTable=%t isStream=%t", mtpo.Pid, isContinous, isTable, isStream)
switch {
case isTable:
if !isContinous {
// TODO: reset
}
// TODO: version, current section etc
tableBuf, tableBufOk := tableReassemble[mtpo.Pid]
if !tableBufOk {
tableBuf = &tsBuffer{length: -1}
tableReassemble[mtpo.Pid] = tableBuf
}
tableBuf.packetIndexes = append(tableBuf.packetIndexes, packetIndex)
b := &tableBuf.buf
b.Write(mtpo.Payload)
// log.Printf(" b.Len() 1: %p %#+v %d\n", &b, b.Len(), tableBuf.length)
const sectionHeaderLength = 3
if tableBuf.length == -1 && b.Len() >= sectionHeaderLength {
// length is BE 10 bits of byte 1 and 2 and add header length to know expected full length
tableBuf.length = int(binary.BigEndian.Uint16(b.Bytes()[1:])&0b0000_0011_1111_1111) + sectionHeaderLength
}
// log.Printf(" b.Len() 2: %#+v %d\n", b.Len(), tableBuf.length)
if b.Len() >= tableBuf.length {
program, isPMT := pidProgramMap[mtpo.Pid]
tablesD.FieldStructRootBitBufFn("table", bitio.NewBitReader(b.Bytes(), -1), func(d *decode.D) {
d.FieldValueUint("pid", uint64(mtpo.Pid), tsPidMap, scalar.UintHex) // TODO: more things? or less?
d.FieldArray("indexes", func(d *decode.D) {
for _, i := range tableBuf.packetIndexes {
d.FieldValueUint("index", uint64(i))
}
})
switch {
case mtpo.Pid == pidPAT:
_, v, err := d.TryFieldFormat("payload", &mpegTsMpegTsPatGroup, nil)
if err != nil {
// TODO: malformted table, how?
d.FieldRawLen("payload", tsPacketLength)
} else {
mtpo, mtpoOk := v.(format.MpegTsPatOut)
if !mtpoOk {
panic(fmt.Sprintf("expected MpegTsPatOut got %#+v", v))
}
// TODO: correct? remove streams for program?
for mapPid, mapNum := range mtpo.PidMap {
if prevProgram, ok := pidProgramMap[mapPid]; ok {
for _, streamPid := range prevProgram.StreamPids {
delete(pidStreamMap, streamPid)
}
}
pidProgramMap[mapPid] = format.MpegTsProgram{
Number: mapNum,
Pid: mapPid,
}
}
}
case isPMT:
_, v, err := d.TryFieldFormat("payload", &mpegTsMpegTsPmtGroup, nil)
if err != nil {
// TODO: malformted table, how?
d.FieldRawLen("packet", tsPacketLength)
} else {
mtpo, mtpoOk := v.(format.MpegTsPmtOut)
if !mtpoOk {
panic(fmt.Sprintf("expected MpegTsPmtOut got %#+v", v))
}
// TODO: correct? replace streams?
for _, streamPid := range program.StreamPids {
delete(pidStreamMap, streamPid)
}
for streamPid, stream := range mtpo.Streams {
program.StreamPids = append(program.StreamPids, streamPid)
pidStreamMap[streamPid] = format.MpegTsStream{
ProgramPid: program.Pid,
Type: stream.Type,
}
}
}
default:
// TODO: raw table decoder?
d.FieldRawLen("payload", d.BitsLeft())
}
tableBuf.Reset()
})
}
case isStream:
if !isContinous {
// TODO: reset
}
pesBuf, pesBufOk := pesReassemble[mtpo.Pid]
if !pesBufOk {
pesBuf = &tsBuffer{length: -1}
pesReassemble[mtpo.Pid] = pesBuf
}
b := &pesBuf.buf
pesFn := func(d *decode.D) {
tsPesDecode(d, mtpo.Pid, stream.ProgramPid, stream.Type, pesBuf)
pesBuf.Reset()
}
// TODO: when to reset if wrong?
if mtpo.PayloadUnitStart && pesBuf.length == 0 && b.Len() > 0 {
// TODO: only video?
pesD.FieldStructRootBitBufFn("pes", bitio.NewBitReader(b.Bytes(), -1), pesFn)
}
pesBuf.packetIndexes = append(pesBuf.packetIndexes, packetIndex)
b.Write(mtpo.Payload)
const pesHeaderLength = 6 // 3 sync, 1 stream id, 2 length
if pesBuf.length == -1 && b.Len() >= pesHeaderLength {
// length is BE bytes 4 and 5 and add ad header length to know expected full length
length := int(binary.BigEndian.Uint16(b.Bytes()[4:]))
if length == 0 {
pesBuf.length = 0
} else {
pesBuf.length = length + pesHeaderLength
}
}
// log.Printf(" b.Len() 1: %p %#+v %d\n", &b, b.Len(), pesBuf.length)
// TODO: zero length, start flag?
if pesBuf.length > 0 && b.Len() >= pesBuf.length {
pesD.FieldStructRootBitBufFn("pes", bitio.NewBitReader(b.Bytes(), -1), pesFn)
}
default:
// unknown ts packet payload
}
packetIndex++
}
})
// TODO:
// add possible partial pes
for pid, pesBuf := range pesReassemble {
if pesBuf.buf.Len() == 0 {
continue
}
// TODO: can we assume there is a stream?
stream, isStream := pidStreamMap[pid]
if !isStream {
continue
}
pesD.FieldStructRootBitBufFn("pes", bitio.NewBitReader(pesBuf.buf.Bytes(), -1), func(d *decode.D) {
tsPesDecode(d, pid, stream.ProgramPid, stream.Type, pesBuf)
})
}
return nil
}

140
format/mpeg/mpeg_ts_packet.go Normal file
View File

@ -0,0 +1,140 @@
package mpeg
import (
"github.com/wader/fq/format"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
"github.com/wader/fq/pkg/scalar"
)
func init() {
interp.RegisterFormat(
format.MPEG_TS_Packet,
&decode.Format{
Description: "MPEG Transport Stream Packet",
DecodeFn: tsPacketDecode,
})
}
func tsPacketDecode(d *decode.D) any {
var mtpi format.MpegTsPacketIn
if d.ArgAs(&mtpi) {
mtpi.ProgramMap = map[int]format.MpegTsProgram{}
mtpi.StreamMap = map[int]format.MpegTsStream{}
}
var mtpo format.MpegTsPacketOut
d.FramedFn(tsPacketLength, func(d *decode.D) {
d.FieldU8("sync", scalar.UintHex) // TODO: sometimes not 0x47? d.UintAssert(0x47)
d.FieldBool("transport_error_indicator")
mtpo.PayloadUnitStart = d.FieldBool("payload_unit_start")
d.FieldBool("transport_priority")
pid := d.FieldU13("pid", tsPidMap, scalar.UintHex)
if p, ok := mtpi.ProgramMap[int(pid)]; ok {
d.FieldValueUint("program", uint64(p.Number), scalar.UintHex)
} else if s, ok := mtpi.StreamMap[int(pid)]; ok {
if p, ok := mtpi.ProgramMap[s.ProgramPid]; ok {
d.FieldValueUint("program", uint64(p.Number), scalar.UintHex)
}
d.FieldValueUint("stream_type", uint64(s.Type), tsStreamTypeMap)
}
mtpo.Pid = int(pid)
mtpo.TransportScramblingControl = int(d.FieldU2("transport_scrambling_control", scalar.UintMapSymStr{
0b00: "not_scrambled",
0b01: "reserved",
0b10: "even_key",
0b11: "odd_key",
}))
adaptationFieldControl := d.FieldU2("adaptation_field_control", scalar.UintMapSymStr{
0b00: "reserved",
adaptationFieldControlPayloadOnly: "payload_only",
adaptationFieldControlAdaptationFieldOnly: "adaptation_field_only",
adaptationFieldControlAdaptationFieldAndPayload: "adaptation_and_payload",
})
mtpo.ContinuityCounter = int(d.FieldU4("continuity_counter"))
switch adaptationFieldControl {
case adaptationFieldControlAdaptationFieldOnly,
adaptationFieldControlAdaptationFieldAndPayload:
d.FieldStruct("adaptation_field", func(d *decode.D) {
length := d.FieldU8("length") // Number of bytes in the adaptation field immediately following this byte
d.FramedFn(int64(length)*8, func(d *decode.D) {
d.FieldBool("discontinuity_indicator") // Set if current TS packet is in a discontinuity state with respect to either the continuity counter or the program clock reference
d.FieldBool("random_access_indicator") // Set when the stream may be decoded without errors from this point
d.FieldBool("elementary_stream_priority_indicator") // Set when this stream should be considered "high priority"
pcrPresent := d.FieldBool("pcr_present") // Set when PCR field is present
opcrPresent := d.FieldBool("opcr_present") // Set when OPCR field is present
splicingPointPresent := d.FieldBool("splicing_point_present") // Set when splice countdown field is present
transportPrivatePresent := d.FieldBool("transport_private_present") // Set when transport private data is present
adaptationFieldExtensionPresent := d.FieldBool("adaptation_field_extension_present") // Set when adaptation extension data is present
if pcrPresent {
d.FieldU("pcr", 48)
}
if opcrPresent {
d.FieldU("opcr", 48)
}
if splicingPointPresent {
d.FieldU8("splicing_point")
}
if transportPrivatePresent {
d.FieldStruct("transport_private", func(d *decode.D) {
length := d.FieldU8("length")
d.FieldRawLen("data", int64(length)*8)
})
}
if adaptationFieldExtensionPresent {
d.FieldStruct("adaptation_extension", func(d *decode.D) {
length := d.FieldU8("length")
d.FramedFn(int64(length)*8, func(d *decode.D) {
d.FieldBool("legal_time_window")
d.FieldBool("piecewise_rate")
d.FieldBool("seamless_splice")
d.FieldU5("reserved", scalar.UintHex)
d.FieldRawLen("data", d.BitsLeft())
})
})
// Optional fields
// LTW flag set (2 bytes)
// LTW valid flag 1 0x8000
// LTW offset 15 0x7fff Extra information for rebroadcasters to determine the state of buffers when packets may be missing.
// Piecewise flag set (3 bytes)
// Reserved 2 0xc00000
// Piecewise rate 22 0x3fffff The rate of the stream, measured in 188-byte packets, to define the end-time of the LTW.
// Seamless splice flag set (5 bytes)
// Splice type 4 0xf000000000 Indicates the parameters of the H.262 splice.
// DTS next access unit 36 0x0efffefffe The PES DTS of the splice point. Split up as multiple fields, 1 marker bit (0x1), 15 bits, 1 marker bit, 15 bits, and 1 marker bit, for 33 data bits total.
}
if d.BitsLeft() > 0 {
d.FieldRawLen("stuffing", d.BitsLeft())
}
})
})
}
isTable := tsPidIsTable(mtpo.Pid, mtpi.ProgramMap)
if isTable {
var payloadPointer uint64
if mtpo.PayloadUnitStart {
payloadPointer = d.FieldU8("payload_pointer")
}
if payloadPointer > 0 {
d.FieldRawLen("stuffing", int64(payloadPointer)*8)
}
}
switch adaptationFieldControl {
case adaptationFieldControlPayloadOnly,
adaptationFieldControlAdaptationFieldAndPayload:
payload := d.FieldRawLen("payload", d.BitsLeft())
mtpo.Payload = d.ReadAllBits(payload)
default:
// TODO: unknown adaption control flags
d.FieldRawLen("unknown", d.BitsLeft())
}
})
return mtpo
}

59
format/mpeg/mpeg_ts_pat.go Normal file
View File

@ -0,0 +1,59 @@
package mpeg
import (
"github.com/wader/fq/format"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
"github.com/wader/fq/pkg/scalar"
)
func init() {
interp.RegisterFormat(
format.MPEG_TS_PAT,
&decode.Format{
Description: "MPEG TS Program Association Table",
Groups: []*decode.Group{format.Probe},
DecodeFn: mpegTsPatDecode,
})
}
func mpegTsPatDecode(d *decode.D) any {
mtpo := format.MpegTsPatOut{
PidMap: map[int]int{},
}
d.FieldU8("table_id")
d.FieldU1("syntax_indicator")
d.FieldU3("reserved0", scalar.UintHex)
length := d.FieldU12("section_length")
d.FramedFn(int64(length-4)*8, func(d *decode.D) {
d.FieldU16("transport_stream_id")
d.FieldU2("reserved1", scalar.UintHex)
d.FieldU5("version_number") // TODO: output?
d.FieldU1("current_next_indicator")
d.FieldU8("section_number")
d.FieldU8("last_section_number")
d.FieldArray("programs", func(d *decode.D) {
for !d.End() {
d.FieldStruct("program", func(d *decode.D) {
programNumber := d.FieldU16("program_number")
d.FieldU3("reserved", scalar.UintHex)
switch programNumber {
case 0:
d.FieldU13("network_pid", scalar.UintHex)
default:
programPid := d.FieldU13("program_map_pid", scalar.UintHex)
mtpo.PidMap[int(programPid)] = int(programNumber)
}
})
}
})
})
// TODO: move
d.FieldU32("crc", scalar.UintHex)
if d.BitsLeft() > 0 {
d.FieldRawLen("stuffing", d.BitsLeft())
}
return mtpo
}

76
format/mpeg/mpeg_ts_pmt.go Normal file
View File

@ -0,0 +1,76 @@
package mpeg
import (
"github.com/wader/fq/format"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
"github.com/wader/fq/pkg/scalar"
)
func init() {
interp.RegisterFormat(
format.MPEG_TS_PMT,
&decode.Format{
Description: "MPEG TS Program Map Table",
Groups: []*decode.Group{format.Probe},
DecodeFn: mpegTsPmtDecode,
})
}
func mpegTsPmtDecode(d *decode.D) any {
mtpo := format.MpegTsPmtOut{
Streams: map[int]format.MpegTsStream{},
}
d.FieldU8("table_id")
d.FieldU1("syntax_indicator")
d.FieldU3("reserved0", scalar.UintHex)
length := d.FieldU12("section_length")
d.FramedFn(int64(length-4)*8, func(d *decode.D) {
d.FieldU16("program_number")
d.FieldU2("reserved1", scalar.UintHex)
d.FieldU5("version_number")
d.FieldU1("current_next_indicator")
d.FieldU8("section_number")
d.FieldU8("last_section_number")
d.FieldU3("reserved2", scalar.UintHex)
d.FieldU13("pcr_pid")
d.FieldU4("reserved3", scalar.UintHex)
programInfoLength := d.FieldU12("program_info_length")
d.FramedFn(int64(programInfoLength)*8, func(d *decode.D) {
d.FieldArray("decriptors", func(d *decode.D) {
for !d.End() {
d.FieldStruct("decriptor", func(d *decode.D) {
d.FieldU8("tag", tsStreamTagMap, scalar.UintHex)
length := d.FieldU8("length")
// TODO:
d.FieldRawLen("data", int64(length)*8)
})
}
})
})
d.FieldArray("streams", func(d *decode.D) {
for !d.End() {
d.FieldStruct("stream", func(d *decode.D) {
streamType := d.FieldU8("stream_type", scalar.UintHex, tsStreamTypeMap)
d.FieldU3("reserved0", scalar.UintHex)
streamPid := d.FieldU13("elementary_pid", scalar.UintHex)
d.FieldU4("reserved1", scalar.UintHex)
length := d.FieldU12("es_info_length")
d.FramedFn(int64(length)*8, func(d *decode.D) {
// TODO:
d.FieldRawLen("data", d.BitsLeft())
})
mtpo.Streams[int(streamPid)] = format.MpegTsStream{Type: int(streamType)}
})
}
})
})
d.FieldU32("crc", scalar.UintHex)
if d.BitsLeft() > 0 {
d.FieldRawLen("stuffing", d.BitsLeft())
}
return mtpo
}

57
format/mpeg/mpeg_ts_sdt.go Normal file
View File

@ -0,0 +1,57 @@
package mpeg
import (
"github.com/wader/fq/format"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
"github.com/wader/fq/pkg/scalar"
)
func init() {
interp.RegisterFormat(
format.MPEG_TS_SDT,
&decode.Format{
Description: "MPEG TS Service Description Table",
DecodeFn: mpegTsSdtDecode,
})
}
func mpegTsSdtDecode(d *decode.D) any {
d.FieldU8("table_id", tsTableMap, scalar.UintHex)
d.FieldU1("syntax_indicator")
d.FieldU3("reserved0", scalar.UintHex)
length := d.FieldU12("section_length")
d.FramedFn(int64(length-4)*8, func(d *decode.D) {
d.FieldU16("transport_stream_id")
d.FieldU2("reserved1", scalar.UintHex)
d.FieldU5("version_number")
d.FieldU1("current_next_indicator")
d.FieldU8("section_number")
d.FieldU8("last_section_number")
d.FieldU16("original_network_id", scalar.UintHex)
d.FieldU8("reserved3", scalar.UintHex)
d.FieldArray("services", func(d *decode.D) {
for !d.End() {
d.FieldStruct("stream", func(d *decode.D) {
d.FieldU16("service_id")
d.FieldU6("reserved0", scalar.UintHex)
d.FieldBool("eit_schedule_flag")
d.FieldBool("present_following_flag")
d.FieldU3("running_status")
d.FieldBool("free_ca_mode")
descriptorsLoopLength := d.FieldU12("descriptors_loop_length")
d.FramedFn(int64(descriptorsLoopLength)*8, func(d *decode.D) {
// TODO:
d.FieldRawLen("descriptor", d.BitsLeft())
})
})
}
})
})
d.FieldU32("crc", scalar.UintHex)
if d.BitsLeft() > 0 {
d.FieldRawLen("stuffing", d.BitsLeft())
}
return nil
}

148
format/mpeg/shared.go
View File

@ -4,9 +4,11 @@ import (
"bytes"
"io"
"github.com/wader/fq/format"
"github.com/wader/fq/pkg/bitio"
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
"github.com/wader/fq/pkg/scalar"
)
func init() {
@ -111,3 +113,149 @@ func (r nalUnescapeReader) Read(p []byte) (n int, err error) {
return n, err
}
const tsPacketLength = 188 * 8
const (
pidPAT = 0
)
func tsPidIsTable(pid int, pmt map[int]format.MpegTsProgram) bool {
// pid 0x0-0x1f seems to all be tables
if pid >= 0 && pid <= 0x1f {
return true
}
_, isPMT := pmt[pid]
return isPMT
}
var tsStreamTagMap = scalar.UintRangeToScalar{
{Range: [2]uint64{0, 0}, S: scalar.Uint{Description: "Reserved"}},
{Range: [2]uint64{1, 1}, S: scalar.Uint{Description: "Reserved"}},
{Range: [2]uint64{2, 2}, S: scalar.Uint{Description: "video_stream_descriptor"}},
{Range: [2]uint64{3, 3}, S: scalar.Uint{Description: "audio_stream_descriptor"}},
{Range: [2]uint64{4, 4}, S: scalar.Uint{Description: "hierarchy_descriptor"}},
{Range: [2]uint64{5, 5}, S: scalar.Uint{Description: "registration_descriptor"}},
{Range: [2]uint64{6, 6}, S: scalar.Uint{Description: "data_stream_alignment_descriptor"}},
{Range: [2]uint64{7, 7}, S: scalar.Uint{Description: "target_background_grid_descriptor"}},
{Range: [2]uint64{8, 8}, S: scalar.Uint{Description: "video_window_descriptor"}},
{Range: [2]uint64{9, 9}, S: scalar.Uint{Description: "CA_descriptor"}},
{Range: [2]uint64{10, 10}, S: scalar.Uint{Description: "ISO_639_language_descriptor"}},
{Range: [2]uint64{11, 11}, S: scalar.Uint{Description: "system_clock_descriptor"}},
{Range: [2]uint64{12, 12}, S: scalar.Uint{Description: "multiplex_buffer_utilization_descriptor"}},
{Range: [2]uint64{13, 13}, S: scalar.Uint{Description: "copyright_descriptor"}},
{Range: [2]uint64{14, 14}, S: scalar.Uint{Description: "maximum_bitrate_descriptor"}},
{Range: [2]uint64{15, 15}, S: scalar.Uint{Description: "private_data_indicator_descriptor"}},
{Range: [2]uint64{16, 16}, S: scalar.Uint{Description: "smoothing_buffer_descriptor"}},
{Range: [2]uint64{17, 17}, S: scalar.Uint{Description: "STD_descriptor"}},
{Range: [2]uint64{18, 18}, S: scalar.Uint{Description: "IBP_descriptor"}},
{Range: [2]uint64{19, 26}, S: scalar.Uint{Description: "Defined in ISO/IEC 13818-6"}},
{Range: [2]uint64{27, 27}, S: scalar.Uint{Description: "MPEG-4_video_descriptor"}},
{Range: [2]uint64{28, 28}, S: scalar.Uint{Description: "MPEG-4_audio_descriptor"}},
{Range: [2]uint64{29, 29}, S: scalar.Uint{Description: "IOD_descriptor"}},
{Range: [2]uint64{30, 30}, S: scalar.Uint{Description: "SL_descriptor"}},
{Range: [2]uint64{31, 31}, S: scalar.Uint{Description: "FMC_descriptor"}},
{Range: [2]uint64{32, 32}, S: scalar.Uint{Description: "external_ES_ID_descriptor"}},
{Range: [2]uint64{33, 33}, S: scalar.Uint{Description: "MuxCode_descriptor"}},
{Range: [2]uint64{34, 34}, S: scalar.Uint{Description: "FmxBufferSize_descriptor"}},
{Range: [2]uint64{35, 35}, S: scalar.Uint{Description: "multiplexbuffer_descriptor"}},
{Range: [2]uint64{36, 36}, S: scalar.Uint{Description: "content_labeling_descriptor"}},
{Range: [2]uint64{37, 37}, S: scalar.Uint{Description: "metadata_pointer_descriptor"}},
{Range: [2]uint64{38, 38}, S: scalar.Uint{Description: "metadata_descriptor"}},
{Range: [2]uint64{39, 39}, S: scalar.Uint{Description: "metadata_STD_descriptor"}},
{Range: [2]uint64{40, 40}, S: scalar.Uint{Description: "AVC video descriptor"}},
{Range: [2]uint64{41, 41}, S: scalar.Uint{Description: "IPMP_descriptor (defined in ISO/IEC 13818-11, MPEG-2 IPMP)"}},
{Range: [2]uint64{42, 42}, S: scalar.Uint{Description: "AVC timing and HRD descriptor"}},
{Range: [2]uint64{43, 43}, S: scalar.Uint{Description: "MPEG-2_AAC_audio_descriptor"}},
{Range: [2]uint64{44, 44}, S: scalar.Uint{Description: "FlexMuxTiming_descriptor"}},
{Range: [2]uint64{45, 63}, S: scalar.Uint{Description: "ITU-T Rec. H.222.0 | ISO/IEC 13818-1 Reserved"}},
{Range: [2]uint64{64, 255}, S: scalar.Uint{Description: "User Private"}},
}
var tsStreamTypeMap = scalar.UintRangeToScalar{
{Range: [2]uint64{0x00, 0x00}, S: scalar.Uint{Description: "Reserved"}},
{Range: [2]uint64{0x01, 0x01}, S: scalar.Uint{Sym: "video", Description: "ISO/IEC 11172-2 Video"}}, // TODO: video_mpeg? codec?
{Range: [2]uint64{0x02, 0x02}, S: scalar.Uint{Description: "ISO/IEC 13818-2 or ISO/IEC 11172-2"}},
{Range: [2]uint64{0x03, 0x03}, S: scalar.Uint{Sym: "audio_mpeg1", Description: "ISO/IEC 11172-3 Audio"}},
{Range: [2]uint64{0x04, 0x04}, S: scalar.Uint{Sym: "audio_mpeg2", Description: "ISO/IEC 13818-3 Audio"}},
{Range: [2]uint64{0x05, 0x05}, S: scalar.Uint{Description: "ISO/IEC 13818-1 private_sections"}},
{Range: [2]uint64{0x06, 0x06}, S: scalar.Uint{Description: "ISO/IEC 13818-1 PES packets containing private data"}},
{Range: [2]uint64{0x07, 0x07}, S: scalar.Uint{Description: "ISO/IEC 13522 MHEG"}},
{Range: [2]uint64{0x08, 0x08}, S: scalar.Uint{Description: "ISO/IEC 13818-1 Annex A DSM-CC"}},
{Range: [2]uint64{0x09, 0x09}, S: scalar.Uint{Description: "ITU-T Rec. H.222.1"}},
{Range: [2]uint64{0x0a, 0x0a}, S: scalar.Uint{Description: "ISO/IEC 13818-6 type A"}},
{Range: [2]uint64{0x0b, 0x0b}, S: scalar.Uint{Description: "ISO/IEC 13818-6 type B"}},
{Range: [2]uint64{0x0c, 0x0c}, S: scalar.Uint{Description: "ISO/IEC 13818-6 type C"}},
{Range: [2]uint64{0x0d, 0x0d}, S: scalar.Uint{Description: "ISO/IEC 13818-6 type D"}},
{Range: [2]uint64{0x0e, 0x0e}, S: scalar.Uint{Description: "ISO/IEC 13818-1 auxiliary"}},
{Range: [2]uint64{0x0f, 0x0f}, S: scalar.Uint{Sym: "audio_adts", Description: "ISO/IEC 13818-7 Audio with ADTS transport syntax"}},
{Range: [2]uint64{0x10, 0x10}, S: scalar.Uint{Description: "ISO/IEC 14496-2 Visual"}},
{Range: [2]uint64{0x11, 0x11}, S: scalar.Uint{Sym: "audio_latm", Description: "ISO/IEC 14496-3 Audio with the LATM"}},
{Range: [2]uint64{0x12, 0x12}, S: scalar.Uint{Description: "ISO/IEC 14496-1 SL-packetized stream or FlexMux stream carried in PES packets"}},
{Range: [2]uint64{0x13, 0x13}, S: scalar.Uint{Description: "ISO/IEC 14496-1 SL-packetized stream or FlexMux stream carried in ISO/IEC 14496_sections"}},
{Range: [2]uint64{0x14, 0x14}, S: scalar.Uint{Description: "ISO/IEC 13818-6 Synchronized Download Protocol"}},
{Range: [2]uint64{0x15, 0x15}, S: scalar.Uint{Description: "Metadata carried in PES packets"}},
{Range: [2]uint64{0x16, 0x16}, S: scalar.Uint{Description: "Metadata carried in metadata_sections"}},
{Range: [2]uint64{0x17, 0x17}, S: scalar.Uint{Description: "Metadata carried in ISO/IEC 13818-6 Data Carousel"}},
{Range: [2]uint64{0x18, 0x18}, S: scalar.Uint{Description: "Metadata carried in ISO/IEC 13818-6 Object Carousel"}},
{Range: [2]uint64{0x19, 0x19}, S: scalar.Uint{Description: "Metadata carried in ISO/IEC 13818-6 Synchronized Download Protocol"}},
{Range: [2]uint64{0x1a, 0x1a}, S: scalar.Uint{Description: "IPMP stream (defined in ISO/IEC 13818-11, MPEG-2 IPMP)"}},
{Range: [2]uint64{0x1b, 0x1b}, S: scalar.Uint{Sym: "video_avc", Description: "AVC video stream as defined in ITU-T Rec. H.264 | ISO/IEC 14496-10 Video"}},
{Range: [2]uint64{0x1c, 0x7e}, S: scalar.Uint{Description: "ITU-T Rec. H.222.0 | ISO/IEC 13818-1 Reserved"}},
{Range: [2]uint64{0x7f, 0x7f}, S: scalar.Uint{Description: "IPMP stream"}},
{Range: [2]uint64{0x80, 0xff}, S: scalar.Uint{Description: "User Private"}},
}
var tsPidMap = scalar.UintRangeToScalar{
{Range: [2]uint64{pidPAT, pidPAT}, S: scalar.Uint{Sym: "pat", Description: "Program association table"}},
{Range: [2]uint64{0x0001, 0x0001}, S: scalar.Uint{Sym: "cat", Description: "Conditional access table"}},
{Range: [2]uint64{0x0002, 0x0002}, S: scalar.Uint{Description: "Transport stream description table"}},
{Range: [2]uint64{0x0003, 0x0003}, S: scalar.Uint{Description: "IPMP control information table"}},
{Range: [2]uint64{0x0004, 0x000f}, S: scalar.Uint{Description: "Reserved for future use"}},
{Range: [2]uint64{0x0010, 0x001f}, S: scalar.Uint{Description: "DVB metadata"}},
{Range: [2]uint64{0x0010, 0x0010}, S: scalar.Uint{Sym: "nit", Description: "NIT, ST"}},
{Range: [2]uint64{0x0011, 0x0011}, S: scalar.Uint{Sym: "sdt", Description: "SDT, BAT, ST"}},
{Range: [2]uint64{0x0012, 0x0012}, S: scalar.Uint{Sym: "eit", Description: "EIT, ST, CIT"}},
{Range: [2]uint64{0x0013, 0x0013}, S: scalar.Uint{Sym: "rst", Description: "RST, ST"}},
{Range: [2]uint64{0x0014, 0x0014}, S: scalar.Uint{Sym: "tdt", Description: "TDT, TOT, ST"}},
{Range: [2]uint64{0x0015, 0x0015}, S: scalar.Uint{Description: "Network synchronization"}},
{Range: [2]uint64{0x0016, 0x0016}, S: scalar.Uint{Sym: "rnt", Description: "RNT"}},
{Range: [2]uint64{0x0017, 0x001b}, S: scalar.Uint{Description: "Reserved for future use"}},
{Range: [2]uint64{0x001c, 0x001c}, S: scalar.Uint{Description: "Inband signalling"}},
{Range: [2]uint64{0x001d, 0x001d}, S: scalar.Uint{Description: "Measurement"}},
{Range: [2]uint64{0x001e, 0x001e}, S: scalar.Uint{Sym: "dit", Description: "DIT"}},
{Range: [2]uint64{0x001f, 0x001f}, S: scalar.Uint{Sym: "sit", Description: "SIT"}},
{Range: [2]uint64{0x0020, 0x1ffa}, S: scalar.Uint{Description: "Program maps, elementary streams and data"}},
{Range: [2]uint64{0x1ffb, 0x1ffb}, S: scalar.Uint{Description: "DigiCipher 2/ATSC MGT metadata"}},
{Range: [2]uint64{0x1ffc, 0x1ffe}, S: scalar.Uint{Description: "Program association table assigned"}},
{Range: [2]uint64{0x1fff, 0x1fff}, S: scalar.Uint{Description: "Null packet (padding)"}},
}
var tsTableMap = scalar.UintRangeToScalar{
{Range: [2]uint64{0x00, 0x00}, S: scalar.Uint{Description: "program_association_section"}},
{Range: [2]uint64{0x01, 0x01}, S: scalar.Uint{Description: "conditional_access_section"}},
{Range: [2]uint64{0x02, 0x02}, S: scalar.Uint{Description: "program_map_section"}},
{Range: [2]uint64{0x03, 0x03}, S: scalar.Uint{Description: "transport_stream_description_section"}},
{Range: [2]uint64{0x04, 0x3f}, S: scalar.Uint{Description: "reserved"}},
{Range: [2]uint64{0x40, 0x40}, S: scalar.Uint{Description: "network_information_section - actual_network"}},
{Range: [2]uint64{0x41, 0x41}, S: scalar.Uint{Description: "network_information_section - other_network"}},
{Range: [2]uint64{0x42, 0x42}, S: scalar.Uint{Sym: "sdt", Description: "service_description_section - actual_transport_stream"}},
{Range: [2]uint64{0x43, 0x45}, S: scalar.Uint{Description: "reserved for future use"}},
{Range: [2]uint64{0x46, 0x46}, S: scalar.Uint{Description: "service_description_section - other_transport_stream"}},
{Range: [2]uint64{0x47, 0x47}, S: scalar.Uint{Description: "to 0x49 reserved for future use"}},
{Range: [2]uint64{0x4a, 0x4a}, S: scalar.Uint{Description: "bouquet_association_section"}},
{Range: [2]uint64{0x4b, 0x4d}, S: scalar.Uint{Description: "reserved for future use"}},
{Range: [2]uint64{0x4e, 0x4e}, S: scalar.Uint{Description: "event_information_section - actual_transport_stream, present/following"}},
{Range: [2]uint64{0x4f, 0x4f}, S: scalar.Uint{Description: "event_information_section - other_transport_stream, present/following"}},
{Range: [2]uint64{0x50, 0x5f}, S: scalar.Uint{Description: "event_information_section - actual_transport_stream, schedule"}},
{Range: [2]uint64{0x60, 0x6f}, S: scalar.Uint{Description: "event_information_section - other_transport_stream, schedule"}},
{Range: [2]uint64{0x70, 0x70}, S: scalar.Uint{Description: "time_date_section"}},
{Range: [2]uint64{0x71, 0x71}, S: scalar.Uint{Description: "running_status_section"}},
{Range: [2]uint64{0x72, 0x72}, S: scalar.Uint{Description: "stuffing_section"}},
{Range: [2]uint64{0x73, 0x73}, S: scalar.Uint{Description: "time_offset_section"}},
{Range: [2]uint64{0x74, 0x7d}, S: scalar.Uint{Description: "reserved for future use"}},
{Range: [2]uint64{0x7e, 0x7e}, S: scalar.Uint{Description: "discontinuity_information_section"}},
{Range: [2]uint64{0x7f, 0x7f}, S: scalar.Uint{Description: "selection_information_section"}},
{Range: [2]uint64{0x80, 0xfe}, S: scalar.Uint{Description: "user defined"}},
{Range: [2]uint64{0xff, 0xff}, S: scalar.Uint{Description: "reserved"}},
}

8
pkg/scalar/scalar.go
View File

@ -140,13 +140,13 @@ func StrSymParseFloat(base int) StrMapper {
return strMapToSym(func(s string) (any, error) { return strconv.ParseFloat(s, base) }, false)
}
type URangeEntry struct {
type UintRangeEntry struct {
Range [2]uint64
S Uint
}
// UintRangeToScalar maps uint64 ranges to a scalar, first in range is chosen
type UintRangeToScalar []URangeEntry
type UintRangeToScalar []UintRangeEntry
func (rs UintRangeToScalar) MapUint(s Uint) (Uint, error) {
n := s.Actual
@ -162,13 +162,13 @@ func (rs UintRangeToScalar) MapUint(s Uint) (Uint, error) {
}
// SRangeToScalar maps ranges to a scalar, first in range is chosen
type SRangeEntry struct {
type SintRangeEntry struct {
Range [2]int64
S Sint
}
// SRangeToScalar maps sint64 ranges to a scalar, first in range is chosen
type SRangeToScalar []SRangeEntry
type SRangeToScalar []SintRangeEntry
func (rs SRangeToScalar) MapSint(s Sint) (Sint, error) {
n := s.Actual