wip

format/all/all.go

@@ -59,6 +59,7 @@ import (
 	_ "github.com/wader/fq/format/vorbis"
 	_ "github.com/wader/fq/format/vpx"
 	_ "github.com/wader/fq/format/wasm"
+	_ "github.com/wader/fq/format/woff"
 	_ "github.com/wader/fq/format/xml"
 	_ "github.com/wader/fq/format/yaml"
 	_ "github.com/wader/fq/format/zip"

format/format.go

@@ -141,10 +141,10 @@ var (
 	MP4                 = &decode.Group{Name: "mp4"}
 	MPEG_ASC            = &decode.Group{Name: "mpeg_asc"}
 	MPEG_ES             = &decode.Group{Name: "mpeg_es"}
-	MPES_PES            = &decode.Group{Name: "mpeg_pes"}
 	MPEG_PES_Packet     = &decode.Group{Name: "mpeg_pes_packet"}
 	MPEG_SPU            = &decode.Group{Name: "mpeg_spu"}
 	MPEG_TS             = &decode.Group{Name: "mpeg_ts"}
+	MPES_PES            = &decode.Group{Name: "mpeg_pes"}
 	MsgPack             = &decode.Group{Name: "msgpack"}
 	Ogg                 = &decode.Group{Name: "ogg"}
 	Ogg_Page            = &decode.Group{Name: "ogg_page"}
@@ -178,6 +178,7 @@ var (
 	WASM                = &decode.Group{Name: "wasm"}
 	WAV                 = &decode.Group{Name: "wav"}
 	WebP                = &decode.Group{Name: "webp"}
+	WOFF2               = &decode.Group{Name: "woff2"}
 	XML                 = &decode.Group{Name: "xml"}
 	YAML                = &decode.Group{Name: "yaml"}
 	Zip                 = &decode.Group{Name: "zip"}

format/woff/woff2.go

@@ -0,0 +1,288 @@
+package woff
+
+// OpenType https://learn.microsoft.com/en-us/typography/opentype/
+
+import (
+	"bytes"
+	"io"
+	"time"
+
+	"github.com/dsnet/compress/brotli"
+	"github.com/wader/fq/format"
+	"github.com/wader/fq/pkg/bitio"
+	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/interp"
+	"github.com/wader/fq/pkg/scalar"
+)
+
+func init() {
+	interp.RegisterFormat(
+		format.WOFF2,
+		&decode.Format{
+			Description: "Web Open Font Format version 2",
+			Groups:      []*decode.Group{format.Probe},
+			DecodeFn:    woff2Decode,
+		})
+}
+
+var opentypeEpochDate = time.Date(1904, time.January, 1, 0, 0, 0, 0, time.UTC)
+
+// WOFF2 1.3 UIntBase128 Data Type
+func decodeUIntBase128(d *decode.D) uint64 {
+	var accum uint32
+
+	for i := 0; i < 5; i++ {
+		dataByte := uint8(d.U8())
+
+		if i == 0 && dataByte == 0x80 {
+			d.Fatalf("no leading 0")
+		}
+		if accum&0xfe_00_00_00 != 0 {
+			d.Fatalf("overflow")
+		}
+
+		accum = (accum << 7) | uint32(dataByte&0x7f)
+
+		if dataByte&0x80 == 0 {
+			return uint64(accum)
+		}
+	}
+
+	d.Fatalf("exceeds 5 bytes")
+
+	return 0
+}
+
+var knownTags = scalar.UintMapSymStr{
+	0:  "cmap",
+	1:  "head",
+	2:  "hhea",
+	3:  "hmtx",
+	4:  "maxp",
+	5:  "name",
+	6:  "OS/2",
+	7:  "post",
+	8:  "cvt",
+	9:  "fpgm",
+	10: "glyf",
+	11: "loca",
+	12: "prep",
+	13: "CFF",
+	14: "VORG",
+	15: "EBDT",
+	16: "EBLC",
+	17: "gasp",
+	18: "hdmx",
+	19: "kern",
+	20: "LTSH",
+	21: "PCLT",
+	22: "VDMX",
+	23: "vhea",
+	24: "vmtx",
+	25: "BASE",
+	26: "GDEF",
+	27: "GPOS",
+	28: "GSUB",
+	29: "EBSC",
+	30: "JSTF",
+	31: "MATH",
+	32: "CBDT",
+	33: "CBLC",
+	34: "COLR",
+	35: "CPAL",
+	36: "SVG",
+	37: "sbix",
+	38: "acnt",
+	39: "avar",
+	40: "bdat",
+	41: "bloc",
+	42: "bsln",
+	43: "cvar",
+	44: "fdsc",
+	45: "feat",
+	46: "fmtx",
+	47: "fvar",
+	48: "gvar",
+	49: "hsty",
+	50: "just",
+	51: "lcar",
+	52: "mort",
+	53: "morx",
+	54: "opbd",
+	55: "prop",
+	56: "trak",
+	57: "Zapf",
+	58: "Silf",
+	59: "Glat",
+	60: "Gloc",
+	61: "Feat",
+	62: "Sill",
+}
+
+const tagGlyf = 10
+const tagLoca = 11
+
+const flavorTTCF = 0x74746366
+
+func woff2Decode(d *decode.D) any {
+	d.FieldUTF8("signature", 4, d.StrAssert("wOF2"))
+	d.FieldU32("flavor", scalar.UintMapSymStr{
+		flavorTTCF: "collection",
+	})
+	d.FieldU32("length")
+	numTables := d.FieldU16("num_tables")
+	d.FieldU16("reserved")
+	d.FieldU32("total_sfnt_size")
+	totalCompressSize := d.FieldU32("total_compressed_size")
+	d.FieldU16("major_version")
+	d.FieldU16("minor_version")
+	d.FieldU32("meta_offset")
+	d.FieldU32("meta_length")
+	d.FieldU32("meta_orig_length")
+	d.FieldU32("priv_offset")
+	d.FieldU32("priv_length")
+
+	type tableEntry struct {
+		d                     *decode.D
+		tag                   string
+		transformationVersion uint64
+		dataLen               int64
+	}
+
+	var tables []tableEntry
+
+	d.FieldArray("tables", func(d *decode.D) {
+		for i := uint64(0); i < numTables; i++ {
+			d.FieldStruct("entry", func(d *decode.D) {
+				transformationVersion := d.FieldU2("transformation_version")
+				knownTag := d.FieldU6("known_tag", knownTags)
+				var tag string
+				if knownTag < 63 {
+					tag = knownTags[knownTag]
+				} else {
+					tag = d.FieldUTF8("optional_tag", 4)
+				}
+				d.FieldValueStr("tag", tag)
+				dataLen := d.FieldUintFn("orig_length", decodeUIntBase128)
+
+				// For all tables in a font, except for 'glyf' and 'loca' tables, transformation version 0 indicates the null transform ...
+				// For 'glyf' and 'loca' tables, transformation version 3 indicates the null transform ...
+				glyfOrLoca := knownTag == tagGlyf || knownTag == tagLoca
+				hasNullTransform :=
+					(glyfOrLoca && transformationVersion == 0) ||
+						(glyfOrLoca && transformationVersion == 3)
+
+				if hasNullTransform {
+					dataLen = d.FieldUintFn("transform_length", decodeUIntBase128)
+				}
+
+				tables = append(tables, tableEntry{
+					d:                     d,
+					tag:                   tag,
+					transformationVersion: transformationVersion,
+					dataLen:               int64(dataLen),
+				})
+			})
+		}
+	})
+
+	// TODO: CollectionDirectory
+
+	r := d.FieldRawLen("compressed", int64(totalCompressSize)*8)
+	br, err := brotli.NewReader(bitio.NewIOReader(r), &brotli.ReaderConfig{})
+	if err != nil {
+		d.IOPanic(err, "brotli.NewReader")
+	}
+	brBuf := &bytes.Buffer{}
+	_, err = io.Copy(brBuf, br)
+	if err != nil {
+		d.IOPanic(err, "brotli io.Copy")
+	}
+
+	left := brBuf.Bytes()
+	for _, te := range tables {
+		if len(left) < int(te.dataLen) {
+			d.Fatalf("orig_len outside buffer")
+		}
+
+		data := bitio.NewBitReader(left[0:te.dataLen], -1)
+
+		// TODO: move to own decoder?
+		switch te.tag {
+		case "name":
+			// https://learn.microsoft.com/en-us/typography/opentype/spec/head
+			te.d.FieldStructRootBitBufFn("data", data, func(d *decode.D) {
+				version := d.FieldU16("version")
+				count := d.FieldU16("count")
+				storageOffset := d.FieldU16("storage_offset")
+
+				d.FieldArray("records", func(d *decode.D) {
+					for i := uint64(0); i < count; i++ {
+						d.FieldStruct("record", func(d *decode.D) {
+							d.FieldU16("platform_id")
+							d.FieldU16("encoding_id")
+							d.FieldU16("language_id")
+							d.FieldU16("name_id")
+							length := d.FieldU16("length")
+							stringOffset := d.FieldU16("string_offset")
+							d.RangeFn(int64(storageOffset+stringOffset)*8, int64(length)*8, func(d *decode.D) {
+								d.FieldUTF16BE("value", int(length))
+							})
+						})
+					}
+				})
+
+				// TODO: tags?
+				_ = version
+			})
+		case "head":
+			// https://learn.microsoft.com/en-us/typography/opentype/spec/head
+			te.d.FieldStructRootBitBufFn("data", data, func(d *decode.D) {
+				d.FieldU32("version")
+				d.FieldU32("font_revision")
+				d.FieldU32("checksum_adjustment", scalar.UintHex)
+				d.FieldU32("magic_number", scalar.UintHex)
+				d.FieldS16("flags")
+				d.FieldS16("units_per_em")
+				d.FieldU64("created", scalar.UintActualDateDescription(opentypeEpochDate, time.Second, time.RFC3339))
+				d.FieldU64("modified", scalar.UintActualDateDescription(opentypeEpochDate, time.Second, time.RFC3339))
+				d.FieldS16("x_min")
+				d.FieldS16("y_min")
+				d.FieldS16("x_max")
+				d.FieldS16("y_max")
+				d.FieldS16("mac_style")
+				d.FieldS16("lowest_rec_ppem")
+				d.FieldS16("font_direction_hint")
+				d.FieldS16("index_to_loc_format")
+				// d.FieldS16("glyph_data_format")
+			})
+		case "hhea":
+			// https://learn.microsoft.com/en-us/typography/opentype/spec/hhea
+			te.d.FieldStructRootBitBufFn("data", data, func(d *decode.D) {
+				d.FieldU32("version")
+				d.FieldS16("ascent")                  //	Distance from baseline of highest ascender
+				d.FieldS16("descent")                 //	Distance from baseline of lowest descender
+				d.FieldS16("line_gap")                //	typographic line gap
+				d.FieldU16("advance_width_max")       //	must be consistent with horizontal metrics
+				d.FieldS16("min_left_side_bearing")   //	must be consistent with horizontal metrics
+				d.FieldS16("min_right_side_bearing")  //	must be consistent with horizontal metrics
+				d.FieldS16("x_max_extent")            //	max(lsb + (xMax-xMin))
+				d.FieldS16("caret_slope_rise")        //	used to calculate the slope of the caret (rise/run) set to 1 for vertical caret
+				d.FieldS16("caret_slope_run")         //	0 for vertical
+				d.FieldS16("caret_offset")            //	set value to 0 for non-slanted fonts
+				d.FieldS16("reserved0")               //	set value to 0
+				d.FieldS16("reserved1")               //	set value to 0
+				d.FieldS16("reserved2")               //	set value to 0
+				d.FieldS16("reserved3")               //	set value to 0
+				d.FieldS16("metric_data_format")      //	0 for current format
+				d.FieldU16("num_of_long_hor_metrics") // number of advance widths in metrics table
+			})
+		default:
+			te.d.FieldRootBitBuf("data", data)
+		}
+
+		left = left[te.dataLen:]
+	}
+
+	return nil
+}

go.mod

@@ -80,6 +80,8 @@ require (
 	gopkg.in/yaml.v3 v3.0.1
 )
 
+require github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707
+
 require (
 	github.com/itchyny/timefmt-go v0.1.5 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect

go.sum

@@ -2,16 +2,22 @@ github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8
 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/creasty/defaults v1.7.0 h1:eNdqZvc5B509z18lD8yc212CAqJNvfT1Jq6L8WowdBA=
 github.com/creasty/defaults v1.7.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM=
+github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4=
+github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s=
+github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
 github.com/ergochat/readline v0.1.0 h1:KEIiAnyH9qGZB4K8oq5mgDcExlEKwmZDcyyocgJiABc=
 github.com/ergochat/readline v0.1.0/go.mod h1:o3ux9QLHLm77bq7hDB21UTm6HlV2++IPDMfIfKDuOgY=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gomarkdown/markdown v0.0.0-20231115200524-a660076da3fd h1:PppHBegd3uPZ3Y/Iax/2mlCFJm1w4Qf/zP1MdW4ju2o=
 github.com/gomarkdown/markdown v0.0.0-20231115200524-a660076da3fd/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/gopacket/gopacket v1.2.0 h1:eXbzFad7f73P1n2EJHQlsKuvIMJjVXK5tXoSca78I3A=
 github.com/gopacket/gopacket v1.2.0/go.mod h1:BrAKEy5EOGQ76LSqh7DMAr7z0NNPdczWm2GxCG7+I8M=
 github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE=
 github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8=
+github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
@@ -25,6 +31,7 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/wader/gojq v0.12.1-0.20240118170525-e920352821d6 h1:0zhn+HFzBP6i4XjyR+OMKauJ9zOZROpJCW9D75Z0fRE=
 github.com/wader/gojq v0.12.1-0.20240118170525-e920352821d6/go.mod h1:E7walEZ03d5WBrEMutC7+tagVBDdtNTDe0jRxMCC6N0=
 golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
@@ -39,6 +46,7 @@ golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE=
 golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=