From 3b71d184b429938fd3d29519643906a432567ad5 Mon Sep 17 00:00:00 2001 From: Mattias Wadman Date: Wed, 20 Dec 2023 11:36:25 +0100 Subject: [PATCH] wip --- format/all/all.go | 1 + format/format.go | 3 +- format/woff/woff2.go | 288 +++++++++++++++++++++++++++++++++++++++++++ go.mod | 2 + go.sum | 8 ++ 5 files changed, 301 insertions(+), 1 deletion(-) create mode 100644 format/woff/woff2.go diff --git a/format/all/all.go b/format/all/all.go index 1f820865..195d29e1 100644 --- a/format/all/all.go +++ b/format/all/all.go @@ -59,6 +59,7 @@ import ( _ "github.com/wader/fq/format/vorbis" _ "github.com/wader/fq/format/vpx" _ "github.com/wader/fq/format/wasm" + _ "github.com/wader/fq/format/woff" _ "github.com/wader/fq/format/xml" _ "github.com/wader/fq/format/yaml" _ "github.com/wader/fq/format/zip" diff --git a/format/format.go b/format/format.go index 4df45e97..f6208ca0 100644 --- a/format/format.go +++ b/format/format.go @@ -141,10 +141,10 @@ var ( MP4 = &decode.Group{Name: "mp4"} MPEG_ASC = &decode.Group{Name: "mpeg_asc"} MPEG_ES = &decode.Group{Name: "mpeg_es"} - MPES_PES = &decode.Group{Name: "mpeg_pes"} MPEG_PES_Packet = &decode.Group{Name: "mpeg_pes_packet"} MPEG_SPU = &decode.Group{Name: "mpeg_spu"} MPEG_TS = &decode.Group{Name: "mpeg_ts"} + MPES_PES = &decode.Group{Name: "mpeg_pes"} MsgPack = &decode.Group{Name: "msgpack"} Ogg = &decode.Group{Name: "ogg"} Ogg_Page = &decode.Group{Name: "ogg_page"} @@ -178,6 +178,7 @@ var ( WASM = &decode.Group{Name: "wasm"} WAV = &decode.Group{Name: "wav"} WebP = &decode.Group{Name: "webp"} + WOFF2 = &decode.Group{Name: "woff2"} XML = &decode.Group{Name: "xml"} YAML = &decode.Group{Name: "yaml"} Zip = &decode.Group{Name: "zip"} diff --git a/format/woff/woff2.go b/format/woff/woff2.go new file mode 100644 index 00000000..06c99937 --- /dev/null +++ b/format/woff/woff2.go @@ -0,0 +1,288 @@ +package woff + +// OpenType https://learn.microsoft.com/en-us/typography/opentype/ + +import ( + "bytes" + "io" + "time" + + "github.com/dsnet/compress/brotli" + "github.com/wader/fq/format" + "github.com/wader/fq/pkg/bitio" + "github.com/wader/fq/pkg/decode" + "github.com/wader/fq/pkg/interp" + "github.com/wader/fq/pkg/scalar" +) + +func init() { + interp.RegisterFormat( + format.WOFF2, + &decode.Format{ + Description: "Web Open Font Format version 2", + Groups: []*decode.Group{format.Probe}, + DecodeFn: woff2Decode, + }) +} + +var opentypeEpochDate = time.Date(1904, time.January, 1, 0, 0, 0, 0, time.UTC) + +// WOFF2 1.3 UIntBase128 Data Type +func decodeUIntBase128(d *decode.D) uint64 { + var accum uint32 + + for i := 0; i < 5; i++ { + dataByte := uint8(d.U8()) + + if i == 0 && dataByte == 0x80 { + d.Fatalf("no leading 0") + } + if accum&0xfe_00_00_00 != 0 { + d.Fatalf("overflow") + } + + accum = (accum << 7) | uint32(dataByte&0x7f) + + if dataByte&0x80 == 0 { + return uint64(accum) + } + } + + d.Fatalf("exceeds 5 bytes") + + return 0 +} + +var knownTags = scalar.UintMapSymStr{ + 0: "cmap", + 1: "head", + 2: "hhea", + 3: "hmtx", + 4: "maxp", + 5: "name", + 6: "OS/2", + 7: "post", + 8: "cvt", + 9: "fpgm", + 10: "glyf", + 11: "loca", + 12: "prep", + 13: "CFF", + 14: "VORG", + 15: "EBDT", + 16: "EBLC", + 17: "gasp", + 18: "hdmx", + 19: "kern", + 20: "LTSH", + 21: "PCLT", + 22: "VDMX", + 23: "vhea", + 24: "vmtx", + 25: "BASE", + 26: "GDEF", + 27: "GPOS", + 28: "GSUB", + 29: "EBSC", + 30: "JSTF", + 31: "MATH", + 32: "CBDT", + 33: "CBLC", + 34: "COLR", + 35: "CPAL", + 36: "SVG", + 37: "sbix", + 38: "acnt", + 39: "avar", + 40: "bdat", + 41: "bloc", + 42: "bsln", + 43: "cvar", + 44: "fdsc", + 45: "feat", + 46: "fmtx", + 47: "fvar", + 48: "gvar", + 49: "hsty", + 50: "just", + 51: "lcar", + 52: "mort", + 53: "morx", + 54: "opbd", + 55: "prop", + 56: "trak", + 57: "Zapf", + 58: "Silf", + 59: "Glat", + 60: "Gloc", + 61: "Feat", + 62: "Sill", +} + +const tagGlyf = 10 +const tagLoca = 11 + +const flavorTTCF = 0x74746366 + +func woff2Decode(d *decode.D) any { + d.FieldUTF8("signature", 4, d.StrAssert("wOF2")) + d.FieldU32("flavor", scalar.UintMapSymStr{ + flavorTTCF: "collection", + }) + d.FieldU32("length") + numTables := d.FieldU16("num_tables") + d.FieldU16("reserved") + d.FieldU32("total_sfnt_size") + totalCompressSize := d.FieldU32("total_compressed_size") + d.FieldU16("major_version") + d.FieldU16("minor_version") + d.FieldU32("meta_offset") + d.FieldU32("meta_length") + d.FieldU32("meta_orig_length") + d.FieldU32("priv_offset") + d.FieldU32("priv_length") + + type tableEntry struct { + d *decode.D + tag string + transformationVersion uint64 + dataLen int64 + } + + var tables []tableEntry + + d.FieldArray("tables", func(d *decode.D) { + for i := uint64(0); i < numTables; i++ { + d.FieldStruct("entry", func(d *decode.D) { + transformationVersion := d.FieldU2("transformation_version") + knownTag := d.FieldU6("known_tag", knownTags) + var tag string + if knownTag < 63 { + tag = knownTags[knownTag] + } else { + tag = d.FieldUTF8("optional_tag", 4) + } + d.FieldValueStr("tag", tag) + dataLen := d.FieldUintFn("orig_length", decodeUIntBase128) + + // For all tables in a font, except for 'glyf' and 'loca' tables, transformation version 0 indicates the null transform ... + // For 'glyf' and 'loca' tables, transformation version 3 indicates the null transform ... + glyfOrLoca := knownTag == tagGlyf || knownTag == tagLoca + hasNullTransform := + (glyfOrLoca && transformationVersion == 0) || + (glyfOrLoca && transformationVersion == 3) + + if hasNullTransform { + dataLen = d.FieldUintFn("transform_length", decodeUIntBase128) + } + + tables = append(tables, tableEntry{ + d: d, + tag: tag, + transformationVersion: transformationVersion, + dataLen: int64(dataLen), + }) + }) + } + }) + + // TODO: CollectionDirectory + + r := d.FieldRawLen("compressed", int64(totalCompressSize)*8) + br, err := brotli.NewReader(bitio.NewIOReader(r), &brotli.ReaderConfig{}) + if err != nil { + d.IOPanic(err, "brotli.NewReader") + } + brBuf := &bytes.Buffer{} + _, err = io.Copy(brBuf, br) + if err != nil { + d.IOPanic(err, "brotli io.Copy") + } + + left := brBuf.Bytes() + for _, te := range tables { + if len(left) < int(te.dataLen) { + d.Fatalf("orig_len outside buffer") + } + + data := bitio.NewBitReader(left[0:te.dataLen], -1) + + // TODO: move to own decoder? + switch te.tag { + case "name": + // https://learn.microsoft.com/en-us/typography/opentype/spec/head + te.d.FieldStructRootBitBufFn("data", data, func(d *decode.D) { + version := d.FieldU16("version") + count := d.FieldU16("count") + storageOffset := d.FieldU16("storage_offset") + + d.FieldArray("records", func(d *decode.D) { + for i := uint64(0); i < count; i++ { + d.FieldStruct("record", func(d *decode.D) { + d.FieldU16("platform_id") + d.FieldU16("encoding_id") + d.FieldU16("language_id") + d.FieldU16("name_id") + length := d.FieldU16("length") + stringOffset := d.FieldU16("string_offset") + d.RangeFn(int64(storageOffset+stringOffset)*8, int64(length)*8, func(d *decode.D) { + d.FieldUTF16BE("value", int(length)) + }) + }) + } + }) + + // TODO: tags? + _ = version + }) + case "head": + // https://learn.microsoft.com/en-us/typography/opentype/spec/head + te.d.FieldStructRootBitBufFn("data", data, func(d *decode.D) { + d.FieldU32("version") + d.FieldU32("font_revision") + d.FieldU32("checksum_adjustment", scalar.UintHex) + d.FieldU32("magic_number", scalar.UintHex) + d.FieldS16("flags") + d.FieldS16("units_per_em") + d.FieldU64("created", scalar.UintActualDateDescription(opentypeEpochDate, time.Second, time.RFC3339)) + d.FieldU64("modified", scalar.UintActualDateDescription(opentypeEpochDate, time.Second, time.RFC3339)) + d.FieldS16("x_min") + d.FieldS16("y_min") + d.FieldS16("x_max") + d.FieldS16("y_max") + d.FieldS16("mac_style") + d.FieldS16("lowest_rec_ppem") + d.FieldS16("font_direction_hint") + d.FieldS16("index_to_loc_format") + // d.FieldS16("glyph_data_format") + }) + case "hhea": + // https://learn.microsoft.com/en-us/typography/opentype/spec/hhea + te.d.FieldStructRootBitBufFn("data", data, func(d *decode.D) { + d.FieldU32("version") + d.FieldS16("ascent") // Distance from baseline of highest ascender + d.FieldS16("descent") // Distance from baseline of lowest descender + d.FieldS16("line_gap") // typographic line gap + d.FieldU16("advance_width_max") // must be consistent with horizontal metrics + d.FieldS16("min_left_side_bearing") // must be consistent with horizontal metrics + d.FieldS16("min_right_side_bearing") // must be consistent with horizontal metrics + d.FieldS16("x_max_extent") // max(lsb + (xMax-xMin)) + d.FieldS16("caret_slope_rise") // used to calculate the slope of the caret (rise/run) set to 1 for vertical caret + d.FieldS16("caret_slope_run") // 0 for vertical + d.FieldS16("caret_offset") // set value to 0 for non-slanted fonts + d.FieldS16("reserved0") // set value to 0 + d.FieldS16("reserved1") // set value to 0 + d.FieldS16("reserved2") // set value to 0 + d.FieldS16("reserved3") // set value to 0 + d.FieldS16("metric_data_format") // 0 for current format + d.FieldU16("num_of_long_hor_metrics") // number of advance widths in metrics table + }) + default: + te.d.FieldRootBitBuf("data", data) + } + + left = left[te.dataLen:] + } + + return nil +} diff --git a/go.mod b/go.mod index dc44ef6e..9665e07a 100644 --- a/go.mod +++ b/go.mod @@ -80,6 +80,8 @@ require ( gopkg.in/yaml.v3 v3.0.1 ) +require github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 + require ( github.com/itchyny/timefmt-go v0.1.5 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect diff --git a/go.sum b/go.sum index 5543118e..91704d56 100644 --- a/go.sum +++ b/go.sum @@ -2,16 +2,22 @@ github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/creasty/defaults v1.7.0 h1:eNdqZvc5B509z18lD8yc212CAqJNvfT1Jq6L8WowdBA= github.com/creasty/defaults v1.7.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= +github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= +github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/ergochat/readline v0.1.0 h1:KEIiAnyH9qGZB4K8oq5mgDcExlEKwmZDcyyocgJiABc= github.com/ergochat/readline v0.1.0/go.mod h1:o3ux9QLHLm77bq7hDB21UTm6HlV2++IPDMfIfKDuOgY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomarkdown/markdown v0.0.0-20231115200524-a660076da3fd h1:PppHBegd3uPZ3Y/Iax/2mlCFJm1w4Qf/zP1MdW4ju2o= github.com/gomarkdown/markdown v0.0.0-20231115200524-a660076da3fd/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/gopacket/gopacket v1.2.0 h1:eXbzFad7f73P1n2EJHQlsKuvIMJjVXK5tXoSca78I3A= github.com/gopacket/gopacket v1.2.0/go.mod h1:BrAKEy5EOGQ76LSqh7DMAr7z0NNPdczWm2GxCG7+I8M= github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE= github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -25,6 +31,7 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/wader/gojq v0.12.1-0.20240118170525-e920352821d6 h1:0zhn+HFzBP6i4XjyR+OMKauJ9zOZROpJCW9D75Z0fRE= github.com/wader/gojq v0.12.1-0.20240118170525-e920352821d6/go.mod h1:E7walEZ03d5WBrEMutC7+tagVBDdtNTDe0jRxMCC6N0= golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= @@ -39,6 +46,7 @@ golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=