fq/format/bzip2/bzip2.go

package bzip2

// https://en.wikipedia.org/wiki/Bzip2
// https://github.com/dsnet/compress/blob/master/doc/bzip2-format.pdf
// TODO: multiple streams, possible to figure out length of compressed? use footer magic?
// TODO: empty file, no streams

import (
	"compress/bzip2"
	"encoding/binary"
	"hash/crc32"
	"io"
	"math/bits"

	"github.com/wader/fq/format"
	"github.com/wader/fq/format/registry"
	"github.com/wader/fq/pkg/decode"
	"github.com/wader/fq/pkg/scalar"
)

var probeGroup decode.Group

func init() {
	registry.MustRegister(decode.Format{
		Name:        format.BZIP2,
		Description: "bzip2 compression",
		Groups:      []string{format.PROBE},
		DecodeFn:    bzip2Decode,
		Dependencies: []decode.Dependency{
			{Names: []string{format.PROBE}, Group: &probeGroup},
		},
	})
}

const blockMagic = 0x31_41_59_26_53_59
const footerMagic = 0x17_72_45_38_50_90

type bitFlipReader struct {
	r io.Reader
}

func (bfr bitFlipReader) Read(p []byte) (n int, err error) {
	n, err = bfr.r.Read(p)
	for i := 0; i < n; i++ {
		p[i] = bits.Reverse8(p[i])
	}
	return n, err
}

func bzip2Decode(d *decode.D, in interface{}) interface{} {
	// moreStreams := true

	// d.FieldArray("streams", func(d *decode.D) {
	// 	for moreStreams {
	// d.FieldStruct("stream", func(d *decode.D) {

	var blockCRCValue *decode.Value
	var streamCRCN uint32

	d.FieldUTF8("magic", 2, d.AssertStr("BZ"))
	d.FieldU8("version")
	d.FieldU8("hundred_k_blocksize")

	d.FieldStruct("block", func(d *decode.D) {
		// if d.PeekBits(48) != blockHeaderMagic {
		// 	moreStreams = false
		// 	return
		// }
		d.FieldU48("magic", d.AssertU(blockMagic), scalar.Hex)
		d.FieldU32("crc", scalar.Hex)
		blockCRCValue = d.FieldGet("crc")
		d.FieldU1("randomised")
		d.FieldU24("origptr")
		d.FieldU16("syncmapl1")

		d.SeekRel(-16)
		ranges := 0
		for i := 0; i < 16; i++ {
			if d.Bool() {
				ranges++
			}
		}
		d.FieldRawLen("syncmapl2", int64(ranges)*16)
		numTrees := d.FieldU3("num_trees")
		selectorsUsed := d.FieldU15("num_sels")
		selectorsI := uint64(0)
		d.FieldArrayLoop("selector_list", func() bool { return selectorsI < selectorsUsed }, func(d *decode.D) {
			d.FieldU1("selector")
			selectorsI++
		})
		treesI := uint64(0)
		d.FieldArrayLoop("trees", func() bool { return treesI < numTrees }, func(d *decode.D) {
			d.FieldUFn("tree", func(d *decode.D) uint64 {
				l := d.U5()
				if !d.Bool() {
					return l
				}
				if d.Bool() {
					l--
				} else {
					l++
				}
				return l
			})
			treesI++
		})
	})

	compressedStart := d.Pos()

	readCompressedSize, uncompressedBB, dv, _, _ := d.TryFieldReaderRangeFormat("uncompressed", 0, d.Len(), bzip2.NewReader, probeGroup, nil)
	if uncompressedBB != nil {
		if dv == nil {
			d.FieldRootBitBuf("uncompressed", uncompressedBB)
		}

		blockCRC32W := crc32.NewIEEE()
		d.MustCopy(blockCRC32W, bitFlipReader{uncompressedBB.Clone()})
		blockCRC32N := bits.Reverse32(binary.BigEndian.Uint32(blockCRC32W.Sum(nil)))
		_ = blockCRCValue.TryScalarFn(d.ValidateU(uint64(blockCRC32N)))
		streamCRCN = blockCRC32N ^ ((streamCRCN << 1) | (streamCRCN >> 31))

		// HACK: bzip2.NewReader will read from start of whole buffer and then we figure out compressedSize ourself
		// "It is important to note that none of the fields within a StreamBlock or StreamFooter are necessarily byte-aligned"
		const footerByteSize = 10
		compressedSize := (readCompressedSize - compressedStart) - footerByteSize*8
		for i := 0; i < 8; i++ {
			d.SeekAbs(compressedStart + compressedSize)
			if d.PeekBits(48) == footerMagic {
				break
			}
			compressedSize--
		}
		d.SeekAbs(compressedStart)

		d.FieldRawLen("compressed", compressedSize)

		d.FieldStruct("footer", func(d *decode.D) {
			d.FieldU48("magic", d.AssertU(footerMagic), scalar.Hex)
			// TODO: crc of block crcs
			d.FieldU32("crc", scalar.Hex, d.ValidateU(uint64(streamCRCN)))
			d.FieldRawLen("padding", int64(d.ByteAlignBits()))
		})
	}

	// 		moreStreams = false
	// 	}
	// })

	return nil
}