diff --git a/doc/formats.md b/doc/formats.md index dd10a03d..6ee344e8 100644 --- a/doc/formats.md +++ b/doc/formats.md @@ -19,7 +19,7 @@ |`avc_pps` |H.264/AVC Picture Parameter Set || |`avc_sei` |H.264/AVC Supplemental Enhancement Information || |`avc_sps` |H.264/AVC Sequence Parameter Set || -|`avro_ocf` |Avro object container file |`json`| +|`avro_ocf` |Avro object container file || |`bencode` |BitTorrent bencoding || |`bsd_loopback_frame` |BSD loopback frame |`ipv4_packet`| |`bson` |Binary JSON || diff --git a/doc/formats.svg b/doc/formats.svg index 7faafe95..78d7d31b 100644 --- a/doc/formats.svg +++ b/doc/formats.svg @@ -4,1620 +4,1612 @@ - + formats - + adts - -adts - -adts_frame + +adts + +adts_frame adts_frame - -adts_frame - -aac_frame + +adts_frame + +aac_frame adts:e->adts_frame:n - - + + aac_frame - -aac_frame + +aac_frame adts_frame:e->aac_frame:n - - + + apev2 - -apev2 - -image + +apev2 + +image image - -image + +image apev2:e->image:n - - + + - + jpeg - -jpeg - -exif - -icc_profile + +jpeg + +exif + +icc_profile - + image->jpeg:n - - + + - + mp4 - -mp4 - -aac_frame - -av1_ccr - -av1_frame - -flac_frame - -flac_metadatablocks - -id3v2 - -image - -jpeg - -mp3_frame - -avc_au - -avc_dcr - -mpeg_es - -hevc_au - -hevc_dcr - -mpeg_pes_packet - -opus_packet - -protobuf_widevine - -pssh_playready - -vorbis_packet - -vp9_frame - -vpx_ccr + +mp4 + +aac_frame + +av1_ccr + +av1_frame + +flac_frame + +flac_metadatablocks + +id3v2 + +image + +jpeg + +mp3_frame + +avc_au + +avc_dcr + +mpeg_es + +hevc_au + +hevc_dcr + +mpeg_pes_packet + +opus_packet + +protobuf_widevine + +pssh_playready + +vorbis_packet + +vp9_frame + +vpx_ccr - + image->mp4:n - - + + - + png - -png - -icc_profile - -exif + +png + +icc_profile + +exif - + image->png:n - - + + - + tiff - -tiff - -icc_profile + +tiff + +icc_profile - + image->tiff:n - - + + - + webp - -webp - -vp8_frame + +webp + +vp8_frame - + image->webp:n - - + + - + gif - -gif + +gif - + image->gif:n - - + + ar - -ar - -probe + +ar + +probe probe - -probe + +probe ar:e->probe:n - - + + - + probe->adts:n - - + + - + probe->ar:n - - - - - -avro_ocf - -avro_ocf - -json - - - -probe->avro_ocf:n - - - - - -json - -json - - - -probe->json:n - - + + - + bzip2 - -bzip2 - -probe + +bzip2 + +probe - + probe->bzip2:n - - + + - + flac - -flac - -flac_metadatablocks - -flac_frame + +flac + +flac_metadatablocks + +flac_frame - + probe->flac:n - - + + - + gzip - -gzip - -probe + +gzip + +probe - + probe->gzip:n - - + + - + probe->jpeg:n - - + + - + matroska - -matroska - -aac_frame - -av1_ccr - -av1_frame - -avc_au - -avc_dcr - -flac_frame - -flac_metadatablocks - -hevc_au - -hevc_dcr - -image - -mp3_frame - -mpeg_asc - -mpeg_pes_packet - -mpeg_spu - -opus_packet - -vorbis_packet - -vp8_frame - -vp9_cfm - -vp9_frame + +matroska + +aac_frame + +av1_ccr + +av1_frame + +avc_au + +avc_dcr + +flac_frame + +flac_metadatablocks + +hevc_au + +hevc_dcr + +image + +mp3_frame + +mpeg_asc + +mpeg_pes_packet + +mpeg_spu + +opus_packet + +vorbis_packet + +vp8_frame + +vp9_cfm + +vp9_frame - + probe->matroska:n - - + + - + mp3 - -mp3 - -id3v2 - -id3v1 - -id3v11 - -apev2 - -mp3_frame + +mp3 + +id3v2 + +id3v1 + +id3v11 + +apev2 + +mp3_frame - + probe->mp3:n - - + + - + probe->mp4:n - - + + - + ogg - -ogg - -ogg_page - -vorbis_packet - -opus_packet - -flac_metadatablock - -flac_frame + +ogg + +ogg_page + +vorbis_packet + +opus_packet + +flac_metadatablock + +flac_frame - + probe->ogg:n - - + + - + pcap - -pcap - -link_frame - -tcp_stream - -ipv4_packet + +pcap + +link_frame + +tcp_stream + +ipv4_packet - + probe->pcap:n - - + + - + pcapng - -pcapng - -link_frame - -tcp_stream - -ipv4_packet + +pcapng + +link_frame + +tcp_stream + +ipv4_packet - + probe->pcapng:n - - + + - + probe->png:n - - + + - + tar - -tar - -probe + +tar + +probe - + probe->tar:n - - + + - + probe->tiff:n - - + + - + wav - -wav - -id3v2 - -id3v1 - -id3v11 + +wav + +id3v2 + +id3v1 + +id3v11 - + probe->wav:n - - + + - + probe->webp:n - - + + - + zip - -zip - -probe + +zip + +probe - + probe->zip:n - - + + + + + +avro_ocf + +avro_ocf + + + +probe->avro_ocf:n + + - + elf - -elf + +elf - + probe->elf:n - - + + - + probe->gif:n - - + + + + + +json + +json + + + +probe->json:n + + mpeg_ts - -mpeg_ts + +mpeg_ts - + probe->mpeg_ts:n - - + + av1_frame - -av1_frame - -av1_obu + +av1_frame + +av1_obu av1_obu - -av1_obu + +av1_obu av1_frame:e->av1_obu:n - - + + avc_annexb - -avc_annexb - -avc_nalu + +avc_annexb + +avc_nalu avc_nalu - -avc_nalu - -avc_sps - -avc_pps - -avc_sei + +avc_nalu + +avc_sps + +avc_pps + +avc_sei avc_annexb:e->avc_nalu:n - - + + avc_sps - -avc_sps + +avc_sps avc_nalu:e->avc_sps:n - - + + avc_pps - -avc_pps + +avc_pps avc_nalu:e->avc_pps:n - - + + avc_sei - -avc_sei + +avc_sei avc_nalu:e->avc_sei:n - - + + avc_au - -avc_au - -avc_nalu + +avc_au + +avc_nalu avc_au:e->avc_nalu:n - - + + avc_dcr - -avc_dcr - -avc_nalu + +avc_dcr + +avc_nalu avc_dcr:e->avc_nalu:n - - - - - -avro_ocf:e->json:n - - + + - + bsd_loopback_frame - -bsd_loopback_frame - -ipv4_packet + +bsd_loopback_frame + +ipv4_packet - + ipv4_packet - -ipv4_packet - -udp_datagram - -tcp_segment - -icmp + +ipv4_packet + +udp_datagram + +tcp_segment + +icmp - + bsd_loopback_frame:e->ipv4_packet:n - - + + - + udp_datagram - -udp_datagram - -udp_payload + +udp_datagram + +udp_payload - + ipv4_packet:e->udp_datagram:n - - + + - + tcp_segment - -tcp_segment + +tcp_segment - + ipv4_packet:e->tcp_segment:n - - + + - + icmp - -icmp + +icmp - + ipv4_packet:e->icmp:n - - + + - + bzip2:e->probe:n - - + + - + ether8023_frame - -ether8023_frame - -ipv4_packet + +ether8023_frame + +ipv4_packet - + ether8023_frame:e->ipv4_packet:n - - + + - + flac_metadatablocks - -flac_metadatablocks - -flac_metadatablock + +flac_metadatablocks + +flac_metadatablock - + flac:e->flac_metadatablocks:n - - + + - + flac_frame - -flac_frame + +flac_frame - + flac:e->flac_frame:n - - + + - + flac_metadatablock - -flac_metadatablock - -flac_streaminfo - -flac_picture - -vorbis_comment + +flac_metadatablock + +flac_streaminfo + +flac_picture + +vorbis_comment - + flac_metadatablocks:e->flac_metadatablock:n - - + + - + flac_streaminfo - -flac_streaminfo + +flac_streaminfo - + flac_metadatablock:e->flac_streaminfo:n - - + + - + flac_picture - -flac_picture - -image + +flac_picture + +image - + flac_metadatablock:e->flac_picture:n - - + + - + vorbis_comment - -vorbis_comment - -flac_picture + +vorbis_comment + +flac_picture - + flac_metadatablock:e->vorbis_comment:n - - + + - + flac_picture:e->image:n - - + + - + vorbis_comment:e->flac_picture:n - - + + - + gzip:e->probe:n - - + + - + hevc_annexb - -hevc_annexb - -hevc_nalu + +hevc_annexb + +hevc_nalu - + hevc_nalu - -hevc_nalu + +hevc_nalu - + hevc_annexb:e->hevc_nalu:n - - + + - + hevc_au - -hevc_au - -hevc_nalu + +hevc_au + +hevc_nalu - + hevc_au:e->hevc_nalu:n - - + + - + hevc_dcr - -hevc_dcr - -hevc_nalu + +hevc_dcr + +hevc_nalu - + hevc_dcr:e->hevc_nalu:n - - + + - + id3v2 - -id3v2 - -image + +id3v2 + +image - + id3v2:e->image:n - - + + - + udp_payload - -udp_payload + +udp_payload - + udp_datagram:e->udp_payload:n - - + + - + exif - -exif + +exif - + jpeg:e->exif:n - - + + - + icc_profile - -icc_profile + +icc_profile - + jpeg:e->icc_profile:n - - + + - + matroska:e->aac_frame:n - - + + - + matroska:e->image:n - - + + - + matroska:e->av1_frame:n - - + + - + matroska:e->avc_au:n - - + + - + matroska:e->avc_dcr:n - - + + - + matroska:e->flac_metadatablocks:n - - + + - + matroska:e->flac_frame:n - - + + - + matroska:e->hevc_au:n - - + + - + matroska:e->hevc_dcr:n - - + + - + av1_ccr - -av1_ccr + +av1_ccr - + matroska:e->av1_ccr:n - - + + - + mp3_frame - -mp3_frame - -xing + +mp3_frame + +xing - + matroska:e->mp3_frame:n - - + + - + mpeg_asc - -mpeg_asc + +mpeg_asc - + matroska:e->mpeg_asc:n - - + + - + mpeg_pes_packet - -mpeg_pes_packet + +mpeg_pes_packet - + matroska:e->mpeg_pes_packet:n - - + + - + mpeg_spu - -mpeg_spu + +mpeg_spu - + matroska:e->mpeg_spu:n - - + + - + opus_packet - -opus_packet - -vorbis_comment + +opus_packet + +vorbis_comment - + matroska:e->opus_packet:n - - + + - + vorbis_packet - -vorbis_packet - -vorbis_comment + +vorbis_packet + +vorbis_comment - + matroska:e->vorbis_packet:n - - + + - + vp8_frame - -vp8_frame + +vp8_frame - + matroska:e->vp8_frame:n - - + + - + vp9_cfm - -vp9_cfm + +vp9_cfm - + matroska:e->vp9_cfm:n - - + + - + vp9_frame - -vp9_frame + +vp9_frame - + matroska:e->vp9_frame:n - - + + - + xing - -xing + +xing - + mp3_frame:e->xing:n - - + + - + opus_packet:e->vorbis_comment:n - - + + - + vorbis_packet:e->vorbis_comment:n - - + + - + mp3:e->apev2:n - - + + - + mp3:e->id3v2:n - - + + - + mp3:e->mp3_frame:n - - + + - + id3v1 - -id3v1 + +id3v1 - + mp3:e->id3v1:n - - + + - + id3v11 - -id3v11 + +id3v11 - + mp3:e->id3v11:n - - + + - + mp4:e->aac_frame:n - - + + - + mp4:e->image:n - - + + - + mp4:e->av1_frame:n - - + + - + mp4:e->avc_au:n - - + + - + mp4:e->avc_dcr:n - - + + - + mp4:e->flac_metadatablocks:n - - + + - + mp4:e->flac_frame:n - - + + - + mp4:e->hevc_au:n - - + + - + mp4:e->hevc_dcr:n - - + + - + mp4:e->id3v2:n - - + + - + mp4:e->jpeg:n - - + + - + mp4:e->av1_ccr:n - - + + - + mp4:e->mp3_frame:n - - + + - + mp4:e->mpeg_pes_packet:n - - + + - + mp4:e->opus_packet:n - - + + - + mp4:e->vorbis_packet:n - - + + - + mp4:e->vp9_frame:n - - + + - + mpeg_es - -mpeg_es - -mpeg_asc - -vorbis_packet + +mpeg_es + +mpeg_asc + +vorbis_packet - + mp4:e->mpeg_es:n - - + + - + protobuf_widevine - -protobuf_widevine - -protobuf + +protobuf_widevine + +protobuf - + mp4:e->protobuf_widevine:n - - + + - + pssh_playready - -pssh_playready + +pssh_playready - + mp4:e->pssh_playready:n - - + + - + vpx_ccr - -vpx_ccr + +vpx_ccr - + mp4:e->vpx_ccr:n - - + + - + mpeg_es:e->mpeg_asc:n - - + + - + mpeg_es:e->vorbis_packet:n - - + + - + protobuf - -protobuf + +protobuf - + protobuf_widevine:e->protobuf:n - - + + - + mpeg_pes - -mpeg_pes - -mpeg_pes_packet - -mpeg_spu + +mpeg_pes + +mpeg_pes_packet + +mpeg_spu - + mpeg_pes:e->mpeg_pes_packet:n - - + + - + mpeg_pes:e->mpeg_spu:n - - + + - + ogg:e->flac_frame:n - - + + - + ogg:e->flac_metadatablock:n - - + + - + ogg:e->opus_packet:n - - + + - + ogg:e->vorbis_packet:n - - + + - + ogg_page - -ogg_page + +ogg_page - + ogg:e->ogg_page:n - - + + - + pcap:e->ipv4_packet:n - - + + - + link_frame - -link_frame + +link_frame - + pcap:e->link_frame:n - - + + - + tcp_stream - -tcp_stream + +tcp_stream - + pcap:e->tcp_stream:n - - + + - + link_frame->bsd_loopback_frame:n - - + + - + link_frame->ether8023_frame:n - - + + - + sll2_packet - -sll2_packet - -ether8023_frame + +sll2_packet + +ether8023_frame - + link_frame->sll2_packet:n - - + + - + sll_packet - -sll_packet - -ether8023_frame + +sll_packet + +ether8023_frame - + link_frame->sll_packet:n - - + + - + dns - -dns + +dns - + tcp_stream->dns:n - - + + - + pcapng:e->ipv4_packet:n - - + + - + pcapng:e->link_frame:n - - + + - + pcapng:e->tcp_stream:n - - + + - + png:e->exif:n - - + + - + png:e->icc_profile:n - - + + - + sll2_packet:e->ether8023_frame:n - - + + - + sll_packet:e->ether8023_frame:n - - + + - + tar:e->probe:n - - + + - + tiff:e->icc_profile:n - - + + - + udp_payload->dns:n - - + + - + wav:e->id3v2:n - - + + - + wav:e->id3v1:n - - + + - + wav:e->id3v11:n - - + + - + webp:e->vp8_frame:n - - + + - + zip:e->probe:n - - + + bencode - -bencode + +bencode bson - -bson + +bson cbor - -cbor + +cbor dns_tcp - -dns_tcp + +dns_tcp msgpack - -msgpack + +msgpack raw - -raw + +raw diff --git a/format/avro/decoders/map.go b/format/avro/decoders/map.go index 63803aa8..86e0b567 100644 --- a/format/avro/decoders/map.go +++ b/format/avro/decoders/map.go @@ -47,23 +47,27 @@ func decodeMapFn(s schema.SimplifiedSchema) (DecodeFn, error) { val := make(map[string]interface{}) rawV := subFn(s, d) - impl, ok := rawV.([]map[string]interface{}) + rawSlice, ok := rawV.([]interface{}) if !ok { - d.Fatalf("decode map: expected array of maps, got %T", rawV) + d.Fatalf("decode map: expected array of interfaces, got %v", rawV) return nil } - for entry := range impl { - rawKey, ok := impl[entry]["key"] + for _, rawEntry := range rawSlice { + entry, ok := rawEntry.(map[string]interface{}) if !ok { - d.Fatalf("decode map: expected key in map %v", impl[entry]) + d.Fatalf("decode map: expected map, got %T", rawEntry) } - value, ok := impl[entry]["key"] + rawKey, ok := entry["key"] if !ok { - d.Fatalf("decode map: expected value in map %v", impl[entry]) + d.Fatalf("decode map: expected key in map %v", entry) + } + value, ok := entry["value"] + if !ok { + d.Fatalf("decode map: expected value in map %v", entry) } key, ok := rawKey.(string) if !ok { - d.Fatalf("decode map: expected string key in map %v", impl[entry]) + d.Fatalf("decode map: expected string key in map %v", entry) } val[key] = value } diff --git a/format/avro/ocf.go b/format/avro/ocf.go index ddb298fc..1695a553 100644 --- a/format/avro/ocf.go +++ b/format/avro/ocf.go @@ -9,79 +9,71 @@ import ( "github.com/wader/fq/pkg/scalar" ) -var jsonGroup decode.Group - func init() { registry.MustRegister(decode.Format{ Name: format.AVRO_OCF, Description: "Avro object container file", Groups: []string{format.PROBE}, DecodeFn: avroDecodeOCF, - Dependencies: []decode.Dependency{ - {Names: []string{format.JSON}, Group: &jsonGroup}, - }, }) } type HeaderData struct { - Schema *schema.SimplifiedSchema + Schema schema.SimplifiedSchema Codec string Sync []byte } +const headerSchemaSpec = ` +{ + "type": "record", + "name": "org.apache.avro.file.Header", + "fields": [ + {"name": "meta", "type": {"type": "map", "values": "string"}}, + {"name": "sync", "type": {"type": "fixed", "name": "Sync", "size": 16}} + ] +}` + func decodeHeader(d *decode.D) HeaderData { + d.FieldRawLen("magic", 4*8, d.AssertBitBuf([]byte{'O', 'b', 'j', 1})) + var headerData HeaderData - // Header is encoded in avro so could use avro decoder, but doing it manually so we can - // keep asserts and treating schema as JSON - d.FieldRawLen("magic", 4*8, d.AssertBitBuf([]byte{'O', 'b', 'j', 1})) - var blockCount int64 = -1 - d.FieldStructArrayLoop("meta", "block", - func() bool { return blockCount != 0 }, - func(d *decode.D) { - blockCount = d.FieldSFn("count", decoders.VarZigZag) - // If its negative, then theres another long representing byte size - if blockCount < 0 { - blockCount *= -1 - d.FieldSFn("size", decoders.VarZigZag) - } - if blockCount == 0 { - return - } - - var i int64 - d.FieldStructArrayLoop("entries", "entry", func() bool { return i < blockCount }, func(d *decode.D) { - keyL := d.FieldSFn("key_len", decoders.VarZigZag) - key := d.FieldUTF8("key", int(keyL)) - valL := d.FieldSFn("value_len", decoders.VarZigZag) - if key == "avro.schema" { - v, _ := d.FieldFormatLen("value", valL*8, jsonGroup, nil) - s, err := schema.From(v.V.(*scalar.S).Actual) - headerData.Schema = &s - if err != nil { - d.Fatalf("Failed to parse schema: %s", err) - } - } else if key == "avro.codec" { - headerData.Codec = d.FieldUTF8("value", int(valL)) - } else { - d.FieldUTF8("value", int(valL)) - } - i++ - }) - }) - if headerData.Schema == nil { - d.Fatalf("No schema found in header") + headerSchema, err := schema.FromSchemaString(headerSchemaSpec) + if err != nil { + d.Fatalf("Failed to parse header schema: %v", err) + } + decodeHeaderFn, err := decoders.DecodeFnForSchema(headerSchema) + if err != nil { + d.Fatalf("failed to parse header: %v", err) } - if headerData.Codec == "null" { + header := decodeHeaderFn("header", d) + headerRecord, ok := header.(map[string]interface{}) + if !ok { + d.Fatalf("header is not a map") + } + meta, ok := headerRecord["meta"].(map[string]interface{}) + if !ok { + d.Fatalf("header.meta is not a map") + } + + headerData.Schema, err = schema.FromSchemaString(meta["avro.schema"].(string)) + if err != nil { + d.Fatalf("failed to parse schema: %v", err) + } + if codec, ok := meta["avro.codec"]; ok && codec != "null" { + headerData.Codec, ok = codec.(string) + if !ok { + d.Fatalf("avro.codec is not a string") + } + } else { headerData.Codec = "" } - syncbb := d.FieldRawLen("sync", 16*8) - var err error - headerData.Sync, err = syncbb.BytesLen(16) - if err != nil { - d.Fatalf("unable to read sync bytes: %v", err) + headerData.Sync, ok = headerRecord["sync"].([]byte) + if !ok { + d.Fatalf("header.sync is not a byte array") } return headerData } @@ -89,7 +81,7 @@ func decodeHeader(d *decode.D) HeaderData { func avroDecodeOCF(d *decode.D, in interface{}) interface{} { header := decodeHeader(d) - decodeFn, err := decoders.DecodeFnForSchema(*header.Schema) + decodeFn, err := decoders.DecodeFnForSchema(header.Schema) if err != nil { d.Fatalf("unable to create codec: %v", err) } diff --git a/format/avro/schema/schema.go b/format/avro/schema/schema.go index e2e5c5c0..125fe470 100644 --- a/format/avro/schema/schema.go +++ b/format/avro/schema/schema.go @@ -1,6 +1,7 @@ package schema import ( + "encoding/json" "errors" "fmt" ) @@ -34,9 +35,9 @@ type SimplifiedSchema struct { Symbols []string `json:"symbols,omitempty"` Values *SimplifiedSchema `json:"values,omitempty"` UnionTypes []SimplifiedSchema - //Choosing not to handle Default as it adds a lot of complexity and this is used for showing the binary - //representation of the data, not fully parsing it. See https://github.com/linkedin/goavro/blob/master/record.go - //for how it could be handled. + // Choosing not to handle Default as it adds a lot of complexity and this is used for showing the binary + // representation of the data, not fully parsing it. See https://github.com/linkedin/goavro/blob/master/record.go + // for how it could be handled. } type Field struct { @@ -44,6 +45,15 @@ type Field struct { Type SimplifiedSchema } +func FromSchemaString(schemaString string) (SimplifiedSchema, error) { + var jsonSchema interface{} + if err := json.Unmarshal([]byte(schemaString), &jsonSchema); err != nil { + return SimplifiedSchema{}, fmt.Errorf("failed to unmarshal header schema: %w", err) + } + + return From(jsonSchema) +} + func From(schema interface{}) (SimplifiedSchema, error) { if schema == nil { return SimplifiedSchema{}, errors.New("schema cannot be nil") @@ -81,7 +91,7 @@ func From(schema interface{}) (SimplifiedSchema, error) { if s.Precision, err = getInt(v, "precision", false); err != nil { return s, err } - if s.Size, err = getInt(v, "precision", false); err != nil { + if s.Size, err = getInt(v, "size", false); err != nil { return s, err } if s.Type == RECORD { @@ -190,9 +200,9 @@ func getInt(m map[string]interface{}, key string, required bool) (int, error) { } return 0, nil } - s, ok := v.(int) + s, ok := v.(float64) if !ok { - return 0, fmt.Errorf("%s must be a string", key) + return 0, fmt.Errorf("%s must be a int", key) } - return s, nil + return int(s), nil } diff --git a/format/avro/testdata/firstBlockCountNotGreaterThanZero.fqtest b/format/avro/testdata/firstBlockCountNotGreaterThanZero.fqtest index 93decf65..e8187b68 100644 --- a/format/avro/testdata/firstBlockCountNotGreaterThanZero.fqtest +++ b/format/avro/testdata/firstBlockCountNotGreaterThanZero.fqtest @@ -2,20 +2,24 @@ $ fq verbose firstBlockCountNotGreaterThanZero.avro |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: firstBlockCountNotGreaterThanZero.avro (avro_ocf) 0x0-0x32.7 (51) 0x00|4f 62 6a 01 |Obj. | magic: raw bits (valid) 0x0-0x3.7 (4) - | | | meta[0:2]: 0x4-0x21.7 (30) - | | | [0]{}: block 0x4-0x20.7 (29) -0x00| 02 | . | count: 1 0x4-0x4.7 (1) - | | | entries[0:1]: 0x5-0x20.7 (28) - | | | [0]{}: entry 0x5-0x20.7 (28) -0x00| 16 | . | key_len: 11 0x5-0x5.7 (1) -0x00| 61 76 72 6f 2e 73 63 68 65 6d| avro.schem| key: "avro.schema" 0x6-0x10.7 (11) + | | | header{}: 0x4-0x31.7 (46) + | | | meta[0:2]: 0x4-0x21.7 (30) + | | | [0]{}: block 0x4-0x20.7 (29) +0x00| 02 | . | count: 1 0x4-0x4.7 (1) + | | | data[0:1]: 0x5-0x20.7 (28) + | | | [0]{}: entry 0x5-0x20.7 (28) + | | | key{}: 0x5-0x10.7 (12) +0x00| 16 | . | length: 11 0x5-0x5.7 (1) +0x00| 61 76 72 6f 2e 73 63 68 65 6d| avro.schem| data: "avro.schema" 0x6-0x10.7 (11) 0x10|61 |a | -0x10| 1e | . | value_len: 15 0x11-0x11.7 (1) -0x10| 7b 22 74 79 70 65 22 3a 22 6c 6f 6e 67 22| {"type":"long"| value: {} (json) 0x12-0x20.7 (15) + | | | value{}: 0x11-0x20.7 (16) +0x10| 1e | . | length: 15 0x11-0x11.7 (1) +0x10| 7b 22 74 79 70 65 22 3a 22 6c 6f 6e 67 22| {"type":"long"| data: "{\"type\":\"long\"}" 0x12-0x20.7 (15) 0x20|7d |} | - | | | [1]{}: block 0x21-0x21.7 (1) -0x20| 00 | . | count: 0 0x21-0x21.7 (1) -0x20| 30 31 32 33 34 35 36 37 38 39 61 62 63 64| 0123456789abcd| sync: raw bits 0x22-0x31.7 (16) + | | | [1]{}: block 0x21-0x21.7 (1) +0x20| 00 | . | count: 0 0x21-0x21.7 (1) + | | | data[0:0]: 0x22-NA (0) +0x20| 30 31 32 33 34 35 36 37 38 39 61 62 63 64| 0123456789abcd| sync: raw bits 0x22-0x31.7 (16) 0x30|65 66 |ef | | | | blocks[0:1]: 0x32-0x32.7 (1) | | | [0]{}: block 0x32-0x32.7 (1) diff --git a/format/avro/testdata/quickstop-deflate.fqtest b/format/avro/testdata/quickstop-deflate.fqtest index ff2d5026..4651413d 100644 --- a/format/avro/testdata/quickstop-deflate.fqtest +++ b/format/avro/testdata/quickstop-deflate.fqtest @@ -2,26 +2,32 @@ $ fq verbose quickstop-deflate.avro |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: quickstop-deflate.avro (avro_ocf) 0x0-0x5835.7 (22582) 0x0000|4f 62 6a 01 |Obj. | magic: raw bits (valid) 0x0-0x3.7 (4) - | | | meta[0:2]: 0x4-0x119.7 (278) - | | | [0]{}: block 0x4-0x118.7 (277) -0x0000| 04 | . | count: 2 0x4-0x4.7 (1) - | | | entries[0:2]: 0x5-0x118.7 (276) - | | | [0]{}: entry 0x5-0x17.7 (19) -0x0000| 14 | . | key_len: 10 0x5-0x5.7 (1) -0x0000| 61 76 72 6f 2e 63 6f 64 65 63| avro.codec| key: "avro.codec" 0x6-0xf.7 (10) -0x0010|0e |. | value_len: 7 0x10-0x10.7 (1) -0x0010| 64 65 66 6c 61 74 65 | deflate | value: "deflate" 0x11-0x17.7 (7) - | | | [1]{}: entry 0x18-0x118.7 (257) -0x0010| 16 | . | key_len: 11 0x18-0x18.7 (1) -0x0010| 61 76 72 6f 2e 73 63| avro.sc| key: "avro.schema" 0x19-0x23.7 (11) + | | | header{}: 0x4-0x129.7 (294) + | | | meta[0:2]: 0x4-0x119.7 (278) + | | | [0]{}: block 0x4-0x118.7 (277) +0x0000| 04 | . | count: 2 0x4-0x4.7 (1) + | | | data[0:2]: 0x5-0x118.7 (276) + | | | [0]{}: entry 0x5-0x17.7 (19) + | | | key{}: 0x5-0xf.7 (11) +0x0000| 14 | . | length: 10 0x5-0x5.7 (1) +0x0000| 61 76 72 6f 2e 63 6f 64 65 63| avro.codec| data: "avro.codec" 0x6-0xf.7 (10) + | | | value{}: 0x10-0x17.7 (8) +0x0010|0e |. | length: 7 0x10-0x10.7 (1) +0x0010| 64 65 66 6c 61 74 65 | deflate | data: "deflate" 0x11-0x17.7 (7) + | | | [1]{}: entry 0x18-0x118.7 (257) + | | | key{}: 0x18-0x23.7 (12) +0x0010| 16 | . | length: 11 0x18-0x18.7 (1) +0x0010| 61 76 72 6f 2e 73 63| avro.sc| data: "avro.schema" 0x19-0x23.7 (11) 0x0020|68 65 6d 61 |hema | -0x0020| e6 03 | .. | value_len: 243 0x24-0x25.7 (2) -0x0020| 7b 22 74 79 70 65 22 3a 22 72| {"type":"r| value: {} (json) 0x26-0x118.7 (243) + | | | value{}: 0x24-0x118.7 (245) +0x0020| e6 03 | .. | length: 243 0x24-0x25.7 (2) +0x0020| 7b 22 74 79 70 65 22 3a 22 72| {"type":"r| data: "{\"type\":\"record\",\"name\":\"Person\",\"fields\":[{\"name\""... 0x26-0x118.7 (243) 0x0030|65 63 6f 72 64 22 2c 22 6e 61 6d 65 22 3a 22 50|ecord","name":"P| * |until 0x118.7 (243) | | - | | | [1]{}: block 0x119-0x119.7 (1) -0x0110| 00 | . | count: 0 0x119-0x119.7 (1) -0x0110| 93 e7 87 9e 02 95| ......| sync: raw bits 0x11a-0x129.7 (16) + | | | [1]{}: block 0x119-0x119.7 (1) +0x0110| 00 | . | count: 0 0x119-0x119.7 (1) + | | | data[0:0]: 0x11a-NA (0) +0x0110| 93 e7 87 9e 02 95| ......| sync: raw bits 0x11a-0x129.7 (16) 0x0120|d5 9e 4f 58 37 ad b2 a2 ce cd |..OX7..... | | | | blocks[0:12]: 0x12a-0x5835.7 (22284) | | | [0]{}: block 0x12a-0x9dc.7 (2227) diff --git a/format/avro/testdata/twitter.fqtest b/format/avro/testdata/twitter.fqtest index 239d60fe..ebbe42ae 100644 --- a/format/avro/testdata/twitter.fqtest +++ b/format/avro/testdata/twitter.fqtest @@ -2,27 +2,33 @@ $ fq verbose twitter.avro |00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f|0123456789abcdef|.{}: twitter.avro (avro_ocf) 0x0-0x21e.7 (543) 0x000|4f 62 6a 01 |Obj. | magic: raw bits (valid) 0x0-0x3.7 (4) - | | | meta[0:2]: 0x4-0x197.7 (404) - | | | [0]{}: block 0x4-0x196.7 (403) -0x000| 04 | . | count: 2 0x4-0x4.7 (1) - | | | entries[0:2]: 0x5-0x196.7 (402) - | | | [0]{}: entry 0x5-0x186.7 (386) -0x000| 16 | . | key_len: 11 0x5-0x5.7 (1) -0x000| 61 76 72 6f 2e 73 63 68 65 6d| avro.schem| key: "avro.schema" 0x6-0x10.7 (11) + | | | header{}: 0x4-0x1a7.7 (420) + | | | meta[0:2]: 0x4-0x197.7 (404) + | | | [0]{}: block 0x4-0x196.7 (403) +0x000| 04 | . | count: 2 0x4-0x4.7 (1) + | | | data[0:2]: 0x5-0x196.7 (402) + | | | [0]{}: entry 0x5-0x186.7 (386) + | | | key{}: 0x5-0x10.7 (12) +0x000| 16 | . | length: 11 0x5-0x5.7 (1) +0x000| 61 76 72 6f 2e 73 63 68 65 6d| avro.schem| data: "avro.schema" 0x6-0x10.7 (11) 0x010|61 |a | -0x010| e8 05 | .. | value_len: 372 0x11-0x12.7 (2) -0x010| 7b 22 74 79 70 65 22 3a 22 72 65 63 6f| {"type":"reco| value: {} (json) 0x13-0x186.7 (372) + | | | value{}: 0x11-0x186.7 (374) +0x010| e8 05 | .. | length: 372 0x11-0x12.7 (2) +0x010| 7b 22 74 79 70 65 22 3a 22 72 65 63 6f| {"type":"reco| data: "{\"type\":\"record\",\"name\":\"twitter_schema\",\"namespac"... 0x13-0x186.7 (372) 0x020|72 64 22 2c 22 6e 61 6d 65 22 3a 22 74 77 69 74|rd","name":"twit| * |until 0x186.7 (372) | | - | | | [1]{}: entry 0x187-0x196.7 (16) -0x180| 14 | . | key_len: 10 0x187-0x187.7 (1) -0x180| 61 76 72 6f 2e 63 6f 64| avro.cod| key: "avro.codec" 0x188-0x191.7 (10) + | | | [1]{}: entry 0x187-0x196.7 (16) + | | | key{}: 0x187-0x191.7 (11) +0x180| 14 | . | length: 10 0x187-0x187.7 (1) +0x180| 61 76 72 6f 2e 63 6f 64| avro.cod| data: "avro.codec" 0x188-0x191.7 (10) 0x190|65 63 |ec | -0x190| 08 | . | value_len: 4 0x192-0x192.7 (1) -0x190| 6e 75 6c 6c | null | value: "null" 0x193-0x196.7 (4) - | | | [1]{}: block 0x197-0x197.7 (1) -0x190| 00 | . | count: 0 0x197-0x197.7 (1) -0x190| 67 c7 35 29 73 ef df 94| g.5)s...| sync: raw bits 0x198-0x1a7.7 (16) + | | | value{}: 0x192-0x196.7 (5) +0x190| 08 | . | length: 4 0x192-0x192.7 (1) +0x190| 6e 75 6c 6c | null | data: "null" 0x193-0x196.7 (4) + | | | [1]{}: block 0x197-0x197.7 (1) +0x190| 00 | . | count: 0 0x197-0x197.7 (1) + | | | data[0:0]: 0x198-NA (0) +0x190| 67 c7 35 29 73 ef df 94| g.5)s...| sync: raw bits 0x198-0x1a7.7 (16) 0x1a0|ad d3 00 7e 9e eb ff ae |...~.... | | | | blocks[0:1]: 0x1a8-0x21e.7 (119) | | | [0]{}: block 0x1a8-0x21e.7 (119)