mirror of
https://github.com/wader/fq.git
synced 2024-12-01 19:12:34 +03:00
e9d9f8aef9
Rename s/interface{}/any/g
Preparation for using generics in decode API and native jq funcations etc
Remove some unused linter ignores as linter has been fixed
326 lines
11 KiB
Go
326 lines
11 KiB
Go
package zip
|
|
|
|
// https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT
|
|
// https://opensource.apple.com/source/zip/zip-6/unzip/unzip/proginfo/extra.fld
|
|
// TODO: zip64
|
|
|
|
import (
|
|
"bytes"
|
|
"compress/flate"
|
|
"io"
|
|
|
|
"github.com/wader/fq/format"
|
|
"github.com/wader/fq/format/registry"
|
|
"github.com/wader/fq/pkg/decode"
|
|
"github.com/wader/fq/pkg/scalar"
|
|
)
|
|
|
|
var probeFormat decode.Group
|
|
|
|
func init() {
|
|
registry.MustRegister(decode.Format{
|
|
Name: format.ZIP,
|
|
Description: "ZIP archive",
|
|
Groups: []string{format.PROBE},
|
|
DecodeFn: zipDecode,
|
|
Dependencies: []decode.Dependency{
|
|
{Names: []string{format.PROBE}, Group: &probeFormat},
|
|
},
|
|
})
|
|
}
|
|
|
|
const (
|
|
compressionMethodNone = 0
|
|
compressionMethodShrunk = 1
|
|
compressionMethodReducedCompressionFactor1 = 2
|
|
compressionMethodReducedCompressionFactor2 = 3
|
|
compressionMethodReducedCompressionFactor3 = 4
|
|
compressionMethodReducedCompressionFactor4 = 5
|
|
compressionMethodImploded = 6
|
|
compressionMethodDeflated = 8
|
|
compressionMethodEnhancedDeflated = 9
|
|
compressionMethodPKWareDCLImploded = 10
|
|
compressionMethodBzip2 = 12
|
|
compressionMethodLZMA = 14
|
|
compressionMethodIBMTERSE = 18
|
|
compressionMethodIBMLZ77z = 19
|
|
compressionMethodPPMd = 98
|
|
)
|
|
|
|
var compressionMethodMap = scalar.UToSymStr{
|
|
compressionMethodNone: "none",
|
|
compressionMethodShrunk: "shrunk",
|
|
compressionMethodReducedCompressionFactor1: "reduced_compression_factor1",
|
|
compressionMethodReducedCompressionFactor2: "reduced_compression_factor2",
|
|
compressionMethodReducedCompressionFactor3: "reduced_compression_factor3",
|
|
compressionMethodReducedCompressionFactor4: "reduced_compression_factor4",
|
|
compressionMethodImploded: "imploded",
|
|
compressionMethodDeflated: "deflated",
|
|
compressionMethodEnhancedDeflated: "enhanced_deflated",
|
|
compressionMethodPKWareDCLImploded: "pk_ware_dcl_imploded",
|
|
compressionMethodBzip2: "bzip2",
|
|
compressionMethodLZMA: "lzma",
|
|
compressionMethodIBMTERSE: "ibmterse",
|
|
compressionMethodIBMLZ77z: "ibmlz77z",
|
|
compressionMethodPPMd: "pp_md",
|
|
}
|
|
|
|
var (
|
|
centralDirectorySignature = []byte("PK\x01\x02")
|
|
endOfCentralDirectorySignature = []byte("PK\x05\x06")
|
|
endOfCentralDirectorySignatureN = 0x06054b50
|
|
localFileSignature = []byte("PK\x03\x04")
|
|
dataIndicatorSignature = []byte("PK\x07\x08")
|
|
)
|
|
|
|
var headerIDMap = scalar.UToDescription{
|
|
0x0001: "ZIP64 extended information extra field",
|
|
0x0007: "AV Info",
|
|
0x0009: "OS/2 extended attributes",
|
|
0x000a: "NTFS (Win9x/WinNT FileTimes)",
|
|
0x000c: "OpenVMS",
|
|
0x000d: "Unix",
|
|
0x000f: "Patch Descriptor",
|
|
0x0014: "PKCS#7 Store for X.509 Certificates",
|
|
0x0015: "X.509 Certificate ID and Signature for individual file",
|
|
0x0016: "X.509 Certificate ID for Central Directory",
|
|
0x0065: "IBM S/390 attributes - uncompressed",
|
|
0x0066: "IBM S/390 attributes - compressed",
|
|
0x07c8: "Info-ZIP Macintosh (old, J. Lee)",
|
|
0x2605: "ZipIt Macintosh (first version)",
|
|
0x2705: "ZipIt Macintosh v 1.3.5 and newer (w/o full filename)",
|
|
0x334d: "Info-ZIP Macintosh (new, D. Haase's 'Mac3' field )",
|
|
0x4154: "Tandem NSK",
|
|
0x4341: "Acorn/SparkFS (David Pilling)",
|
|
0x4453: "Windows NT security descriptor (binary ACL)",
|
|
0x4704: "VM/CMS",
|
|
0x470f: "MVS",
|
|
// "inofficial" in original table
|
|
//nolint:misspell
|
|
0x4854: "Theos, old inofficial port",
|
|
0x4b46: "FWKCS MD5 (see below)",
|
|
0x4c41: "OS/2 access control list (text ACL)",
|
|
0x4d49: "Info-ZIP OpenVMS (obsolete)",
|
|
0x4d63: "Macintosh SmartZIP, by Macro Bambini",
|
|
0x4f4c: "Xceed original location extra field",
|
|
0x5356: "AOS/VS (binary ACL)",
|
|
0x5455: "extended timestamp",
|
|
0x5855: "Info-ZIP Unix (original; also OS/2, NT, etc.)",
|
|
0x554e: "Xceed unicode extra field",
|
|
0x6542: "BeOS (BeBox, PowerMac, etc.)",
|
|
0x6854: "Theos",
|
|
0x756e: "ASi Unix",
|
|
0x7855: "Info-ZIP Unix (new)",
|
|
0x7875: "UNIX UID/GID",
|
|
0xfb4a: "SMS/QDOS",
|
|
}
|
|
|
|
// "MS-DOS uses year values relative to 1980 and 2 second precision."
|
|
func fieldMSDOSTime(d *decode.D) {
|
|
d.FieldU5("hours")
|
|
d.FieldU6("minutes")
|
|
d.FieldU5("seconds")
|
|
}
|
|
|
|
func fieldMSDOSDate(d *decode.D) {
|
|
d.FieldU7("year")
|
|
d.FieldU4("month")
|
|
d.FieldU5("day")
|
|
}
|
|
|
|
func zipDecode(d *decode.D, in any) any {
|
|
// TODO: just decode instead?
|
|
if !bytes.Equal(d.PeekBytes(4), []byte("PK\x03\x04")) {
|
|
d.Errorf("expected PK header")
|
|
}
|
|
|
|
d.Endian = decode.LittleEndian
|
|
|
|
d.SeekAbs(d.Len())
|
|
|
|
// TODO: better EOCD probe
|
|
p, _, err := d.TryPeekFind(32, -8, -10000, func(v uint64) bool {
|
|
return v == uint64(endOfCentralDirectorySignatureN)
|
|
})
|
|
if err != nil {
|
|
d.Fatalf("can't find end of central directory")
|
|
}
|
|
d.SeekAbs(d.Len() + p)
|
|
|
|
var offsetCD uint64
|
|
var sizeCD uint64
|
|
var diskNr uint64
|
|
|
|
d.FieldStruct("end_of_central_directory", func(d *decode.D) {
|
|
d.FieldRawLen("signature", 4*8, d.AssertBitBuf(endOfCentralDirectorySignature))
|
|
diskNr = d.FieldU16("disk_nr")
|
|
d.FieldU16("central_directory_start_disk_nr")
|
|
d.FieldU16("nr_of_central_directory_records_on_disk")
|
|
d.FieldU16("nr_of_central_directory_records")
|
|
sizeCD = d.FieldU32("size_of_central directory")
|
|
offsetCD = d.FieldU32("offset_of_start_of_central_directory")
|
|
commentLength := d.FieldU16("comment_length")
|
|
d.FieldUTF8("comment", int(commentLength))
|
|
})
|
|
|
|
var localFileOffsets []uint64
|
|
|
|
d.SeekAbs(int64(offsetCD) * 8)
|
|
d.FieldArray("central_directories", func(d *decode.D) {
|
|
d.FramedFn(int64(sizeCD)*8, func(d *decode.D) {
|
|
for !d.End() {
|
|
d.FieldStruct("central_directory", func(d *decode.D) {
|
|
d.FieldRawLen("signature", 4*8, d.AssertBitBuf(centralDirectorySignature))
|
|
d.FieldU16("version_made_by")
|
|
d.FieldU16("version_needed")
|
|
d.FieldStruct("flags", func(d *decode.D) {
|
|
// TODO: 16LE, should have some kind of native endian flag reader helper?
|
|
d.FieldU1("unused0")
|
|
d.FieldBool("strong_encryption")
|
|
d.FieldBool("compressed_patched_data")
|
|
d.FieldBool("enhanced_deflation")
|
|
d.FieldBool("data_descriptor")
|
|
d.FieldBool("compression0")
|
|
d.FieldBool("compression1")
|
|
d.FieldBool("encrypted")
|
|
|
|
d.FieldU2("reserved0")
|
|
d.FieldBool("mask_header_values")
|
|
d.FieldBool("reserved1")
|
|
d.FieldBool("language_encoding")
|
|
d.FieldU3("unused1")
|
|
})
|
|
d.FieldU16("compression_method", compressionMethodMap)
|
|
d.FieldStruct("last_modification_date", fieldMSDOSTime)
|
|
d.FieldStruct("last_modification_time", fieldMSDOSDate)
|
|
d.FieldU32("crc32_uncompressed", scalar.ActualHex)
|
|
d.FieldU32("compressed_size")
|
|
d.FieldU32("uncompressed_size")
|
|
fileNameLength := d.FieldU16("file_name_length")
|
|
extraFieldLength := d.FieldU16("extra_field_length")
|
|
fileCommentLength := d.FieldU16("file_comment_length")
|
|
diskNrStart := d.FieldU16("disk_number_where_file_starts")
|
|
d.FieldU16("internal_file_attributes")
|
|
d.FieldU32("external_file_attributes")
|
|
localFileOffset := d.FieldU32("relative_offset_of_local_file_header")
|
|
d.FieldUTF8("file_name", int(fileNameLength))
|
|
d.FieldArray("extra_fields", func(d *decode.D) {
|
|
d.FramedFn(int64(extraFieldLength)*8, func(d *decode.D) {
|
|
for !d.End() {
|
|
d.FieldStruct("extra_field", func(d *decode.D) {
|
|
d.FieldU16("header_id", headerIDMap, scalar.ActualHex)
|
|
dataSize := d.FieldU16("data_size")
|
|
d.FieldRawLen("data", int64(dataSize)*8)
|
|
})
|
|
}
|
|
})
|
|
})
|
|
d.FieldUTF8("file_comment", int(fileCommentLength))
|
|
|
|
if diskNrStart == diskNr {
|
|
localFileOffsets = append(localFileOffsets, localFileOffset)
|
|
}
|
|
})
|
|
}
|
|
})
|
|
})
|
|
|
|
d.FieldArray("local_files", func(d *decode.D) {
|
|
for _, o := range localFileOffsets {
|
|
d.SeekAbs(int64(o) * 8)
|
|
d.FieldStruct("local_file", func(d *decode.D) {
|
|
var hasDataDescriptor bool
|
|
d.FieldRawLen("signature", 4*8, d.AssertBitBuf(localFileSignature))
|
|
d.FieldU16("version_needed")
|
|
d.FieldStruct("flags", func(d *decode.D) {
|
|
// TODO: 16LE, should have some kind of native endian flag reader helper?
|
|
d.FieldU1("unused0")
|
|
d.FieldBool("strong_encryption")
|
|
d.FieldBool("compressed_patched_data")
|
|
d.FieldBool("enhanced_deflation")
|
|
hasDataDescriptor = d.FieldBool("data_descriptor")
|
|
d.FieldBool("compression0")
|
|
d.FieldBool("compression1")
|
|
d.FieldBool("encrypted")
|
|
|
|
d.FieldU2("reserved0")
|
|
d.FieldBool("mask_header_values")
|
|
d.FieldBool("reserved1")
|
|
d.FieldBool("language_encoding")
|
|
d.FieldU3("unused1")
|
|
})
|
|
compressionMethod := d.FieldU16("compression_method", compressionMethodMap)
|
|
d.FieldStruct("last_modification_date", fieldMSDOSTime)
|
|
d.FieldStruct("last_modification_time", fieldMSDOSDate)
|
|
d.FieldU32("crc32_uncompressed", scalar.ActualHex)
|
|
compressedSizeBytes := d.FieldU32("compressed_size")
|
|
d.FieldU32("uncompressed_size")
|
|
fileNameLength := d.FieldU16("file_name_length")
|
|
extraFieldLength := d.FieldU16("extra_field_length")
|
|
d.FieldUTF8("file_name", int(fileNameLength))
|
|
d.FieldArray("extra_fields", func(d *decode.D) {
|
|
d.FramedFn(int64(extraFieldLength)*8, func(d *decode.D) {
|
|
for !d.End() {
|
|
d.FieldStruct("extra_field", func(d *decode.D) {
|
|
d.FieldU16("header_id", headerIDMap, scalar.ActualHex)
|
|
dataSize := d.FieldU16("data_size")
|
|
d.FieldRawLen("data", int64(dataSize)*8)
|
|
})
|
|
}
|
|
})
|
|
})
|
|
compressedSize := int64(compressedSizeBytes) * 8
|
|
compressedStart := d.Pos()
|
|
|
|
compressedLimit := compressedSize
|
|
if compressedLimit == 0 {
|
|
compressedLimit = d.BitsLeft()
|
|
}
|
|
|
|
if compressionMethod == compressionMethodNone {
|
|
d.FieldFormatOrRawLen("uncompressed", compressedSize, probeFormat, nil)
|
|
} else {
|
|
var rFn func(r io.Reader) io.Reader
|
|
switch compressionMethod {
|
|
case compressionMethodDeflated:
|
|
// bitio.NewIOReadSeeker implements io.ByteReader so that deflate don't do own
|
|
// buffering and might read more than needed messing up knowing compressed size
|
|
rFn = func(r io.Reader) io.Reader { return flate.NewReader(r) }
|
|
}
|
|
|
|
if rFn != nil {
|
|
readCompressedSize, uncompressedBR, dv, _, _ := d.TryFieldReaderRangeFormat("uncompressed", d.Pos(), compressedLimit, rFn, probeFormat, nil)
|
|
if dv == nil && uncompressedBR != nil {
|
|
d.FieldRootBitBuf("uncompressed", uncompressedBR)
|
|
}
|
|
if compressedSize == 0 {
|
|
compressedSize = readCompressedSize
|
|
}
|
|
d.FieldRawLen("compressed", compressedSize)
|
|
|
|
} else {
|
|
if compressedSize != 0 {
|
|
d.FieldRawLen("compressed", compressedSize)
|
|
}
|
|
}
|
|
}
|
|
|
|
d.SeekAbs(compressedStart + compressedSize)
|
|
|
|
if hasDataDescriptor {
|
|
d.FieldStruct("data_indicator", func(d *decode.D) {
|
|
if bytes.Equal(d.PeekBytes(4), dataIndicatorSignature) {
|
|
d.FieldRawLen("signature", 4*8, d.AssertBitBuf(dataIndicatorSignature))
|
|
}
|
|
d.FieldU32("crc32_uncompressed", scalar.ActualHex)
|
|
d.FieldU32("compressed_size")
|
|
d.FieldU32("uncompressed_size")
|
|
})
|
|
}
|
|
})
|
|
}
|
|
})
|
|
|
|
return nil
|
|
}
|