ios: deflate the text entries in the shared scan zip

ios/HouseScanKit/Sources/HouseScanKit/Scene/ZipWriter.swift

@@ -1,7 +1,13 @@
 import Foundation
+#if canImport(Compression)
+import Compression
+#endif
 
-// Store-only ZIP writer for the scan bundle (scene.json plus JPEGs). JPEGs do not compress, so
-// method 0 (stored) costs almost nothing and keeps the writer free of a deflate dependency.
+// ZIP writer for the scan bundle: JPEGs, motion CSVs, manifest.json, scene.json and on LiDAR
+// phones a mesh and depth maps. Entries that shrink are deflated (method 8) with Apple's
+// Compression framework, whose ZLIB algorithm writes raw RFC 1951 DEFLATE, the stream ZIP stores.
+// JPEGs skip the attempt, since they do not compress, and anything that would not get smaller is
+// stored (method 0). Without the Compression framework every entry is stored.
 // Layout per PKWARE APPNOTE 6.3.x: local header + data for each file, then the central directory,
 // then the end-of-central-directory record. No ZIP64: anything that would need it throws.
 
@@ -56,10 +62,11 @@ public enum ZipCRC32 {
 }
 
 public enum ZipWriter {
-    /// Builds a stored (uncompressed) ZIP archive in memory.
+    /// Builds a ZIP archive in memory, deflating each entry that shrinks and storing the rest.
     /// - Parameters:
     ///   - modified: timestamp written for every entry; nil writes the DOS epoch 1980-01-01 00:00 so
-    ///     the same inputs give byte-identical archives.
+    ///     the same inputs give byte-identical archives on the same OS build (the DEFLATE bytes come
+    ///     from Apple's encoder, which can change between releases).
     ///   - timeZone: zone the DOS local time is expressed in.
     public static func archive(_ entries: [ZipEntry], modified: Date? = nil, timeZone: TimeZone = .gmt) throws -> Data {
         var out = Data()
@@ -68,7 +75,8 @@ public enum ZipWriter {
     }
 
     /// Writes the same archive as `archive` to `url`, loading one entry at a time, so memory holds
-    /// one file and the central directory instead of the whole bundle.
+    /// one file, its compressed copy and the central directory instead of the whole bundle. A
+    /// compressed copy is never allowed to reach the file's own size.
     ///
     /// The archive is written to a temporary file beside `url` (`temporaryName(for:)`) and moved
     /// over `url` only once it is complete, so a file already at `url` stays whole until then: a
@@ -134,35 +142,40 @@ public enum ZipWriter {
             guard written <= 0xFFFF_FFFF else { throw ZipWriterError.archiveTooLarge }
 
             let offset = UInt32(written)
+            // The CRC and the uncompressed size describe the raw data whichever method is used.
             let crc = ZipCRC32.checksum(data)
             let size = UInt32(data.count)
+            let packed = isPrecompressed(entry.name) ? nil : deflated(data)
+            let method = packed == nil ? methodStored : methodDeflated
+            let body = packed ?? data
+            let compressedSize = UInt32(body.count)
 
             var local = Data()
             local.appendLE(UInt32(0x0403_4B50))  // local file header signature
             local.appendLE(versionNeeded)
             local.appendLE(flagUTF8)
-            local.appendLE(UInt16(0))  // method: stored
+            local.appendLE(method)
             local.appendLE(dosTime)
             local.appendLE(dosDate)
             local.appendLE(crc)
-            local.appendLE(size)  // compressed size
+            local.appendLE(compressedSize)
             local.appendLE(size)  // uncompressed size
             local.appendLE(UInt16(name.count))
             local.appendLE(UInt16(0))  // extra field length
             local.append(name)
             try emit(local)
-            try emit(data)
-            written += local.count + data.count
+            try emit(body)
+            written += local.count + body.count
 
             central.appendLE(UInt32(0x0201_4B50))  // central directory header signature
             central.appendLE(versionMadeBy)
             central.appendLE(versionNeeded)
             central.appendLE(flagUTF8)
-            central.appendLE(UInt16(0))
+            central.appendLE(method)
             central.appendLE(dosTime)
             central.appendLE(dosDate)
             central.appendLE(crc)
-            central.appendLE(size)
+            central.appendLE(compressedSize)
             central.appendLE(size)
             central.appendLE(UInt16(name.count))
             central.appendLE(UInt16(0))  // extra field length
@@ -188,15 +201,61 @@ public enum ZipWriter {
         try emit(tail)
     }
 
-    /// 2.0: the lowest version that defines the UTF-8 name flag readers check against.
+    /// 2.0: the lowest version that defines the UTF-8 name flag readers check against, and the
+    /// version APPNOTE asks for when an entry is deflated.
     private static let versionNeeded: UInt16 = 20
+    private static let methodStored: UInt16 = 0
+    private static let methodDeflated: UInt16 = 8
+    /// Already-compressed formats: deflating them costs time and saves nothing.
+    private static let precompressedSuffixes = [".jpg", ".jpeg", ".heic", ".png"]
     /// High byte 3 = Unix, so readers apply the mode in the external attributes.
     private static let versionMadeBy: UInt16 = (3 << 8) | 20
     /// General purpose bit 11: names are UTF-8.
     private static let flagUTF8: UInt16 = 1 << 11
     /// S_IFREG | 0644.
     private static let regularFileMode: UInt32 = 0o100644
 
+    private static func isPrecompressed(_ name: String) -> Bool {
+        let lower = name.lowercased()
+        return precompressedSuffixes.contains { lower.hasSuffix($0) }
+    }
+
+    /// Raw DEFLATE of `data` when that is strictly smaller than `data`, else nil (store it).
+    /// Streams through a fixed 64 KiB buffer, so the only memory added next to the loaded entry is
+    /// the compressed output, and gives up as soon as that output is no smaller than the input.
+    private static func deflated(_ data: Data) -> Data? {
+        #if canImport(Compression)
+        guard data.count > 1 else { return nil }
+        let chunk = 64 * 1024
+        let buffer = UnsafeMutablePointer<UInt8>.allocate(capacity: chunk)
+        defer { buffer.deallocate() }
+        let stream = UnsafeMutablePointer<compression_stream>.allocate(capacity: 1)
+        defer { stream.deallocate() }
+        // COMPRESSION_ZLIB is raw RFC 1951 DEFLATE, no zlib header: ZIP method 8.
+        guard compression_stream_init(stream, COMPRESSION_STREAM_ENCODE, COMPRESSION_ZLIB) == COMPRESSION_STATUS_OK else {
+            return nil
+        }
+        defer { compression_stream_destroy(stream) }
+        return data.withUnsafeBytes { (src: UnsafeRawBufferPointer) -> Data? in
+            guard let srcBase = src.bindMemory(to: UInt8.self).baseAddress else { return nil }
+            stream.pointee.src_ptr = srcBase
+            stream.pointee.src_size = data.count
+            var out = Data()
+            while true {
+                stream.pointee.dst_ptr = buffer
+                stream.pointee.dst_size = chunk
+                let status = compression_stream_process(stream, Int32(COMPRESSION_STREAM_FINALIZE.rawValue))
+                guard status != COMPRESSION_STATUS_ERROR else { return nil }
+                out.append(buffer, count: chunk - stream.pointee.dst_size)
+                guard out.count < data.count else { return nil }  // no saving: store it
+                if status == COMPRESSION_STATUS_END { return out }
+            }
+        }
+        #else
+        return nil
+        #endif
+    }
+
     private static func dosTimestamp(_ date: Date?, timeZone: TimeZone) throws -> (time: UInt16, date: UInt16) {
         guard let date else { return (0, (1 << 5) | 1) }  // 1980-01-01 00:00:00
         var calendar = Calendar(identifier: .gregorian)

ios/HouseScanKit/Tests/HouseScanKitTests/ZipWriterTests.swift

@@ -7,8 +7,72 @@ import Testing
         ZipEntry(name: "scene.json", data: Data(#"{"schema_version":"1.0"}"#.utf8)),
         ZipEntry(name: "k1.jpg", data: Data((0..<5000).map { UInt8($0 % 251) })),
         ZipEntry(name: "stills/meter_close.jpg", data: Data()),
+        ZipEntry(name: "streams/motion.csv", data: ZipWriterTests.motionCSV(rows: 2000)),
     ]
 
+    /// Synthetic 100 Hz motion rows shaped like a packet stream: text that deflates well.
+    static func motionCSV(rows: Int) -> Data {
+        var text = "t,qx,qy,qz,qw,ax,ay,az\n"
+        for i in 0..<rows {
+            let t = Double(i) / 100
+            text += "\(t),0.0123,\(0.5 + Double(i % 7) / 1000),-0.0042,0.8660,0.001,-0.981,\(Double(i % 13) / 100)\n"
+        }
+        return Data(text.utf8)
+    }
+
+    /// Bytes from a fixed linear congruential generator: no redundancy for DEFLATE to remove.
+    static func noise(count: Int) -> Data {
+        var state: UInt32 = 0x1234_5678
+        return Data((0..<count).map { _ -> UInt8 in
+            state = state &* 1_664_525 &+ 1_013_904_223
+            return UInt8(truncatingIfNeeded: state >> 24)
+        })
+    }
+
+    /// One entry as the local header and the central directory both describe it.
+    struct Parsed {
+        var name: String
+        var method: UInt16
+        var crc: UInt32
+        var compressedSize: UInt32
+        var size: UInt32
+        var body: [UInt8]
+    }
+
+    /// Walks the central directory from the end record and reads each local header it points to,
+    /// checking the two agree on method, CRC and sizes.
+    static func parse(_ archive: Data) throws -> [Parsed] {
+        let bytes = [UInt8](archive)
+        func u16(_ at: Int) -> UInt16 { UInt16(bytes[at]) | UInt16(bytes[at + 1]) << 8 }
+        func u32(_ at: Int) -> UInt32 { UInt32(u16(at)) | UInt32(u16(at + 2)) << 16 }
+        let end = bytes.count - 22  // no archive comment
+        try #require(end >= 0 && u32(end) == 0x0605_4B50)
+        let count = Int(u16(end + 10))
+        var at = Int(u32(end + 16))
+        var parsed: [Parsed] = []
+        for _ in 0..<count {
+            try #require(u32(at) == 0x0201_4B50)
+            let nameLength = Int(u16(at + 28))
+            let name = String(decoding: bytes[(at + 46)..<(at + 46 + nameLength)], as: UTF8.self)
+            let entry = Parsed(
+                name: name, method: u16(at + 10), crc: u32(at + 16),
+                compressedSize: u32(at + 20), size: u32(at + 24), body: []
+            )
+            let local = Int(u32(at + 42))
+            try #require(u32(local) == 0x0403_4B50)
+            #expect(u16(local + 8) == entry.method, "\(name)")
+            #expect(u32(local + 14) == entry.crc, "\(name)")
+            #expect(u32(local + 18) == entry.compressedSize, "\(name)")
+            #expect(u32(local + 22) == entry.size, "\(name)")
+            let start = local + 30 + Int(u16(local + 26)) + Int(u16(local + 28))
+            var withBody = entry
+            withBody.body = Array(bytes[start..<(start + Int(entry.compressedSize))])
+            parsed.append(withBody)
+            at += 46 + nameLength + Int(u16(at + 30)) + Int(u16(at + 32))
+        }
+        return parsed
+    }
+
     @Test func crc32CheckValue() {
         // The standard CRC-32 check value for the ASCII string "123456789".
         #expect(ZipCRC32.checksum(Data("123456789".utf8)) == 0xCBF4_3926)
@@ -43,6 +107,64 @@ import Testing
         #expect(!FileManager.default.fileExists(atPath: url.path))
     }
 
+    #if canImport(Compression)
+    /// Text deflates (method 8) and inflates back to the original; the CRC and the uncompressed
+    /// size describe the raw data.
+    @Test func textEntryIsDeflated() throws {
+        let csv = Self.motionCSV(rows: 2000)
+        let parsed = try Self.parse(ZipWriter.archive([ZipEntry(name: "streams/motion.csv", data: csv)]))
+        let entry = try #require(parsed.first)
+        #expect(entry.method == 8)
+        #expect(entry.size == UInt32(csv.count))
+        #expect(entry.compressedSize < entry.size / 2, "\(entry.compressedSize) of \(entry.size)")
+        #expect(entry.crc == ZipCRC32.checksum(csv))
+        // NSData's zlib algorithm is the same raw DEFLATE, so it inflates the entry as a reader would.
+        let inflated = try (Data(entry.body) as NSData).decompressed(using: .zlib) as Data
+        #expect(inflated == csv)
+    }
+
+    /// An entry whose compressed form spans several of the encoder's 64 KiB output chunks still
+    /// deflates and round-trips, and large noise is stored even though its first chunks were
+    /// already encoded.
+    @Test func largeEntriesCrossOutputChunks() throws {
+        let halfEntropy = Data(Self.noise(count: 400_000).map { $0 & 0x0F })  // about 200 KB deflated
+        let noise = Self.noise(count: 400_000)
+        let parsed = try Self.parse(ZipWriter.archive([
+            ZipEntry(name: "depth/k1.bin", data: halfEntropy),
+            ZipEntry(name: "depth/k2.bin", data: noise),
+        ]))
+        try #require(parsed.count == 2)
+        #expect(parsed[0].method == 8)
+        #expect(parsed[0].compressedSize > 128 * 1024, "\(parsed[0].compressedSize)")
+        #expect(parsed[0].compressedSize < parsed[0].size)
+        let inflated = try (Data(parsed[0].body) as NSData).decompressed(using: .zlib) as Data
+        #expect(inflated == halfEntropy)
+        #expect(parsed[1].method == 0)
+        #expect(Data(parsed[1].body) == noise)
+    }
+    #endif
+
+    /// JPEGs are stored even when their bytes would deflate, and so is anything that wouldn't
+    /// shrink: random bytes, a single byte, an empty file.
+    @Test func incompressibleEntriesAreStored() throws {
+        let compressibleJPEG = Data(repeating: 0x41, count: 4096)
+        let noise = Self.noise(count: 4096)
+        let entries = [
+            ZipEntry(name: "photos/k1.jpg", data: compressibleJPEG),
+            ZipEntry(name: "stills/Meter.JPG", data: compressibleJPEG),
+            ZipEntry(name: "depth/k1.f32", data: noise),
+            ZipEntry(name: "a", data: Data([1])),
+            ZipEntry(name: "empty", data: Data()),
+        ]
+        let parsed = try Self.parse(ZipWriter.archive(entries))
+        #expect(parsed.map(\.name) == entries.map(\.name))
+        for (entry, original) in zip(parsed, entries) {
+            #expect(entry.method == 0, "\(entry.name)")
+            #expect(entry.compressedSize == entry.size, "\(entry.name)")
+            #expect(Data(entry.body) == original.data, "\(entry.name)")
+        }
+    }
+
     @Test func rejectsBadNames() {
         #expect(throws: ZipWriterError.duplicateName("a")) {
             try ZipWriter.archive([ZipEntry(name: "a", data: Data()), ZipEntry(name: "a", data: Data())])