RUMM-2819 Inject max object size to TLV

Sources/Datadog/DatadogCore/Storage/DataBlock.swift

@@ -9,8 +9,8 @@ import Foundation
 /// Block size binary type
 internal typealias BlockSize = UInt32
 
-/// Block max size (safety check) - 10 MB
-private let MAX_BLOCK_SIZE = 10 * 1_024 * 1_024
+/// Default max data lenght in block (safety check) - 10 MB
+private let MAX_DATA_LENGHT: UInt64 = 10 * 1_024 * 1_024
 
 /// Block type supported in data stream
 internal enum BlockType: UInt16 {
@@ -20,8 +20,9 @@ internal enum BlockType: UInt16 {
 /// Reported errors while manipulating data blocks.
 internal enum DataBlockError: Error {
     case readOperationFailed(streamError: Error?)
+    case invalidDataType
     case invalidByteSequence
-    case dataLengthExceedsLimit
+    case bytesLengthExceedsLimit(limit: UInt64)
     case dataAllocationFailure
     case endOfStream
 }
@@ -43,15 +44,15 @@ internal struct DataBlock {
     ///     +-  2 bytes -+-   4 bytes   -+- n bytes -|
     ///     | block type | data size (n) |    data   |
     ///     +------------+---------------+-----------+
-    ///
+    /// - Parameter maxLenght: Maximum data lenght of a block.
     /// - Returns: a data block in TLV.
-    func serialize() throws -> Data {
+    func serialize(maxLenght: UInt64 = MAX_DATA_LENGHT) throws -> Data {
         var buffer = Data()
         // T
         withUnsafeBytes(of: type.rawValue) { buffer.append(contentsOf: $0) }
         // L
-        guard let length = BlockSize(exactly: data.count), length < MAX_BLOCK_SIZE else {
-            throw DataBlockError.dataLengthExceedsLimit
+        guard let length = BlockSize(exactly: data.count), length <= maxLenght else {
+            throw DataBlockError.bytesLengthExceedsLimit(limit: maxLenght)
         }
         withUnsafeBytes(of: length) { buffer.append(contentsOf: $0) }
         // V
@@ -68,14 +69,25 @@ internal final class DataBlockReader {
     /// The input data stream.
     private let stream: InputStream
 
+    /// Maximum data lenght of a block.
+    private let maxBlockLenght: UInt64
+
     /// Reads block from data input.
     ///
     /// At initilization, the reader will open a stream targeting the input data.
     /// The stream will be closed when the reader instance is deallocated.
     ///
-    /// - Parameter data: The data input
-    convenience init(data: Data) {
-        self.init(input: InputStream(data: data))
+    /// - Parameters:
+    ///   - data: The data input
+    ///   - maxBlockLenght: Maximum data lenght of a block.
+    convenience init(
+        data: Data,
+        maxBlockLenght: UInt64 = MAX_DATA_LENGHT
+    ) {
+        self.init(
+            input: InputStream(data: data),
+            maxBlockLenght: maxBlockLenght
+        )
     }
 
     /// Reads block from an input stream.
@@ -84,7 +96,11 @@ internal final class DataBlockReader {
     /// when the reader instance is deallocated.
     ///
     /// - Parameter stream: The input stream
-    init(input stream: InputStream) {
+    init(
+        input stream: InputStream,
+        maxBlockLenght: UInt64 = MAX_DATA_LENGHT
+    ) {
+        self.maxBlockLenght = maxBlockLenght
         self.stream = stream
         stream.open()
     }
@@ -102,34 +118,25 @@ internal final class DataBlockReader {
     /// - Returns: The next block or nil if none could be found.
     func next() throws -> DataBlock? {
         // look for the next known block
-        while stream.hasBytesAvailable {
-            // read an entire block before inferring the data type
-            // to leave the stream in a usuable state if an unkown
-            // type was encountered.
-            let type: BlockType.RawValue
+        while true {
             do {
-                type = try readType()
+                return try readBlock()
+            } catch DataBlockError.invalidDataType {
+                continue
             } catch DataBlockError.endOfStream {
                 // Some streams won't return false for hasBytesAvailable until a read is attempted
                 return nil
             } catch {
                 throw error
             }
-            let data = try readData()
-
-            if let type = BlockType(rawValue: type) {
-                return DataBlock(type: type, data: data)
-            }
         }
-
-        return nil
     }
 
     /// Reads all data blocks from current index in the stream.
     ///
     /// - Throws: `DataBlockError` while reading the input stream.
     /// - Returns: The block sequence found in the input
-    func all() throws -> [DataBlock] {
+    func all(maxDataLenght: UInt64 = MAX_DATA_LENGHT) throws -> [DataBlock] {
         var blocks: [DataBlock] = []
 
         while let block = try next() {
@@ -151,8 +158,8 @@ internal final class DataBlockReader {
 
         // Load from stream directly to data without unnecessary copies
         var data = Data(count: length)
-        let count = try data.withUnsafeMutableBytes { (bytes: UnsafeMutableRawBufferPointer) in
-            guard let buffer = bytes.assumingMemoryBound(to: UInt8.self).baseAddress else {
+        let count = try data.withUnsafeMutableBytes {
+            guard let buffer = $0.assumingMemoryBound(to: UInt8.self).baseAddress else {
                 throw DataBlockError.dataAllocationFailure
             }
             return stream.read(buffer, maxLength: length)
@@ -173,6 +180,21 @@ internal final class DataBlockReader {
         return data
     }
 
+    /// Reads a block.
+    private func readBlock() throws -> DataBlock {
+        // read an entire block before inferring the data type
+        // to leave the stream in a usuable state if an unkown
+        // type was encountered.
+        let type = try readType()
+        let data = try readData()
+
+        guard let type = BlockType(rawValue: type) else {
+            throw DataBlockError.invalidDataType
+        }
+
+        return DataBlock(type: type, data: data)
+    }
+
     /// Reads a block type.
     private func readType() throws -> BlockType.RawValue {
         let data = try read(length: MemoryLayout<BlockType.RawValue>.size)
@@ -189,8 +211,8 @@ internal final class DataBlockReader {
         // length.
         // Additionally check that length hasn't been corrupted and
         // we don't try to generate a huge buffer.
-        guard let length = Int(exactly: size), length < MAX_BLOCK_SIZE else {
-            throw DataBlockError.dataLengthExceedsLimit
+        guard let length = Int(exactly: size), length <= maxBlockLenght else {
+            throw DataBlockError.bytesLengthExceedsLimit(limit: maxBlockLenght)
         }
 
         return try read(length: length)

Sources/Datadog/DatadogCore/Storage/Files/File.swift

@@ -27,16 +27,17 @@ internal protocol ReadableFile {
     /// Name of this file.
     var name: String { get }
 
-    /// Reads the available data in this file.
-    func read() throws -> Data
-
     /// Creates InputStream for reading the available data from this file.
-    func readStream() throws -> InputStream
+    func stream() throws -> InputStream
 
     /// Deletes this file.
     func delete() throws
 }
 
+private enum FileError: Error {
+    case unableToCreateInputStream
+}
+
 /// An immutable `struct` designed to provide optimized and thread safe interface for file manipulation.
 /// It doesn't own the file, which means the file presence is not guaranteed - the file can be deleted by OS at any time (e.g. due to memory pressure).
 internal struct File: WritableFile, ReadableFile {
@@ -93,48 +94,9 @@ internal struct File: WritableFile, ReadableFile {
         }
     }
 
-    func read() throws -> Data {
-        let fileHandle = try FileHandle(forReadingFrom: url)
-
-        // NOTE: RUMM-669
-        // https://github.com/DataDog/dd-sdk-ios/issues/214
-        // https://en.wikipedia.org/wiki/Xcode#11.x_series
-        // compiler version needs to have iOS 13.4+ as base SDK
-        #if compiler(>=5.2)
-        /**
-         Even though the `fileHandle.seekToEnd()` should be available since iOS 13.0:
-         ```
-         @available(OSX 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *)
-         public func readToEnd() throws -> Data?
-         ```
-         it crashes on iOS Simulators prior to iOS 13.4:
-         ```
-         Symbol not found: _$sSo12NSFileHandleC10FoundationE9readToEndAC4DataVSgyKF
-         ```
-        This is fixed in iOS 14/Xcode 12
-        */
-        if #available(iOS 13.4, tvOS 13.4, *) {
-            defer { try? fileHandle.close() }
-            return try fileHandle.readToEnd() ?? Data()
-        } else {
-            return try legacyRead(from: fileHandle)
-        }
-        #else
-        return try legacyRead(from: fileHandle)
-        #endif
-    }
-
-    private func legacyRead(from fileHandle: FileHandle) throws -> Data {
-        let data = fileHandle.readDataToEndOfFile()
-        try? objcExceptionHandler.rethrowToSwift {
-            fileHandle.closeFile()
-        }
-        return data
-    }
-
-    func readStream() throws -> InputStream {
+    func stream() throws -> InputStream {
         guard let stream = InputStream(url: url) else {
-            throw Errors.unableToCreateInputStream
+            throw FileError.unableToCreateInputStream
         }
         return stream
     }
@@ -148,7 +110,3 @@ internal struct File: WritableFile, ReadableFile {
         try FileManager.default.removeItem(at: url)
     }
 }
-
-private enum Errors: Error {
-    case unableToCreateInputStream
-}

Sources/Datadog/DatadogCore/Storage/FilesOrchestrator.swift

@@ -9,11 +9,12 @@ import Foundation
 /// Orchestrates files in a single directory.
 internal class FilesOrchestrator {
     /// Directory where files are stored.
-    private let directory: Directory
+    let directory: Directory
     /// Date provider.
-    private let dateProvider: DateProvider
+    let dateProvider: DateProvider
     /// Performance rules for writing and reading files.
-    private let performance: StoragePerformancePreset
+    let performance: StoragePerformancePreset
+
     /// Name of the last file returned by `getWritableFile()`.
     private var lastWritableFileName: String? = nil
     /// Tracks number of times the file at `lastWritableFileURL` was returned from `getWritableFile()`.
@@ -33,10 +34,6 @@ internal class FilesOrchestrator {
     // MARK: - `WritableFile` orchestration
 
     func getWritableFile(writeSize: UInt64) throws -> WritableFile {
-        if writeSize > performance.maxObjectSize {
-            throw InternalError(description: "data exceeds the maximum size of \(performance.maxObjectSize) bytes.")
-        }
-
         let lastWritableFileOrNil = reuseLastWritableFileIfPossible(writeSize: writeSize)
 
         if let lastWritableFile = lastWritableFileOrNil { // if last writable file can be reused

Sources/Datadog/DatadogCore/Storage/Reading/FileReader.swift

@@ -31,7 +31,7 @@ internal final class FileReader: Reader {
         }
 
         do {
-            let events = try decode(stream: file.readStream())
+            let events = try decode(stream: file.stream())
             return Batch(events: events, file: file)
         } catch {
             DD.telemetry.error("Failed to read data from file", error: error)
@@ -48,7 +48,10 @@ internal final class FileReader: Reader {
     /// - Parameter stream: The InputStream that provides data to decode.
     /// - Returns: The decoded and formatted data.
     private func decode(stream: InputStream) throws -> [Data] {
-        let reader = DataBlockReader(input: stream)
+        let reader = DataBlockReader(
+            input: stream,
+            maxBlockLenght: orchestrator.performance.maxObjectSize
+        )
 
         var failure: String? = nil
         defer {

Sources/Datadog/DatadogCore/Storage/Writing/FileWriter.swift

@@ -55,7 +55,9 @@ internal final class FileWriter: Writer {
         return try DataBlock(
             type: .event,
             data: encrypt(data: data)
-        ).serialize()
+        ).serialize(
+            maxLenght: orchestrator.performance.maxObjectSize
+        )
     }
 
     /// Encrypts data if encryption is available.

Tests/DatadogTests/Datadog/Core/Persistence/DataBlockTests.swift

@@ -96,27 +96,50 @@ class DataBlockTests: XCTestCase {
         XCTAssertEqual(block?.data, Data([0xFF, 0xFF]))
     }
 
-    func testDataBlockReader_readsLargeBytesBlock() throws {
-        let data = Data([0x00, 0x00, 0x80, 0x96, 0x98, 0x00]) + Data.mockRepeating(byte: 0xFF, times: 10_000_000) // 10MB
-        let reader = DataBlockReader(data: data)
+    func testDataBlockReader_readsBytesUnderLengthLimit() throws {
+        let data = Data([0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0xFF, 0xFF])
+        let reader = DataBlockReader(data: data, maxBlockLenght: 2)
 
         let block = try reader.next()
         XCTAssertEqual(block?.type, .event)
         XCTAssertEqual(block?.data.first, 0xFF)
-        XCTAssertEqual(block?.data.count, 10_000_000)
+        XCTAssertEqual(block?.data.count, 2)
     }
 
-    func testDataBlockReader_skipsVeryLargeBytesBlock() throws {
-        let data = Data([0x00, 0x00, 0x00, 0x2D, 0x31, 0x01]) + Data.mockRepeating(byte: 0xFF, times: 20_000_000) // 20MB
-        let reader = DataBlockReader(data: data)
+    func testDataBlockReader_skipsExceedingBytesLengthLimit() throws {
+        let data = Data([0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0xFF, 0xFF])
+        let reader = DataBlockReader(data: data, maxBlockLenght: 1)
 
         do {
             _ = try reader.next()
             XCTFail("Expected error to be thrown")
-        } catch DataBlockError.dataLengthExceedsLimit {
+        } catch DataBlockError.bytesLengthExceedsLimit(let limit) where limit == 1 {
             // Expected
         } catch {
             XCTFail("Unexpected error: \(error)")
         }
     }
+
+    func testDataBlockReader_whenIOErrorHappens_itThrowsWhenReading() throws {
+        temporaryDirectory.create()
+        defer { temporaryDirectory.delete() }
+
+        let file = try temporaryDirectory.createFile(named: "file")
+        try file.delete()
+
+        let stream = try file.stream()
+        let reader = DataBlockReader(input: stream)
+
+        do {
+            _ = try reader.next()
+            XCTFail("Expected error to be thrown")
+        } catch DataBlockError.readOperationFailed(let error) {
+            XCTAssertEqual(
+                (error as? NSError)?.localizedDescription,
+                "The operation couldn’t be completed. No such file or directory"
+            )
+        } catch {
+            XCTFail("Unexpected error: \(error)")
+        }
+    }
 }