Merge pull request #137 from Infomaniak/chunking

feat: Move chunking to core

Package.resolved

@@ -68,8 +68,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/getsentry/sentry-cocoa",
       "state" : {
-        "revision" : "d2ced2d961b34573ebd2ea0567a2f1408e90f0ae",
-        "version" : "8.34.0"
+        "revision" : "54cc2e3e4fcbf9d4c7708ce00d3b6eee29aecbb1",
+        "version" : "8.38.0"
       }
     },
     {

Sources/InfomaniakCore/Chunking/ChunkProvider.swift

@@ -0,0 +1,115 @@
+/*
+ Infomaniak kDrive - iOS App
+ Copyright (C) 2021 Infomaniak Network SA
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import Foundation
+
+/// Something that builds chunks and provide them with an iterator.
+public protocol ChunkProvidable: IteratorProtocol {
+    init?(fileURL: URL, ranges: [DataRange])
+}
+
+/// Something that can chunk a file part by part, in memory, given specified ranges.
+///
+/// Memory considerations: Max memory use ≈sizeOf(one chunk). So from 1Mb to 50Mb
+/// Thread safety: Not thread safe
+///
+@available(macOS 10.15.4, iOS 13.4, watchOS 6.2, tvOS 13.4, *)
+public final class ChunkProvider: ChunkProvidable {
+    public typealias Element = Data
+
+    let fileHandle: FileHandlable
+
+    var ranges: [DataRange]
+
+    deinit {
+        do {
+            // For the sake of consistency
+            try fileHandle.close()
+        } catch {}
+    }
+
+    public init?(fileURL: URL, ranges: [DataRange]) {
+        self.ranges = ranges
+
+        do {
+            fileHandle = try FileHandle(forReadingFrom: fileURL)
+        } catch {
+            return nil
+        }
+    }
+
+    /// Internal testing method
+    init(mockedHandlable: FileHandlable, ranges: [DataRange]) {
+        self.ranges = ranges
+        fileHandle = mockedHandlable
+    }
+
+    /// Will provide chunks one by one, using the IteratorProtocol
+    /// Starting by the first range available.
+    public func next() -> Data? {
+        guard !ranges.isEmpty else {
+            return nil
+        }
+
+        let range = ranges.removeFirst()
+
+        do {
+            let chunk = try readChunk(range: range)
+            return chunk
+        } catch {
+            return nil
+        }
+    }
+
+    // MARK: Internal
+
+    func readChunk(range: DataRange) throws -> Data? {
+        let offset = range.lowerBound
+        try fileHandle.seek(toOffset: offset)
+
+        let byteCount = Int(range.upperBound - range.lowerBound) + 1
+        let chunk = try fileHandle.read(upToCount: byteCount)
+        return chunk
+    }
+}
+
+/// Print the FileHandle shows the current offset
+extension FileHandle {
+    override open var description: String {
+        if #available(macOS 10.15.4, iOS 13.4, watchOS 6.2, tvOS 13.4, *) {
+            let superDescription = super.description
+
+            let offsetString: String
+            do {
+                let offset = try offset()
+                offsetString = "\(offset)"
+            } catch {
+                offsetString = "\(error)"
+            }
+
+            let buffer = """
+            <\(superDescription)>
+            <offset:\(offsetString)>
+            """
+
+            return buffer
+        } else {
+            return super.description
+        }
+    }
+}

Sources/InfomaniakCore/Chunking/FileHandlable.swift

@@ -0,0 +1,46 @@
+/*
+ Infomaniak kDrive - iOS App
+ Copyright (C) 2023 Infomaniak Network SA
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import Foundation
+
+/// Something that matches most of the FileHandle specification, used for testing
+@available(macOS 10.15.4, iOS 13.4, watchOS 6.2, tvOS 13.4, *)
+protocol FileHandlable {
+    var availableData: Data { get }
+
+    var description: String { get }
+
+    func seek(toOffset offset: UInt64) throws
+
+    func truncate(atOffset offset: UInt64) throws
+
+    func synchronize() throws
+
+    func close() throws
+
+    func readToEnd() throws -> Data?
+
+    func read(upToCount count: Int) throws -> Data?
+
+    func offset() throws -> UInt64
+
+    func seekToEnd() throws -> UInt64
+}
+
+/// Protocol conformance
+extension FileHandle: FileHandlable {}

Sources/InfomaniakCore/Chunking/RangeProvider.swift

@@ -0,0 +1,110 @@
+/*
+ Infomaniak kDrive - iOS App
+ Copyright (C) 2021 Infomaniak Network SA
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import Foundation
+
+/// A range of Bytes on a `Data` buffer
+///   - start: start byte index, first index at 0 by convention.
+///   - end: end byte index, last index at fileSize -1 by convention.
+public typealias DataRange = ClosedRange<UInt64>
+
+/// Something that can provide a sequence of ranges where the file should be split if necessary.
+public protocol RangeProvidable {
+    /// Computes and return all the contiguous ranges for a file at the moment of calling.
+    ///
+    /// Result may change over time if file is modified in between calls.
+    /// Throws if file too large or too small, also if file system issue.
+    /// Minimum size support is one byte (low bound == high bound)
+    var allRanges: [DataRange] { get throws }
+
+    /// Return the file size in bytes at the moment of calling.
+    var fileSize: UInt64 { get throws }
+}
+
+public struct RangeProvider: RangeProvidable {
+    /// Encapsulating API parameters used to compute ranges
+    public enum APIConstants {
+        static let chunkMinSize: UInt64 = 1 * 1024 * 1024
+        static let chunkMaxSizeClient: UInt64 = 50 * 1024 * 1024
+        static let chunkMaxSizeServer: UInt64 = 1 * 1024 * 1024 * 1024
+        static let optimalChunkCount: UInt64 = 200
+        static let maxTotalChunks: UInt64 = 10000
+        static let minTotalChunks: UInt64 = 1
+
+        /// the limit supported by the app
+        static let fileMaxSizeClient = APIConstants.maxTotalChunks * APIConstants.chunkMaxSizeClient
+
+        /// the limit supported by the server
+        static let fileMaxSizeServer = APIConstants.maxTotalChunks * APIConstants.chunkMaxSizeServer
+    }
+
+    enum ErrorDomain: Error {
+        /// Unable to read file system metadata
+        case UnableToReadFileAttributes
+
+        /// file is over the supported size
+        case FileTooLarge
+
+        /// We ask for chunks that do not make sense
+        case ChunkedSizeLargerThanSourceFile
+
+        /// At least one chunk is expected
+        case IncorrectTotalChunksCount
+
+        /// A non zero size is expected
+        case IncorrectChunkSize
+    }
+
+    /// The internal methods split into another type, make testing easier
+    var guts: RangeProviderGutsable
+
+    public init(fileURL: URL) {
+        guts = RangeProviderGuts(fileURL: fileURL)
+    }
+
+    public var fileSize: UInt64 {
+        get throws {
+            let size = try guts.readFileByteSize()
+            return size
+        }
+    }
+
+    public var allRanges: [DataRange] {
+        get throws {
+            let size = try fileSize
+
+            // Check for files too large to be processed by mobile app or the server
+            guard size < APIConstants.fileMaxSizeClient,
+                  size < APIConstants.fileMaxSizeServer else {
+                // TODO: notify Sentry
+                throw ErrorDomain.FileTooLarge
+            }
+
+            let preferredChunkSize = guts.preferredChunkSize(for: size)
+
+            // Make sure an empty file resolves to one chunk
+            let totalChunksCount = max(size / max(preferredChunkSize, 1), 1)
+
+            let ranges = try guts.buildRanges(fileSize: size,
+                                              totalChunksCount: totalChunksCount,
+                                              chunkSize: preferredChunkSize)
+
+            return ranges
+        }
+    }
+}