diff --git a/Tests/WhisperKitTests/RegressionTestUtils.swift b/Tests/WhisperKitTests/RegressionTestUtils.swift index 9d88fe6..07a25db 100644 --- a/Tests/WhisperKitTests/RegressionTestUtils.swift +++ b/Tests/WhisperKitTests/RegressionTestUtils.swift @@ -54,6 +54,7 @@ class TestInfo: JSONCodable { let datasetDir: String let datasetRepo: String let model: String + let modelSizeMB: Double let date: String let timeElapsedInSeconds: TimeInterval let timings: TranscriptionTimings? @@ -68,6 +69,7 @@ class TestInfo: JSONCodable { datasetDir: String, datasetRepo: String, model: String, + modelSizeMB: Double, date: String, timeElapsedInSeconds: TimeInterval, timings: TranscriptionTimings?, @@ -81,6 +83,7 @@ class TestInfo: JSONCodable { self.datasetDir = datasetDir self.datasetRepo = datasetRepo self.model = model + self.modelSizeMB = modelSizeMB self.date = date self.timeElapsedInSeconds = timeElapsedInSeconds self.timings = timings diff --git a/Tests/WhisperKitTests/RegressionTests.swift b/Tests/WhisperKitTests/RegressionTests.swift index d2309b8..29b8c33 100644 --- a/Tests/WhisperKitTests/RegressionTests.swift +++ b/Tests/WhisperKitTests/RegressionTests.swift @@ -287,12 +287,15 @@ final class RegressionTests: XCTestCase { generatedTranscript: transcriptionResult.text ) + let modelSizeMB = try? getFolderSize(atUrl: whisperKit.modelFolder) + let testInfo = TestInfo( device: device, audioFile: URL(fileURLWithPath: audioFilePath).lastPathComponent, datasetDir: config.dataset, datasetRepo: datasetRepo, model: config.model, + modelSizeMB: modelSizeMB ?? -1, date: startTime.formatted(Date.ISO8601FormatStyle().dateSeparator(.dash)), timeElapsedInSeconds: Date().timeIntervalSince(startTime), timings: transcriptionResult.timings, @@ -598,6 +601,15 @@ final class RegressionTests: XCTestCase { return cacheSize / (1024 * 1024) // Convert to MB } + private func getFolderSize(atUrl folder: URL?) throws -> Double { + guard let folder = folder else { + return -1 + } + let fileManager = FileManager.default + let modelSize = try fileManager.allocatedSizeOfDirectory(at: folder) + return Double(modelSize / (1024 * 1024)) // Convert to MB + } + func createWithMemoryCheck( model: String, computeOptions: ModelComputeOptions, diff --git a/fastlane/Fastfile b/fastlane/Fastfile index a575b0b..c83341f 100644 --- a/fastlane/Fastfile +++ b/fastlane/Fastfile @@ -52,7 +52,7 @@ BENCHMARK_CONFIGS = { debug: { test_identifier: 'WhisperAXTests/RegressionTests/testModelPerformanceWithDebugConfig', name: 'debug', - models: ['small.en'] # , 'crash_test', 'unknown_model', 'tiny'] + models: ['tiny', 'crash_test', 'unknown_model', 'small.en'] } }.freeze @@ -385,7 +385,7 @@ def prepare_upload(files) upload_folder = File.expand_path(BASE_UPLOAD_PATH) # Clear out the existing upload folder - if Dir.exist?(upload_folder) && !Dir.glob("#{upload_folder}").empty? + if Dir.exist?(upload_folder) && !Dir.glob(upload_folder.to_s).empty? UI.message "Clearing existing upload folder: #{upload_folder}" sh("trash #{upload_folder}/*") else