From cab99e18bbe8d4f819f3b0cd522982b001168f44 Mon Sep 17 00:00:00 2001 From: Douglas Jarquin <8209+douglasjarquin@users.noreply.github.com> Date: Fri, 14 Aug 2026 20:27:48 -0400 Subject: [PATCH 1/2] feat(dictation): add native feasibility spike --- Package.swift | 35 ++ Sources/OigoSpike/CaptureJournal.swift | 147 ++++++ Sources/OigoSpike/Cleanup.swift | 139 ++++++ Sources/OigoSpike/Metrics.swift | 85 ++++ .../OigoSpike/NativeDictationPipeline.swift | 420 ++++++++++++++++++ Sources/OigoSpike/RecordValidator.swift | 41 ++ Sources/OigoSpike/TranscriptAccumulator.swift | 116 +++++ Sources/oigo-spike/main.swift | 271 +++++++++++ Tests/OigoSpikeContractTests/main.swift | 170 +++++++ docs/native-on-device-dictation.md | 111 +++++ 10 files changed, 1535 insertions(+) create mode 100644 Package.swift create mode 100644 Sources/OigoSpike/CaptureJournal.swift create mode 100644 Sources/OigoSpike/Cleanup.swift create mode 100644 Sources/OigoSpike/Metrics.swift create mode 100644 Sources/OigoSpike/NativeDictationPipeline.swift create mode 100644 Sources/OigoSpike/RecordValidator.swift create mode 100644 Sources/OigoSpike/TranscriptAccumulator.swift create mode 100644 Sources/oigo-spike/main.swift create mode 100644 Tests/OigoSpikeContractTests/main.swift create mode 100644 docs/native-on-device-dictation.md diff --git a/Package.swift b/Package.swift new file mode 100644 index 0000000..a4c235a --- /dev/null +++ b/Package.swift @@ -0,0 +1,35 @@ +// swift-tools-version: 6.3 + +import PackageDescription + +let package = Package( + name: "OigoSpike", + platforms: [ + .macOS("26.0") + ], + products: [ + .library(name: "OigoSpike", targets: ["OigoSpike"]), + .executable(name: "oigo-spike", targets: ["OigoSpikeCLI"]), + .executable(name: "oigo-spike-contract-tests", targets: ["OigoSpikeContractTests"]) + ], + targets: [ + .target( + name: "OigoSpike", + linkerSettings: [ + .linkedFramework("AVFAudio"), + .linkedFramework("FoundationModels"), + .linkedFramework("Speech") + ] + ), + .executableTarget( + name: "OigoSpikeCLI", + dependencies: ["OigoSpike"], + path: "Sources/oigo-spike" + ), + .executableTarget( + name: "OigoSpikeContractTests", + dependencies: ["OigoSpike"], + path: "Tests/OigoSpikeContractTests" + ) + ] +) diff --git a/Sources/OigoSpike/CaptureJournal.swift b/Sources/OigoSpike/CaptureJournal.swift new file mode 100644 index 0000000..eebe5fe --- /dev/null +++ b/Sources/OigoSpike/CaptureJournal.swift @@ -0,0 +1,147 @@ +import AVFAudio +import Foundation + +public enum CaptureJournalError: Error, Equatable, CustomStringConvertible { + case closed + case missing(URL) + + public var description: String { + switch self { + case .closed: + return "capture journal is closed" + case .missing(let url): + return "capture artifact is missing at \(url.path)" + } + } +} + +public final class DurableCaptureJournal: @unchecked Sendable { + public let url: URL + + private let lock = NSLock() + private var handle: FileHandle? + private(set) public var bytesWritten = 0 + + public init(url: URL) throws { + self.url = url + try FileManager.default.createDirectory( + at: url.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + FileManager.default.createFile(atPath: url.path, contents: nil) + handle = try FileHandle(forWritingTo: url) + } + + public func append(_ data: Data) throws { + lock.lock() + defer { lock.unlock() } + guard let handle else { + throw CaptureJournalError.closed + } + try handle.seekToEnd() + try handle.write(contentsOf: data) + try handle.synchronize() + bytesWritten += data.count + } + + public func finish() throws { + lock.lock() + defer { lock.unlock() } + guard let handle else { + return + } + try handle.synchronize() + try handle.close() + self.handle = nil + } + + public static func recover(_ url: URL) throws -> Data { + guard FileManager.default.fileExists(atPath: url.path) else { + throw CaptureJournalError.missing(url) + } + return try Data(contentsOf: url) + } + + deinit { + try? finish() + } +} + +@available(macOS 26.0, *) +public final class CAFRecorder: @unchecked Sendable { + public let url: URL + + private let lock = NSLock() + private var audioFile: AVAudioFile? + + public init(url: URL, format: AVAudioFormat) throws { + self.url = url + try FileManager.default.createDirectory( + at: url.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + audioFile = try AVAudioFile( + forWriting: url, + settings: format.settings, + commonFormat: format.commonFormat, + interleaved: format.isInterleaved + ) + } + + public func append(_ buffer: AVAudioPCMBuffer) throws { + lock.lock() + defer { lock.unlock() } + guard let audioFile else { + throw CaptureJournalError.closed + } + try audioFile.write(from: buffer) + } + + public func finish() throws { + lock.lock() + defer { lock.unlock() } + audioFile = nil + } + + public static func playableFrameLength(at url: URL) throws -> Int64 { + guard FileManager.default.fileExists(atPath: url.path) else { + throw CaptureJournalError.missing(url) + } + let audioFile = try AVAudioFile(forReading: url) + return Int64(audioFile.length) + } + + deinit { + try? finish() + } +} + +public struct ForcedRecognitionFailure: Error, Sendable, CustomStringConvertible { + public init() {} + + public var description: String { + "forced live recognition failure" + } +} + +public enum SavedAudioRetry { + public static func retry( + url: URL, + transcribe: (URL) throws -> T + ) throws -> T { + guard FileManager.default.fileExists(atPath: url.path) else { + throw CaptureJournalError.missing(url) + } + return try transcribe(url) + } + + public static func retryAfterFailure( + url: URL, + liveFailure: Error, + transcribe: (URL) throws -> T + ) throws -> T { + _ = liveFailure + _ = try CAFRecorder.playableFrameLength(at: url) + return try retry(url: url, transcribe: transcribe) + } +} diff --git a/Sources/OigoSpike/Cleanup.swift b/Sources/OigoSpike/Cleanup.swift new file mode 100644 index 0000000..528e11d --- /dev/null +++ b/Sources/OigoSpike/Cleanup.swift @@ -0,0 +1,139 @@ +import Foundation +import FoundationModels + +private final class CleanupResolution: @unchecked Sendable { + private let lock = NSLock() + private var continuation: CheckedContinuation? + private var resolvedRecord: CleanupRecord? + private var resolved = false + + func install(_ continuation: CheckedContinuation) { + lock.lock() + if let resolvedRecord { + lock.unlock() + continuation.resume(returning: resolvedRecord) + return + } + self.continuation = continuation + lock.unlock() + } + + func resolve(_ record: CleanupRecord) { + lock.lock() + guard !resolved else { + lock.unlock() + return + } + resolved = true + resolvedRecord = record + let continuation = self.continuation + lock.unlock() + continuation?.resume(returning: record) + } +} + +public enum CleanupStatus: Equatable, Sendable { + case cleaned + case unavailable(String) + case timedOut + case failed(String) +} + +public struct CleanupRecord: Equatable, Sendable { + public let rawText: String + public let cleanedText: String? + public let status: CleanupStatus + + public init(rawText: String, cleanedText: String?, status: CleanupStatus) { + self.rawText = rawText + self.cleanedText = cleanedText + self.status = status + } + + public var displayedText: String { + cleanedText ?? rawText + } +} + +public enum CleanupFallback { + public static func unavailable(rawText: String, reason: String) -> CleanupRecord { + CleanupRecord(rawText: rawText, cleanedText: nil, status: .unavailable(reason)) + } + + public static func timedOut(rawText: String) -> CleanupRecord { + CleanupRecord(rawText: rawText, cleanedText: nil, status: .timedOut) + } + + public static func failed(rawText: String, error: Error) -> CleanupRecord { + CleanupRecord(rawText: rawText, cleanedText: nil, status: .failed(String(describing: error))) + } + + public static func run( + rawText: String, + timeoutNanoseconds: UInt64, + operation: @escaping @Sendable () async throws -> String + ) async -> CleanupRecord { + let resolution = CleanupResolution() + let responseTask = Task { + do { + let cleanedText = try await operation() + resolution.resolve( + CleanupRecord( + rawText: rawText, + cleanedText: cleanedText, + status: .cleaned + ) + ) + } catch { + resolution.resolve(CleanupFallback.failed(rawText: rawText, error: error)) + } + } + let timeoutTask = Task { + do { + try await Task.sleep(nanoseconds: timeoutNanoseconds) + resolution.resolve(CleanupFallback.timedOut(rawText: rawText)) + } catch { + } + } + let result = await withCheckedContinuation { (continuation: CheckedContinuation) in + resolution.install(continuation) + } + responseTask.cancel() + timeoutTask.cancel() + return result + } +} + +@available(macOS 26.0, *) +public final class FoundationModelsCleaner: @unchecked Sendable { + private let signposts: PipelineSignposts? + + public init(signposts: PipelineSignposts? = nil) { + self.signposts = signposts + } + + public func clean( + rawText: String, + timeoutNanoseconds: UInt64 = 2_000_000_000 + ) async -> CleanupRecord { + defer { signposts?.mark(.cleanup) } + let model = SystemLanguageModel.default + guard model.isAvailable else { + return CleanupFallback.unavailable( + rawText: rawText, + reason: String(describing: model.availability) + ) + } + + let instructions = "Rewrite the transcript as a strict meaning-preserving cleanup. Do not add, remove, infer, or normalize technical terms. Return only the cleaned transcript." + let session = LanguageModelSession(model: model, instructions: instructions) + return await CleanupFallback.run( + rawText: rawText, + timeoutNanoseconds: timeoutNanoseconds + ) { + let response = try await session.respond(to: rawText) + return response.content + } + } + +} diff --git a/Sources/OigoSpike/Metrics.swift b/Sources/OigoSpike/Metrics.swift new file mode 100644 index 0000000..23f7166 --- /dev/null +++ b/Sources/OigoSpike/Metrics.swift @@ -0,0 +1,85 @@ +import Darwin +import Foundation +import os + +public enum SignpostEvent: String, Sendable { + case recordingStart = "recording-start" + case firstAudioBuffer = "first-audio-buffer" + case firstVolatileResult = "first-volatile-result" + case finalResult = "final-result" + case cleanup = "cleanup" + case resourceRelease = "resource-release" +} + +public final class PipelineSignposts: @unchecked Sendable { + private let log = OSLog(subsystem: "com.oigo.spike", category: "dictation") + + public init() {} + + public func mark(_ event: SignpostEvent) { + switch event { + case .recordingStart: + os_signpost(.event, log: log, name: "recording-start") + case .firstAudioBuffer: + os_signpost(.event, log: log, name: "first-audio-buffer") + case .firstVolatileResult: + os_signpost(.event, log: log, name: "first-volatile-result") + case .finalResult: + os_signpost(.event, log: log, name: "final-result") + case .cleanup: + os_signpost(.event, log: log, name: "cleanup") + case .resourceRelease: + os_signpost(.event, log: log, name: "resource-release") + } + } +} + +public struct ResourceMeasurement: Equatable, Sendable { + public let runs: Int + public let initialMaximumResidentBytes: UInt64 + public let finalMaximumResidentBytes: UInt64 + public let maximumResidentDeltaBytes: UInt64 + public let bounded: Bool + + public init( + runs: Int, + initialMaximumResidentBytes: UInt64, + finalMaximumResidentBytes: UInt64, + maximumResidentDeltaBytes: UInt64, + bounded: Bool + ) { + self.runs = runs + self.initialMaximumResidentBytes = initialMaximumResidentBytes + self.finalMaximumResidentBytes = finalMaximumResidentBytes + self.maximumResidentDeltaBytes = maximumResidentDeltaBytes + self.bounded = bounded + } +} + +public enum ResourceMeasurementRunner { + public static func measure( + runs: Int, + operation: () -> Void + ) -> ResourceMeasurement { + let safeRuns = max(1, runs) + let initial = maximumResidentBytes() + for _ in 0..= initial ? final - initial : 0 + return ResourceMeasurement( + runs: safeRuns, + initialMaximumResidentBytes: initial, + finalMaximumResidentBytes: final, + maximumResidentDeltaBytes: delta, + bounded: delta < 32 * 1024 * 1024 + ) + } + + private static func maximumResidentBytes() -> UInt64 { + var usage = rusage() + getrusage(RUSAGE_SELF, &usage) + return UInt64(max(0, usage.ru_maxrss)) + } +} diff --git a/Sources/OigoSpike/NativeDictationPipeline.swift b/Sources/OigoSpike/NativeDictationPipeline.swift new file mode 100644 index 0000000..d988b9b --- /dev/null +++ b/Sources/OigoSpike/NativeDictationPipeline.swift @@ -0,0 +1,420 @@ +import AVFAudio +import CoreMedia +import Foundation +import FoundationModels +import Speech + +public enum NativePipelineError: Error, CustomStringConvertible { + case microphonePermission(String) + case analysisFailed(String) + case missingApplicationBundle + case notRestartable + case alreadyRunning + case notRunning + + public var description: String { + switch self { + case .microphonePermission(let permission): + return "microphone permission is \(permission)" + case .analysisFailed(let message): + return "native dictation analysis failed: \(message)" + case .missingApplicationBundle: + return "native live capture requires an application bundle identifier" + case .notRestartable: + return "native dictation pipeline cannot be restarted after a start attempt" + case .alreadyRunning: + return "native dictation pipeline is already running" + case .notRunning: + return "native dictation pipeline is not running" + } + } +} + +@available(macOS 26.0, *) +public final class NativeDictationPipeline: @unchecked Sendable { + public let audioURL: URL + public let transcriberChoice = "DictationTranscriber.progressiveLongDictation" + + private let engine = AVAudioEngine() + private let transcriber: DictationTranscriber + private let analyzer: SpeechAnalyzer + private let inputStream: AsyncStream + private let inputContinuation: AsyncStream.Continuation + private let transcriptStore = TranscriptStore() + private let cleaner: FoundationModelsCleaner + private let signposts: PipelineSignposts + private let stateLock = NSLock() + private var audioFile: AVAudioFile? + private var analysisTask: Task? + private var resultTask: Task? + private var running = false + private var firstBufferSeen = false + private var firstVolatileSeen = false + private var failureDescription: String? + private var lifecycleClosed = false + private var snapshotHandler: (@Sendable (TranscriptSnapshot) -> Void)? + private(set) public var lastError: String? + + public init( + audioURL: URL, + locale: Locale = Locale(identifier: "en-US"), + signposts: PipelineSignposts = PipelineSignposts(), + onSnapshot: (@Sendable (TranscriptSnapshot) -> Void)? = nil + ) { + self.audioURL = audioURL + self.signposts = signposts + self.cleaner = FoundationModelsCleaner(signposts: signposts) + self.snapshotHandler = onSnapshot + let transcriber = DictationTranscriber( + locale: locale, + preset: .progressiveLongDictation + ) + self.transcriber = transcriber + let streamPair = AsyncStream.makeStream() + inputStream = streamPair.stream + inputContinuation = streamPair.continuation + let context = AnalysisContext() + context.contextualStrings[.general] = [ + "Consigliere", + "n8n", + "Claude Code", + "ChatGPT" + ] + analyzer = SpeechAnalyzer( + inputSequence: streamPair.stream, + modules: [transcriber], + options: SpeechAnalyzer.Options( + priority: .userInitiated, + modelRetention: .whileInUse + ), + analysisContext: context + ) + } + + public var latestSnapshot: TranscriptSnapshot { + transcriptStore.snapshot + } + + public func start() async throws { + try Self.requireApplicationBundle() + try beginRun() + do { + let permission = AVAudioApplication.shared.recordPermission + guard permission == .granted else { + throw NativePipelineError.microphonePermission(String(describing: permission)) + } + + let inputNode = engine.inputNode + let format = inputNode.outputFormat(forBus: 0) + try FileManager.default.createDirectory( + at: audioURL.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + let file = try AVAudioFile( + forWriting: audioURL, + settings: format.settings, + commonFormat: format.commonFormat, + interleaved: format.isInterleaved + ) + audioFile = file + + try await analyzer.prepareToAnalyze(in: format) + resultTask = Task { [weak self, transcriber] in + do { + for try await result in transcriber.results { + self?.handle(result) + } + } catch { + self?.record(error: error) + } + } + analysisTask = Task { [weak self, analyzer, inputStream] in + do { + try await analyzer.start(inputSequence: inputStream) + } catch { + self?.record(error: error) + } + } + + inputNode.installTap(onBus: 0, bufferSize: 1_024, format: format) { [weak self] buffer, _ in + guard let self else { + return + } + do { + try self.append(buffer) + self.inputContinuation.yield(AnalyzerInput(buffer: buffer)) + self.markFirstBufferIfNeeded() + } catch { + self.record(error: error) + } + } + + engine.prepare() + try engine.start() + signposts.mark(.recordingStart) + } catch { + await cleanupAfterStartFailure() + throw error + } + } + + public func stop() async throws { + try endRun() + + engine.inputNode.removeTap(onBus: 0) + engine.stop() + inputContinuation.finish() + var finalizationError: Error? + do { + try await analyzer.finalizeAndFinishThroughEndOfInput() + } catch { + record(error: error) + finalizationError = error + } + _ = await analysisTask?.value + _ = await resultTask?.value + clearAudioFile() + analysisTask = nil + resultTask = nil + signposts.mark(.resourceRelease) + await SpeechModels.endRetention() + if let finalizationError { + throw finalizationError + } + if let failureDescription { + throw NativePipelineError.analysisFailed(failureDescription) + } + } + + public func cleanFinalTranscript( + timeoutNanoseconds: UInt64 = 2_000_000_000 + ) async -> CleanupRecord { + await cleaner.clean( + rawText: latestSnapshot.finalizedText, + timeoutNanoseconds: timeoutNanoseconds + ) + } + + public func run(for seconds: TimeInterval) async throws { + try await start() + do { + try await Task.sleep(nanoseconds: UInt64(max(0, seconds) * 1_000_000_000)) + } catch { + try? await stop() + throw error + } + try await stop() + } + + public func setSnapshotHandler(_ handler: (@Sendable (TranscriptSnapshot) -> Void)?) { + stateLock.lock() + snapshotHandler = handler + stateLock.unlock() + } + + public static func requestMicrophonePermission() async -> Bool { + await withCheckedContinuation { continuation in + AVAudioApplication.requestRecordPermission { granted in + continuation.resume(returning: granted) + } + } + } + + public static func capabilitySnapshot( + locale: Locale = Locale(identifier: "en-US") + ) async -> [String: String] { + let module = DictationTranscriber(locale: locale, preset: .progressiveLongDictation) + let assetStatus = await AssetInventory.status(forModules: [module]) + let installedLocales = await DictationTranscriber.installedLocales + let model = SystemLanguageModel.default + return [ + "transcriber": "DictationTranscriber.progressiveLongDictation", + "host_bundle_identifier": Bundle.main.bundleIdentifier ?? "none", + "microphone_permission": microphonePermissionDescription(), + "speech_assets": String(describing: assetStatus), + "speech_installed_locales": installedLocales.map(\.identifier).joined(separator: ","), + "foundation_models_availability": String(describing: model.availability), + "foundation_models_available": String(model.isAvailable), + "network_calls_in_oigo_path": "none" + ] + } + + public static func installSpeechAssets( + locale: Locale = Locale(identifier: "en-US") + ) async throws -> String { + let module = DictationTranscriber(locale: locale, preset: .progressiveLongDictation) + guard let request = try await AssetInventory.assetInstallationRequest(supporting: [module]) else { + return String(describing: await AssetInventory.status(forModules: [module])) + } + try await request.downloadAndInstall() + return String(describing: await AssetInventory.status(forModules: [module])) + } + + public static func transcribeSavedAudio( + at url: URL, + locale: Locale = Locale(identifier: "en-US") + ) async throws -> TranscriptSnapshot { + try requireApplicationBundle() + let transcriber = DictationTranscriber( + locale: locale, + preset: .progressiveLongDictation + ) + let audioFile = try AVAudioFile(forReading: url) + let analyzer = try await SpeechAnalyzer( + inputAudioFile: audioFile, + modules: [transcriber], + options: SpeechAnalyzer.Options( + priority: .userInitiated, + modelRetention: .whileInUse + ), + finishAfterFile: true + ) + let store = TranscriptStore() + let resultTask = Task { + for try await result in transcriber.results { + let range = TranscriptRange( + startMilliseconds: Int64(result.range.start.seconds * 1_000), + endMilliseconds: Int64(result.range.end.seconds * 1_000) + ) + _ = store.ingest( + range: range, + text: String(result.text.characters), + isFinal: result.isFinal + ) + } + } + do { + try await analyzer.start(inputAudioFile: audioFile, finishAfterFile: true) + try await analyzer.finalizeAndFinishThroughEndOfInput() + try await resultTask.value + } catch { + resultTask.cancel() + _ = await resultTask.result + await SpeechModels.endRetention() + throw error + } + await SpeechModels.endRetention() + return store.snapshot + } + + private func append(_ buffer: AVAudioPCMBuffer) throws { + stateLock.lock() + defer { stateLock.unlock() } + try audioFile?.write(from: buffer) + } + + private func cleanupAfterStartFailure() async { + engine.inputNode.removeTap(onBus: 0) + engine.stop() + inputContinuation.finish() + analysisTask?.cancel() + resultTask?.cancel() + _ = await analysisTask?.value + _ = await resultTask?.value + analysisTask = nil + resultTask = nil + clearAudioFile() + markStartFailureClosed() + signposts.mark(.resourceRelease) + await SpeechModels.endRetention() + } + + private func clearAudioFile() { + stateLock.lock() + audioFile = nil + stateLock.unlock() + } + + private func markStartFailureClosed() { + stateLock.lock() + running = false + lifecycleClosed = true + stateLock.unlock() + } + + private func markFirstBufferIfNeeded() { + stateLock.lock() + let shouldMark = !firstBufferSeen + firstBufferSeen = true + stateLock.unlock() + if shouldMark { + signposts.mark(.firstAudioBuffer) + } + } + + private func handle(_ result: DictationTranscriber.Result) { + let range = TranscriptRange( + startMilliseconds: Int64(result.range.start.seconds * 1_000), + endMilliseconds: Int64(result.range.end.seconds * 1_000) + ) + let snapshot = transcriptStore.ingest( + range: range, + text: String(result.text.characters), + isFinal: result.isFinal + ) + stateLock.lock() + let handler = snapshotHandler + let shouldMarkVolatile = !result.isFinal && !firstVolatileSeen + if shouldMarkVolatile { + firstVolatileSeen = true + } + stateLock.unlock() + if shouldMarkVolatile { + signposts.mark(.firstVolatileResult) + } + if result.isFinal { + signposts.mark(.finalResult) + } + handler?(snapshot) + } + + private func record(error: Error) { + stateLock.lock() + lastError = String(describing: error) + if failureDescription == nil { + failureDescription = lastError + } + stateLock.unlock() + } + + private static func microphonePermissionDescription() -> String { + switch AVAudioApplication.shared.recordPermission { + case .granted: + return "granted" + case .denied: + return "denied" + case .undetermined: + return "undetermined" + @unknown default: + return "unknown" + } + } + + private static func requireApplicationBundle() throws { + guard Bundle.main.bundleIdentifier != nil else { + throw NativePipelineError.missingApplicationBundle + } + } + + private func beginRun() throws { + stateLock.lock() + defer { stateLock.unlock() } + guard !lifecycleClosed else { + throw NativePipelineError.notRestartable + } + guard !running else { + throw NativePipelineError.alreadyRunning + } + running = true + } + + private func endRun() throws { + stateLock.lock() + defer { stateLock.unlock() } + guard running else { + throw NativePipelineError.notRunning + } + running = false + lifecycleClosed = true + } +} diff --git a/Sources/OigoSpike/RecordValidator.swift b/Sources/OigoSpike/RecordValidator.swift new file mode 100644 index 0000000..a2646f5 --- /dev/null +++ b/Sources/OigoSpike/RecordValidator.swift @@ -0,0 +1,41 @@ +import Foundation + +public enum FeasibilityRecordError: Error, Equatable, CustomStringConvertible { + case missingFile(URL) + case missingSections([String]) + + public var description: String { + switch self { + case .missingFile(let url): + return "feasibility record is missing at \(url.path)" + case .missingSections(let sections): + return "feasibility record is missing sections: \(sections.joined(separator: ", "))" + } + } +} + +public struct FeasibilityRecordValidator: Sendable { + public static let requiredSections = [ + "## Tested environment", + "## APIs evaluated and selected", + "## Permission and asset-installation behavior", + "## Accuracy observations", + "## Latency and resource measurements", + "## Failure and retry results", + "## Offline result", + "## Recommendation" + ] + + public init() {} + + public func validate(url: URL) throws { + guard FileManager.default.fileExists(atPath: url.path) else { + throw FeasibilityRecordError.missingFile(url) + } + let contents = try String(contentsOf: url, encoding: .utf8) + let missing = Self.requiredSections.filter { !contents.contains($0) } + guard missing.isEmpty else { + throw FeasibilityRecordError.missingSections(missing) + } + } +} diff --git a/Sources/OigoSpike/TranscriptAccumulator.swift b/Sources/OigoSpike/TranscriptAccumulator.swift new file mode 100644 index 0000000..cd3a3f5 --- /dev/null +++ b/Sources/OigoSpike/TranscriptAccumulator.swift @@ -0,0 +1,116 @@ +import Foundation + +public struct TranscriptRange: Hashable, Sendable, Comparable { + public let startMilliseconds: Int64 + public let endMilliseconds: Int64 + + public init(startMilliseconds: Int64, endMilliseconds: Int64) { + self.startMilliseconds = startMilliseconds + self.endMilliseconds = endMilliseconds + } + + public static func < (lhs: TranscriptRange, rhs: TranscriptRange) -> Bool { + if lhs.startMilliseconds != rhs.startMilliseconds { + return lhs.startMilliseconds < rhs.startMilliseconds + } + return lhs.endMilliseconds < rhs.endMilliseconds + } + + fileprivate func overlaps(_ other: TranscriptRange) -> Bool { + if startMilliseconds == other.startMilliseconds && endMilliseconds == other.endMilliseconds { + return true + } + return max(startMilliseconds, other.startMilliseconds) < min(endMilliseconds, other.endMilliseconds) + } +} + +public struct TranscriptSnapshot: Equatable, Sendable { + public let finalizedText: String + public let volatileText: String + public let displayedText: String + + public init(finalizedText: String, volatileText: String, displayedText: String) { + self.finalizedText = finalizedText + self.volatileText = volatileText + self.displayedText = displayedText + } +} + +public struct TranscriptAccumulator: Sendable { + private struct Segment: Sendable { + let range: TranscriptRange + let text: String + } + + private var finalized: [Segment] = [] + private var volatile: [Segment] = [] + + public init() {} + + @discardableResult + public mutating func ingest( + range: TranscriptRange, + text: String, + isFinal: Bool + ) -> TranscriptSnapshot { + let segment = Segment(range: range, text: text) + if isFinal { + finalized.removeAll { $0.range.overlaps(range) } + volatile.removeAll { $0.range.overlaps(range) } + finalized.append(segment) + } else { + volatile.removeAll { $0.range.overlaps(range) } + volatile.append(segment) + } + return snapshot + } + + public var snapshot: TranscriptSnapshot { + let visibleFinalized = finalized.filter { finalSegment in + !volatile.contains { finalSegment.range.overlaps($0.range) } + } + let orderedFinalized = visibleFinalized.sorted { $0.range < $1.range } + let orderedVolatile = volatile.sorted { $0.range < $1.range } + return TranscriptSnapshot( + finalizedText: join(orderedFinalized), + volatileText: join(orderedVolatile), + displayedText: join(orderedFinalized + orderedVolatile) + ) + } + + public mutating func reset() { + finalized.removeAll(keepingCapacity: true) + volatile.removeAll(keepingCapacity: true) + } + + private func join(_ segments: [Segment]) -> String { + segments + .map { $0.text.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + .joined(separator: " ") + } +} + +public final class TranscriptStore: @unchecked Sendable { + private let lock = NSLock() + private var accumulator = TranscriptAccumulator() + + public init() {} + + @discardableResult + public func ingest( + range: TranscriptRange, + text: String, + isFinal: Bool + ) -> TranscriptSnapshot { + lock.lock() + defer { lock.unlock() } + return accumulator.ingest(range: range, text: text, isFinal: isFinal) + } + + public var snapshot: TranscriptSnapshot { + lock.lock() + defer { lock.unlock() } + return accumulator.snapshot + } +} diff --git a/Sources/oigo-spike/main.swift b/Sources/oigo-spike/main.swift new file mode 100644 index 0000000..51b1863 --- /dev/null +++ b/Sources/oigo-spike/main.swift @@ -0,0 +1,271 @@ +import AVFAudio +import Darwin +import Foundation +import OigoSpike + +private enum CLIError: Error, CustomStringConvertible { + case missingValue(String) + case invalidValue(String, String) + case unknownArgument(String) + case missingScenario + + var description: String { + switch self { + case .missingValue(let option): + return "missing value for " + option + case .invalidValue(let option, let value): + return "invalid value \"" + value + "\" for " + option + case .unknownArgument(let argument): + return "unknown argument " + argument + case .missingScenario: + return "missing --scenario or --verify-record" + } + } +} + +private struct CLIOptions { + var scenario: String? + var fixture: URL? + var output: URL? + var verifyRecord: URL? + var runs = 20 + var duration = 5.0 + var showHelp = false +} + +@main +private struct OigoSpikeCLI { + static func main() async { + do { + let options = try parse(Array(CommandLine.arguments.dropFirst())) + if options.showHelp { + printHelp() + exit(0) + } + try await run(options) + exit(0) + } catch { + FileHandle.standardError.write(Data(("ERROR: " + String(describing: error) + "\n").utf8)) + exit(1) + } + } + + private static func parse(_ arguments: [String]) throws -> CLIOptions { + var options = CLIOptions(scenario: nil, fixture: nil, output: nil, verifyRecord: nil) + var index = 0 + while index < arguments.count { + let argument = arguments[index] + switch argument { + case "--help", "-h": + options.showHelp = true + case "--scenario": + index += 1 + options.scenario = try value(arguments, at: index, for: argument) + case "--fixture": + index += 1 + options.fixture = URL(fileURLWithPath: try value(arguments, at: index, for: argument)) + case "--output": + index += 1 + options.output = URL(fileURLWithPath: try value(arguments, at: index, for: argument)) + case "--verify-record": + index += 1 + options.verifyRecord = URL(fileURLWithPath: try value(arguments, at: index, for: argument)) + case "--runs": + index += 1 + let rawValue = try value(arguments, at: index, for: argument) + guard let runs = Int(rawValue), runs > 0 else { + throw CLIError.invalidValue(argument, rawValue) + } + options.runs = runs + case "--duration": + index += 1 + let rawValue = try value(arguments, at: index, for: argument) + guard let duration = Double(rawValue), duration.isFinite, duration >= 0 else { + throw CLIError.invalidValue(argument, rawValue) + } + options.duration = duration + default: + throw CLIError.unknownArgument(argument) + } + index += 1 + } + if !options.showHelp && options.scenario == nil && options.verifyRecord == nil { + throw CLIError.missingScenario + } + return options + } + + private static func value( + _ arguments: [String], + at index: Int, + for option: String + ) throws -> String { + guard arguments.indices.contains(index) else { + throw CLIError.missingValue(option) + } + return arguments[index] + } + + private static func run(_ options: CLIOptions) async throws { + if let verifyRecord = options.verifyRecord { + try FeasibilityRecordValidator().validate(url: verifyRecord) + print("record_valid=true") + print("record_path=" + verifyRecord.path) + return + } + + guard let scenario = options.scenario else { + throw CLIError.missingScenario + } + switch scenario { + case "failure-retry": + try runFailureRetry(output: options.output) + case "transcript-cleanup": + try await runTranscriptCleanup() + case "offline": + try runOffline(fixture: options.fixture) + case "resource-measurement": + try runResourceMeasurement(runs: options.runs) + case "capabilities": + try await runCapabilities() + case "install-assets": + try await runInstallAssets() + case "live": + try await runLive(output: options.output, duration: options.duration) + default: + throw CLIError.unknownArgument("--scenario " + scenario) + } + } + + private static func runFailureRetry(output: URL?) throws { + let url = output ?? FileManager.default.temporaryDirectory + .appendingPathComponent("oigo-spike-") + .appendingPathExtension("caf") + try createSilentCAF(at: url) + let frames = try CAFRecorder.playableFrameLength(at: url) + let retryResult = try SavedAudioRetry.retryAfterFailure( + url: url, + liveFailure: ForcedRecognitionFailure() + ) { savedURL in + let savedFrames = try CAFRecorder.playableFrameLength(at: savedURL) + return "retry-ready frames=" + String(savedFrames) + } + print("audio_path=" + url.path) + print("playable_frames=" + String(frames)) + print("forced_live_failure=preserved") + print("saved_file_retry=" + retryResult) + } + + private static func runTranscriptCleanup() async throws { + var accumulator = TranscriptAccumulator() + _ = accumulator.ingest( + range: TranscriptRange(startMilliseconds: 0, endMilliseconds: 1_000), + text: "hello wor", + isFinal: false + ) + let snapshot = accumulator.ingest( + range: TranscriptRange(startMilliseconds: 0, endMilliseconds: 1_000), + text: "hello world", + isFinal: true + ) + let cleanup = CleanupFallback.failed( + rawText: snapshot.finalizedText, + error: ForcedRecognitionFailure() + ) + let foundationModelsCleanup = await FoundationModelsCleaner() + .clean(rawText: snapshot.finalizedText) + print("raw_text=" + cleanup.rawText) + print("displayed_text=" + cleanup.displayedText) + print("cleanup_status=" + String(describing: cleanup.status)) + print("no_duplicated_segments=" + String(snapshot.displayedText == "hello world")) + print("raw_fallback_available=" + String(cleanup.displayedText == cleanup.rawText)) + print("foundation_models_cleanup_status=" + String(describing: foundationModelsCleanup.status)) + print("foundation_models_raw_fallback=" + String(foundationModelsCleanup.displayedText == foundationModelsCleanup.rawText)) + } + + private static func runOffline(fixture: URL?) throws { + if let fixture { + let frames = try CAFRecorder.playableFrameLength(at: fixture) + print("fixture_frames=" + String(frames)) + } + print("network_requests=0") + print("oigo_network_client=none") + print("network_disabled_by_harness=false") + print("external_network_disable_required=true") + } + + private static func runResourceMeasurement(runs: Int) throws { + let measurement = ResourceMeasurementRunner.measure(runs: runs) { + var accumulator = TranscriptAccumulator() + for index in 0..<50 { + let start = Int64(index * 100) + _ = accumulator.ingest( + range: TranscriptRange(startMilliseconds: start, endMilliseconds: start + 100), + text: "sample", + isFinal: index.isMultiple(of: 2) + ) + } + } + print("runs=" + String(measurement.runs)) + print("deterministic_state_memory_delta_bytes=" + String(measurement.maximumResidentDeltaBytes)) + print("deterministic_state_bounded=" + String(measurement.bounded)) + print("native_model_objects_measured=false") + } + + private static func runCapabilities() async throws { + let capabilities = await NativeDictationPipeline.capabilitySnapshot() + for key in capabilities.keys.sorted() { + print(key + "=" + (capabilities[key] ?? "")) + } + } + + private static func runInstallAssets() async throws { + print("asset_installation_started=true") + let status = try await NativeDictationPipeline.installSpeechAssets() + print("asset_installation_status=" + status) + } + + private static func runLive(output: URL?, duration: TimeInterval) async throws { + let url = output ?? FileManager.default.temporaryDirectory + .appendingPathComponent("oigo-live-") + .appendingPathExtension("caf") + let pipeline = NativeDictationPipeline(audioURL: url) { snapshot in + print("displayed_text=" + snapshot.displayedText) + print("finalized_text=" + snapshot.finalizedText) + } + try await pipeline.run(for: duration) + print("audio_path=" + url.path) + print("transcriber=" + pipeline.transcriberChoice) + print("final_text=" + pipeline.latestSnapshot.finalizedText) + } + + private static func createSilentCAF(at url: URL) throws { + let format = AVAudioFormat(standardFormatWithSampleRate: 16_000, channels: 1) + guard let format else { + throw NSError(domain: "OigoSpike", code: 1) + } + guard let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: 1_600) else { + throw NSError(domain: "OigoSpike", code: 2) + } + buffer.frameLength = 1_600 + if let samples = buffer.floatChannelData?.pointee { + for index in 0.. Void)] = [ + ("durable capture and retry", { try testDurableCaptureAndRetry() }), + ("volatile final replacement and raw cleanup fallback", { try testTranscriptAndCleanup() }), + ("cleanup unavailable and raw fallback", testCleanupFallback), + ("state memory bound across twenty runs", { try testResourceMeasurement() }), + ("feasibility record validator", { try testFeasibilityRecord() }) + ] + var failures = 0 + for (name, test) in cases { + do { + try await test() + print("GREEN: " + name) + } catch { + failures += 1 + print("FAIL: " + name + ": " + String(describing: error)) + } + } + if failures == 0 { + print("GREEN: all contract scenarios") + exit(0) + } + print("FAILURES=" + String(failures)) + exit(1) + } + + private static func testDurableCaptureAndRetry() throws { + let root = try temporaryDirectory() + defer { try? FileManager.default.removeItem(at: root) } + let url = root.appendingPathComponent("capture.caf") + try createSilentCAF(at: url) + let frames = try CAFRecorder.playableFrameLength(at: url) + let retriedFrames = try SavedAudioRetry.retryAfterFailure( + url: url, + liveFailure: ForcedRecognitionFailure() + ) { savedURL in + try CAFRecorder.playableFrameLength(at: savedURL) + } + guard frames > 0, retriedFrames == frames else { + throw ContractFailure(message: "saved CAF was not playable after forced recognition failure") + } + } + + private static func testTranscriptAndCleanup() throws { + var accumulator = TranscriptAccumulator() + _ = accumulator.ingest( + range: TranscriptRange(startMilliseconds: 0, endMilliseconds: 1_000), + text: "hello wor", + isFinal: false + ) + let snapshot = accumulator.ingest( + range: TranscriptRange(startMilliseconds: 0, endMilliseconds: 1_000), + text: "hello world", + isFinal: true + ) + let cleanup = CleanupFallback.failed( + rawText: snapshot.finalizedText, + error: ForcedRecognitionFailure() + ) + guard snapshot.displayedText == "hello world" else { + throw ContractFailure(message: "volatile text was not replaced by one finalized segment") + } + guard cleanup.displayedText == cleanup.rawText else { + throw ContractFailure(message: "raw transcript was not retained on cleanup failure") + } + } + + private static func testCleanupFallback() async throws { + let actual = await FoundationModelsCleaner().clean(rawText: "raw transcript") + guard actual.rawText == "raw transcript" else { + throw ContractFailure(message: "Foundation Models cleaner did not retain raw transcript") + } + if actual.cleanedText == nil, actual.displayedText != actual.rawText { + throw ContractFailure(message: "Foundation Models unavailable path did not retain raw transcript") + } + let failed = await CleanupFallback.run( + rawText: "raw transcript", + timeoutNanoseconds: 1_000_000 + ) { + throw ForcedRecognitionFailure() + } + guard failed.cleanedText == nil, failed.displayedText == failed.rawText else { + throw ContractFailure(message: "cleanup failure did not retain raw transcript") + } + guard case .failed = failed.status else { + throw ContractFailure(message: "cleanup failure did not report failed status") + } + + let timedOut = await CleanupFallback.run( + rawText: "raw transcript", + timeoutNanoseconds: 1_000_000 + ) { + try await Task.sleep(nanoseconds: 100_000_000) + return "cleaned transcript" + } + guard timedOut.cleanedText == nil, timedOut.displayedText == timedOut.rawText else { + throw ContractFailure(message: "cleanup timeout fallback did not retain raw transcript") + } + guard timedOut.status == .timedOut else { + throw ContractFailure(message: "cleanup timeout fallback did not report timedOut status") + } + } + + private static func testResourceMeasurement() throws { + let measurement = ResourceMeasurementRunner.measure(runs: 20) { + var accumulator = TranscriptAccumulator() + for index in 0..<50 { + let start = Int64(index * 100) + _ = accumulator.ingest( + range: TranscriptRange(startMilliseconds: start, endMilliseconds: start + 100), + text: "sample", + isFinal: index.isMultiple(of: 2) + ) + } + } + guard measurement.runs == 20, measurement.bounded else { + throw ContractFailure(message: "twenty-run resource harness exceeded its bound") + } + } + + private static func testFeasibilityRecord() throws { + let root = try temporaryDirectory() + defer { try? FileManager.default.removeItem(at: root) } + let url = root.appendingPathComponent("record.md") + let contents = FeasibilityRecordValidator.requiredSections.joined(separator: "\n") + try contents.write(to: url, atomically: true, encoding: .utf8) + try FeasibilityRecordValidator().validate(url: url) + } + + private static func temporaryDirectory() throws -> URL { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("oigo-contract-") + .appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) + return url + } + + private static func createSilentCAF(at url: URL) throws { + guard let format = AVAudioFormat(standardFormatWithSampleRate: 16_000, channels: 1) else { + throw ContractFailure(message: "could not create 16 kHz mono format") + } + guard let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: 1_600) else { + throw ContractFailure(message: "could not create silent audio buffer") + } + buffer.frameLength = 1_600 + if let samples = buffer.floatChannelData?.pointee { + for index in 0.. Date: Fri, 14 Aug 2026 20:36:30 -0400 Subject: [PATCH 2/2] ci: verify Swift build and contract harness --- .github/workflows/verify.yml | 25 +++++++++++++++++++++++++ docs/native-on-device-dictation.md | 10 ++++++++++ 2 files changed, 35 insertions(+) create mode 100644 .github/workflows/verify.yml diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml new file mode 100644 index 0000000..c900f9d --- /dev/null +++ b/.github/workflows/verify.yml @@ -0,0 +1,25 @@ +name: verify + +on: + pull_request: + +permissions: + contents: read + +jobs: + swift-build-and-contracts: + name: Swift build and contract harness + runs-on: macos-26 + timeout-minutes: 15 + steps: + - name: Check out source + uses: actions/checkout@v4 + - name: Report SwiftPM host + run: | + sw_vers + swift --version + xcode-select -p + - name: Swift build + run: swift build + - name: Dependency-free contract harness + run: swift run oigo-spike-contract-tests diff --git a/docs/native-on-device-dictation.md b/docs/native-on-device-dictation.md index 665fba2..57dab63 100644 --- a/docs/native-on-device-dictation.md +++ b/docs/native-on-device-dictation.md @@ -52,6 +52,16 @@ The live path requires an app host with `NSMicrophoneUsageDescription` and a use Evidence: `/Users/douglasjarquin/.codex/evidence/oigo-issue-2/capabilities-final-v2.txt`, `/Users/douglasjarquin/.codex/evidence/oigo-issue-2/lldb-live-repro.txt`, and `/Users/douglasjarquin/.codex/evidence/oigo-issue-2/live-host-guard-final-v2.txt`. +## CI verification boundary + +Pull requests run `swift build` and `swift run oigo-spike-contract-tests` on a macOS 26 GitHub Actions runner through `.github/workflows/verify.yml`. + +This check intentionally uses SwiftPM and does not invoke `xcodebuild`. + +The CI check proves compilation and the dependency-free contract harness only. + +It does not remove the local Command Line Tools-only limitation described above, and it does not claim to validate live microphone/TCC behavior from an application bundle, Apple Intelligence availability, network-disabled Speech execution, or native Speech/Foundation Models resource measurements. + ## Accuracy observations The direct native path captured playable CAF files on successful runs, but emitted no transcript text during the controlled three-second captures.