From f2088abfef1fb8be32161b060159c70b515e121a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?AI=E4=BA=A7=E5=93=81=E9=BB=84=E5=8F=94?= Date: Fri, 7 Aug 2026 17:50:33 +0800 Subject: [PATCH] feat: reduce background energy usage --- README.md | 12 +- .../Sources/TokenStepHelper/main.swift | 6 +- .../TokenStepSwift/Services/DataService.swift | 116 +- .../Services/TokenRankService.swift | 2 +- .../Services/UsageCollector.swift | 2042 ++++++++++++++++- .../TokenStepSwift/Stores/AppState.swift | 160 +- .../TokenStepSwift/Support/AppPaths.swift | 2 + .../Support/EnergyRefreshPolicy.swift | 67 + .../TokenStepSwift/Support/Localization.swift | 4 +- .../Support/MainWindowPresenter.swift | 18 +- .../Support/TokenIslandWindowPresenter.swift | 3 + .../TokenStepSwift/Views/MainWindowView.swift | 3 + .../Views/PopoverPanelView.swift | 5 + .../SettingsDisplayRefreshCards.swift | 2 +- .../Fixtures/CCSwitchProxyFixtureCheck.swift | 19 + .../CodexCumulativeFixtureCheck.swift | 353 ++- .../Fixtures/EnergyEfficiencyBenchmark.swift | 166 ++ ...geRecalibrationMigrationFixtureCheck.swift | 161 ++ .../EnergyRefreshPolicyTests.swift | 89 + .../UsageCollectorCodexTests.swift | 28 +- script/benchmark_energy_efficiency.sh | 51 + script/build_swiftui_and_run.sh | 2 +- script/package_release.sh | 4 +- script/test_usage_recalibration_migration.sh | 1 + 24 files changed, 3177 insertions(+), 139 deletions(-) create mode 100644 TokenStepSwift/Sources/TokenStepSwift/Support/EnergyRefreshPolicy.swift create mode 100644 TokenStepSwift/Tests/Fixtures/EnergyEfficiencyBenchmark.swift create mode 100644 TokenStepSwift/Tests/TokenStepSwiftTests/EnergyRefreshPolicyTests.swift create mode 100755 script/benchmark_energy_efficiency.sh diff --git a/README.md b/README.md index 91476c4..95339f0 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ TokenStep 是一个 macOS 菜单栏 App，用来本地统计你在 Codex、Claud 下载最新版 DMG，打开后把 `TokenStep.app` 拖进「应用程序」即可使用： -[下载 TokenStep 最新版](https://github.com/Backtthefuture/TokenStep/releases/latest/download/TokenStep-0.1.47.dmg) +[下载 TokenStep 最新版](https://github.com/Backtthefuture/TokenStep/releases/latest/download/TokenStep-0.1.48.dmg) 也可以从 Release 页面查看所有版本： @@ -55,7 +55,7 @@ TokenStep 适合这些人： - 按客户端、按模型查看用量统计。 - 粗略估算 Token 消耗金额。 - 每日目标可设置，默认每天一个亿。 -- 自动刷新，默认 5 分钟，兼顾及时性和常驻内存/电量占用。 +- 打开面板时按设置的新鲜度刷新；后台在接电时最低 15 分钟、电池或低电量模式下最低 30 分钟刷新，并跳过未变化的数据。 - 开机启动，可在设置里关闭。 - 多种主题色，菜单栏、圆环、活动墙和按钮会一起变化。 - 一键截图分享当前页面。 @@ -66,7 +66,7 @@ TokenStep 适合这些人： ## 当前支持 -- Codex：优先读取 Codex 本地 SQLite token 汇总，必要时回退 JSONL。 +- Codex：读取本地 JSONL 用量元数据并维护逐会话增量缓存；缓存异常时自动重建，必要时回退 Codex 本地 SQLite 汇总。 - Claude Code：读取 `~/.claude/projects/**/*.jsonl` 里的 usage 元数据。 - CC Switch：实验支持，读取本机 `proxy_request_logs` 中成功且 token 数大于 0 的请求行。 - 额度显示：Codex 读取本机 Codex 账户限额；Claude Code 会在本机读取 Claude Code 钥匙串凭证，并请求 Anthropic usage 接口获取 5 小时 / 7 天剩余额度。 @@ -89,7 +89,7 @@ TokenStep 默认只做本地统计。 ## 安装方式 -1. 下载 [TokenStep 最新版 DMG](https://github.com/Backtthefuture/TokenStep/releases/latest/download/TokenStep-0.1.47.dmg)。 +1. 下载 [TokenStep 最新版 DMG](https://github.com/Backtthefuture/TokenStep/releases/latest/download/TokenStep-0.1.48.dmg)。 2. 打开 DMG。 3. 把 `TokenStep.app` 拖到「应用程序」。 4. 启动 TokenStep。 @@ -149,7 +149,7 @@ TokenStepSwift/dist/TokenStep.app Developer ID 签名： ```bash -TOKENSTEP_VERSION=0.1.47 \ +TOKENSTEP_VERSION=0.1.48 \ CODE_SIGN_IDENTITY="Developer ID Application: Your Name (TEAMID)" \ ./script/package_release.sh ``` @@ -157,7 +157,7 @@ CODE_SIGN_IDENTITY="Developer ID Application: Your Name (TEAMID)" \ 签名 + Apple 公证： ```bash -TOKENSTEP_VERSION=0.1.47 \ +TOKENSTEP_VERSION=0.1.48 \ CODE_SIGN_IDENTITY="Developer ID Application: Your Name (TEAMID)" \ TOKENSTEP_NOTARY_PROFILE="tokenstep-notary" \ ./script/package_release.sh --notarize diff --git a/TokenStepSwift/Sources/TokenStepHelper/main.swift b/TokenStepSwift/Sources/TokenStepHelper/main.swift index 6af5ae5..2dc5f31 100644 --- a/TokenStepSwift/Sources/TokenStepHelper/main.swift +++ b/TokenStepSwift/Sources/TokenStepHelper/main.swift @@ -15,7 +15,11 @@ enum TokenStepHelper { switch command { case "collect": let historyDays = arguments.first.flatMap(Int.init) ?? DataService.loadSettings().historyDays - try DataService.runCollector(historyDays: historyDays) + let outcome = try DataService.runCollector( + historyDays: historyDays, + force: arguments.contains("--force") + ) + FileHandle.standardOutput.write(Data("\(outcome.rawValue)\n".utf8)) case "install": try UpdateInstaller(arguments: arguments).run() default: diff --git a/TokenStepSwift/Sources/TokenStepSwift/Services/DataService.swift b/TokenStepSwift/Sources/TokenStepSwift/Services/DataService.swift index ad2fe04..d8072f2 100644 --- a/TokenStepSwift/Sources/TokenStepSwift/Services/DataService.swift +++ b/TokenStepSwift/Sources/TokenStepSwift/Services/DataService.swift @@ -43,14 +43,33 @@ enum DataService { try data.write(to: AppPaths.settingsJSON, options: .atomic) } - static func runCollector(historyDays: Int = TokenStepSettings.defaults.historyDays) throws { + @discardableResult + static func runCollector( + historyDays: Int = TokenStepSettings.defaults.historyDays, + force: Bool = false + ) throws -> CollectionRunOutcome { defer { MemoryPressure.relieveAllocatorPressure() } let settings = loadSettings() let previousSnapshot = try? loadSnapshot() - let collectedSnapshot = UsageCollector.collect( + let beforeState = UsageCollector.collectionState( historyDays: historyDays, includeExperimentalAgentSources: settings.showExperimentalAgentSources ) + let existingCheckpoint = loadCollectionCheckpoint() + if CollectionCheckpointPolicy.shouldSkipCollection( + force: force, + hasSnapshot: previousSnapshot != nil, + checkpoint: existingCheckpoint, + state: beforeState, + now: Date() + ) { + return .unchanged + } + let collectedSnapshot = UsageCollector.collect( + historyDays: historyDays, + includeExperimentalAgentSources: settings.showExperimentalAgentSources, + forceFullValidation: force || existingCheckpoint?.isFresh(at: Date()) != true + ) try validateRecalibrationCandidate( collectedSnapshot, previousSnapshot: previousSnapshot @@ -60,6 +79,23 @@ enum DataService { previousSnapshot: previousSnapshot ) try persist(snapshot: snapshot) + let afterState = UsageCollector.collectionState( + historyDays: historyDays, + includeExperimentalAgentSources: settings.showExperimentalAgentSources + ) + let sourceStateWasStable = CollectionCheckpointPolicy.shouldPersist( + beforeCollection: beforeState, + afterCollection: afterState + ) + if sourceStateWasStable { + saveCollectionCheckpoint( + CollectionCheckpoint( + verifiedAt: Date(), + state: afterState + ) + ) + } + return sourceStateWasStable ? .updated : .updatedWhileSourcesChanged } private static func snapshotWithMigrationMetadata( @@ -148,20 +184,45 @@ enum DataService { FileManager.default.fileExists(atPath: AppPaths.usageRecalibrationNoticeMarker.path) } + private static func loadCollectionCheckpoint() -> CollectionCheckpoint? { + guard let data = try? Data(contentsOf: AppPaths.collectionCheckpointJSON) else { + return nil + } + return try? JSONDecoder().decode(CollectionCheckpoint.self, from: data) + } + + private static func saveCollectionCheckpoint(_ checkpoint: CollectionCheckpoint) { + do { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + let data = try encoder.encode(checkpoint) + try FileManager.default.createDirectory( + at: AppPaths.collectionCheckpointJSON.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + try data.write(to: AppPaths.collectionCheckpointJSON, options: .atomic) + } catch { + // A missing checkpoint only costs another safe collection. + } + } + static func acknowledgeUsageRecalibrationNotice() { try? FileManager.default.removeItem(at: AppPaths.usageRecalibrationNoticeMarker) } - static func runCollectorInHelper(historyDays: Int = TokenStepSettings.defaults.historyDays) throws { + static func runCollectorInHelper( + historyDays: Int = TokenStepSettings.defaults.historyDays, + force: Bool = false + ) throws -> CollectionRunOutcome { guard let helperURL = bundledHelperURL() else { - try runCollector(historyDays: historyDays) - return + return try runCollector(historyDays: historyDays, force: force) } let process = Process() process.executableURL = helperURL - process.arguments = ["collect", "\(historyDays)"] - process.standardOutput = Pipe() + process.arguments = ["collect", "\(historyDays)"] + (force ? ["--force"] : []) + let standardOutput = Pipe() + process.standardOutput = standardOutput let standardError = Pipe() process.standardError = standardError @@ -197,6 +258,10 @@ enum DataService { userInfo: [NSLocalizedDescriptionKey: message?.isEmpty == false ? message! : "Token collector failed."] ) } + let outputData = standardOutput.fileHandleForReading.readDataToEndOfFile() + let output = String(data: outputData, encoding: .utf8)? + .trimmingCharacters(in: .whitespacesAndNewlines) + return CollectionRunOutcome(rawValue: output ?? "") ?? .updated } static func bundledHelperURL() -> URL? { @@ -238,3 +303,40 @@ enum DataService { ) } } + +enum CollectionRunOutcome: String { + case updated + case unchanged + case updatedWhileSourcesChanged = "updated_source_changed" +} + +struct CollectionCheckpoint: Codable, Equatable { + static let validationTTL: TimeInterval = 24 * 60 * 60 + + var verifiedAt: Date + var state: UsageCollectionState + + func isFresh(at now: Date = Date()) -> Bool { + now.timeIntervalSince(verifiedAt) < Self.validationTTL + } +} + +enum CollectionCheckpointPolicy { + static func shouldSkipCollection( + force: Bool, + hasSnapshot: Bool, + checkpoint: CollectionCheckpoint?, + state: UsageCollectionState, + now: Date + ) -> Bool { + guard !force, hasSnapshot, let checkpoint else { return false } + return checkpoint.isFresh(at: now) && checkpoint.state == state + } + + static func shouldPersist( + beforeCollection: UsageCollectionState, + afterCollection: UsageCollectionState + ) -> Bool { + beforeCollection == afterCollection + } +} diff --git a/TokenStepSwift/Sources/TokenStepSwift/Services/TokenRankService.swift b/TokenStepSwift/Sources/TokenStepSwift/Services/TokenRankService.swift index c2368c4..b282f99 100644 --- a/TokenStepSwift/Sources/TokenStepSwift/Services/TokenRankService.swift +++ b/TokenStepSwift/Sources/TokenStepSwift/Services/TokenRankService.swift @@ -4,7 +4,7 @@ enum AgentWorkRankService { static let defaultClient = "all" static let defaultRange = "today" static let defaultUsageMode = "all" - static let cacheTTL: TimeInterval = 5 * 60 + static let cacheTTL: TimeInterval = 30 * 60 static let leaderboardPageURL = URL(string: "https://www.zhenganhuo.com/token-rank")! static let myPageURL = URL(string: "https://www.zhenganhuo.com/token-rank/me")! diff --git a/TokenStepSwift/Sources/TokenStepSwift/Services/UsageCollector.swift b/TokenStepSwift/Sources/TokenStepSwift/Services/UsageCollector.swift index b269710..7973a04 100644 --- a/TokenStepSwift/Sources/TokenStepSwift/Services/UsageCollector.swift +++ b/TokenStepSwift/Sources/TokenStepSwift/Services/UsageCollector.swift @@ -1,4 +1,35 @@ +import CryptoKit import Foundation +import SQLite3 + +struct UsageCollectionFileState: Codable, Equatable { + var path: String + var size: UInt64 + var modificationTime: TimeInterval +} + +struct UsageCollectionState: Codable, Equatable { + var schemaVersion = 2 + var historyDays: Int + var includesExperimentalAgentSources: Bool + var windowDay: String + var files: [UsageCollectionFileState] +} + +struct CodexIncrementalCacheStats: Equatable { + var generation: Int + var sessions: Int + var records: Int + var lastLogicalWriteBytes: Int +} + +struct CodexAccountingComparisonDiagnostics { + var incrementalSnapshot: UsageSnapshot + var referenceSnapshot: UsageSnapshot + var mismatchedPathHashes: [String] + var incrementalRecordCount: Int + var referenceRecordCount: Int +} enum UsageCollector { static let codexAccountingRevision = 8 @@ -14,18 +45,27 @@ enum UsageCollector { includeExperimentalAgentSources: Bool = false, zCodeDatabaseURL: URL? = nil, hermesDatabaseURL: URL? = nil, - workBuddyRootURLs: [URL]? = nil + workBuddyRootURLs: [URL]? = nil, + forceFullValidation: Bool = false ) -> UsageSnapshot { let cacheLoad = loadCache() var cache = cacheLoad.cache var livePaths = Set() let sourceCutoff = sourceFileCutoffDate(historyDays: historyDays) - var codex = collectCodex(cache: &cache, livePaths: &livePaths, modifiedSince: sourceCutoff) - codex.source.recalibratedFromRevision = cacheLoad.recalibratedFromRevision - let claude = collectClaudeCode(cache: &cache, livePaths: &livePaths, modifiedSince: sourceCutoff) var ccSwitch = includeCCSwitchProxyUsage ? collectCCSwitchProxyUsage(databaseURL: ccSwitchDatabaseURL) : CollectorResult(records: [], source: SourceInfo(status: "disabled", files: nil, records: 0)) + let codexOutcome = collectCodex( + cache: &cache, + livePaths: &livePaths, + modifiedSince: sourceCutoff, + databaseURL: AppPaths.codexIncrementalCacheSQLite, + forceFullValidation: forceFullValidation, + requiresDetailedRecords: !ccSwitch.records.isEmpty + ) + var codex = codexOutcome.result + codex.source.recalibratedFromRevision = cacheLoad.recalibratedFromRevision + let claude = collectClaudeCode(cache: &cache, livePaths: &livePaths, modifiedSince: sourceCutoff) let zCode = includeExperimentalAgentSources ? collectZCodeUsage(databaseURL: zCodeDatabaseURL) : CollectorResult(records: [], source: SourceInfo(status: "disabled", files: nil, records: 0)) @@ -35,7 +75,11 @@ enum UsageCollector { let workBuddy = includeExperimentalAgentSources ? collectWorkBuddyUsage(rootURLs: workBuddyRootURLs, modifiedSince: sourceCutoff) : CollectorResult(records: [], source: SourceInfo(status: "disabled", files: nil, records: 0)) - cache.files = cache.files.filter { livePaths.contains($0.key) } + if codexOutcome.usedIncrementalStore { + cache.files = cache.files.filter { $0.value.tool != "Codex" && livePaths.contains($0.key) } + } else { + cache.files = cache.files.filter { livePaths.contains($0.key) } + } saveCache(cache) let nativeRecords = codex.records + claude.records @@ -64,6 +108,175 @@ enum UsageCollector { ) } + static func collectionState( + historyDays: Int, + includeExperimentalAgentSources: Bool, + homeURL: URL = FileManager.default.homeDirectoryForCurrentUser, + now: Date = Date() + ) -> UsageCollectionState { + let cutoff = sourceFileCutoffDate(historyDays: historyDays) + var urls = defaultCodexSessionRoots(homeURL: homeURL) + .flatMap { jsonlFiles(under: $0, modifiedSince: cutoff) } + urls.append(contentsOf: jsonlFiles( + under: homeURL.appendingPathComponent(".claude/projects", isDirectory: true), + modifiedSince: cutoff + )) + + let databases = [ + homeURL.appendingPathComponent(".codex/state_5.sqlite"), + homeURL.appendingPathComponent(".codex/sqlite/state_5.sqlite"), + homeURL.appendingPathComponent(".cc-switch/cc-switch.db") + ] + urls.append(contentsOf: existingDatabaseFiles(databases)) + + if includeExperimentalAgentSources { + urls.append(contentsOf: existingDatabaseFiles([ + homeURL.appendingPathComponent(".zcode/cli/db/db.sqlite"), + homeURL.appendingPathComponent(".hermes/state.db") + ])) + urls.append(contentsOf: [ + homeURL.appendingPathComponent(".workbuddy/projects", isDirectory: true), + homeURL.appendingPathComponent("Library/Application Support/WorkBuddyExtension", isDirectory: true) + ].flatMap { jsonlFiles(under: $0, modifiedSince: cutoff) }) + } + + let files = Dictionary(grouping: urls, by: \.path) + .compactMap { _, duplicates in duplicates.first.flatMap(collectionFileState) } + .sorted { $0.path < $1.path } + return UsageCollectionState( + historyDays: historyDays, + includesExperimentalAgentSources: includeExperimentalAgentSources, + windowDay: dayFormatter.string(from: now), + files: files + ) + } + + static func codexIncrementalCacheStatsForTests(databaseURL: URL) -> CodexIncrementalCacheStats? { + try? CodexIncrementalStore(url: databaseURL).stats() + } + + static func codexCollectionStateForTests( + homeURL: URL + ) -> [UsageCollectionFileState] { + defaultCodexSessionRoots(homeURL: homeURL) + .flatMap { jsonlFiles(under: $0, modifiedSince: nil) } + .compactMap(collectionFileState) + .sorted { $0.path < $1.path } + } + + static func compareIncrementalCodexAccountingForTests( + homeURL: URL, + databaseURL: URL + ) throws -> CodexAccountingComparisonDiagnostics { + let incremental = try collectCodexIncrementally( + modifiedSince: nil, + databaseURL: databaseURL, + forceFullValidation: false, + homeURL: homeURL, + requiresDetailedRecords: true + ) + var cache = CollectorCache() + var livePaths = Set() + let reference = collectCodexFromJSONL( + cache: &cache, + livePaths: &livePaths, + modifiedSince: nil, + homeURL: homeURL + ) + + return try accountingComparisonDiagnostics( + incremental: incremental, + reference: reference + ) + } + + static func compareLegacyMigrationCodexAccountingForTests( + homeURL: URL, + databaseURL: URL + ) throws -> CodexAccountingComparisonDiagnostics { + var legacyCache = CollectorCache() + var livePaths = Set() + let reference = collectCodexFromJSONL( + cache: &legacyCache, + livePaths: &livePaths, + modifiedSince: nil, + homeURL: homeURL + ) + let incremental = try collectCodexIncrementally( + modifiedSince: nil, + databaseURL: databaseURL, + forceFullValidation: false, + homeURL: homeURL, + requiresDetailedRecords: true, + legacyCache: legacyCache + ) + return try accountingComparisonDiagnostics( + incremental: incremental, + reference: reference + ) + } + + private static func accountingComparisonDiagnostics( + incremental: CollectorResult, + reference: CollectorResult + ) throws -> CodexAccountingComparisonDiagnostics { + let incrementalByPath = Dictionary(grouping: incremental.records) { + $0.sourcePath ?? "" + } + let referenceByPath = Dictionary(grouping: reference.records) { + $0.sourcePath ?? "" + } + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + let paths = Set(incrementalByPath.keys).union(referenceByPath.keys) + let mismatches = try paths.compactMap { path -> String? in + let incrementalData = try encoder.encode(incrementalByPath[path] ?? []) + let referenceData = try encoder.encode(referenceByPath[path] ?? []) + return incrementalData == referenceData ? nil : anonymousPathHash(path) + }.sorted() + + return CodexAccountingComparisonDiagnostics( + incrementalSnapshot: aggregate( + records: incremental.records, + sources: ["Codex": incremental.source] + ), + referenceSnapshot: aggregate( + records: reference.records, + sources: ["Codex": reference.source] + ), + mismatchedPathHashes: mismatches, + incrementalRecordCount: incremental.records.count, + referenceRecordCount: reference.records.count + ) + } + + private static func anonymousPathHash(_ path: String) -> String { + var hash: UInt64 = 14_695_981_039_346_656_037 + for byte in path.utf8 { + hash ^= UInt64(byte) + hash &*= 1_099_511_628_211 + } + return String(format: "%016llx", hash) + } + + private static func existingDatabaseFiles(_ databases: [URL]) -> [URL] { + databases.flatMap { database in + [ + database, + URL(fileURLWithPath: database.path + "-wal") + ].filter { FileManager.default.fileExists(atPath: $0.path) } + } + } + + private static func collectionFileState(_ url: URL) -> UsageCollectionFileState? { + guard let metadata = fileMetadata(for: url) else { return nil } + return UsageCollectionFileState( + path: url.standardizedFileURL.path, + size: metadata.size, + modificationTime: metadata.modificationTime + ) + } + static func collectCCSwitchProxyUsageSnapshot(databaseURL: URL) -> UsageSnapshot { let result = collectCCSwitchProxyUsage(databaseURL: databaseURL) return aggregate( @@ -79,8 +292,30 @@ enum UsageCollector { return aggregate(records: result.records, sources: ["Claude Code": result.source]) } - static func collectCodexUsageSnapshotForTests(homeURL: URL, cacheURL: URL? = nil) -> UsageSnapshot { - var cache = cacheURL.map(loadCurrentCache(at:)) ?? CollectorCache() + static func collectCodexUsageSnapshotForTests( + homeURL: URL, + cacheURL: URL? = nil, + forceFullValidation: Bool = false, + requiresDetailedRecords: Bool = false + ) -> UsageSnapshot { + if let cacheURL { + do { + let result = try collectCodexIncrementally( + modifiedSince: nil, + databaseURL: cacheURL, + forceFullValidation: forceFullValidation, + homeURL: homeURL, + requiresDetailedRecords: requiresDetailedRecords + ) + return aggregate(records: result.records, sources: ["Codex": result.source]) + } catch { + return aggregate( + records: [], + sources: ["Codex": SourceInfo(status: "incremental_cache_error", files: 0, records: 0)] + ) + } + } + var cache = CollectorCache() var livePaths = Set() let result = collectCodexFromJSONL( cache: &cache, @@ -88,13 +323,65 @@ enum UsageCollector { modifiedSince: nil, homeURL: homeURL ) - if let cacheURL { - cache.files = cache.files.filter { livePaths.contains($0.key) } - saveCache(cache, to: cacheURL) - } return aggregate(records: result.records, sources: ["Codex": result.source]) } + static func collectIncrementalCodexAndProxySnapshotForTests( + codexRoots: [URL], + cacheURL: URL, + ccSwitchDatabaseURL: URL + ) -> UsageSnapshot { + let codex: CollectorResult + do { + codex = try collectCodexIncrementally( + modifiedSince: nil, + databaseURL: cacheURL, + forceFullValidation: false, + requiresDetailedRecords: true, + roots: codexRoots + ) + } catch { + codex = CollectorResult( + records: [], + source: SourceInfo(status: "incremental_cache_error", files: 0, records: 0) + ) + } + var proxy = collectCCSwitchProxyUsage(databaseURL: ccSwitchDatabaseURL) + let deduped = deduplicateCrossSource( + nativeRecords: codex.records, + proxyRecords: proxy.records + ) + proxy.source = sourceInfo(proxy.source, annotatedWith: deduped) + return aggregate( + records: deduped.records, + sources: [ + "Codex": codex.source, + ccSwitchSourceName: proxy.source + ] + ) + } + + static func collectCodexWithIncrementalFallbackForTests( + homeURL: URL, + cacheURL: URL + ) -> UsageSnapshot { + var cache = CollectorCache() + var livePaths = Set() + let outcome = collectCodex( + cache: &cache, + livePaths: &livePaths, + modifiedSince: nil, + databaseURL: cacheURL, + forceFullValidation: false, + requiresDetailedRecords: false, + homeURL: homeURL + ) + return aggregate( + records: outcome.result.records, + sources: ["Codex": outcome.result.source] + ) + } + static func collectorCacheRecalibrationRevisionForTests(cacheURL: URL) -> Int? { loadCache(at: cacheURL).recalibratedFromRevision } @@ -157,12 +444,337 @@ enum UsageCollector { ) } - private static func collectCodex(cache: inout CollectorCache, livePaths: inout Set, modifiedSince cutoffDate: Date?) -> CollectorResult { - let jsonlResult = collectCodexFromJSONL(cache: &cache, livePaths: &livePaths, modifiedSince: cutoffDate) + private static func collectCodex( + cache: inout CollectorCache, + livePaths: inout Set, + modifiedSince cutoffDate: Date?, + databaseURL: URL, + forceFullValidation: Bool, + requiresDetailedRecords: Bool, + homeURL: URL = FileManager.default.homeDirectoryForCurrentUser + ) -> CodexCollectionOutcome { + func runIncremental() throws -> CollectorResult { + try collectCodexIncrementally( + modifiedSince: cutoffDate, + databaseURL: databaseURL, + forceFullValidation: forceFullValidation, + homeURL: homeURL, + requiresDetailedRecords: requiresDetailedRecords, + legacyCache: cache + ) + } + + do { + let incremental = try runIncremental() + if incremental.source.status == "ok" { + return CodexCollectionOutcome(result: incremental, usedIncrementalStore: true) + } + } catch { + if let cacheError = error as? CodexIncrementalStoreError, + cacheError.shouldRebuildCache { + CodexIncrementalStore.discardDatabase(at: databaseURL) + if let rebuilt = try? runIncremental(), rebuilt.source.status == "ok" { + return CodexCollectionOutcome(result: rebuilt, usedIncrementalStore: true) + } + } + } + + let jsonlResult = collectCodexFromJSONL( + cache: &cache, + livePaths: &livePaths, + modifiedSince: cutoffDate, + homeURL: homeURL + ) if jsonlResult.source.status == "ok" { - return jsonlResult + return CodexCollectionOutcome(result: jsonlResult, usedIncrementalStore: false) + } + return CodexCollectionOutcome( + result: collectCodexFromSQLite() ?? jsonlResult, + usedIncrementalStore: false + ) + } + + private static func collectCodexIncrementally( + modifiedSince cutoffDate: Date?, + databaseURL: URL, + forceFullValidation: Bool, + homeURL: URL = FileManager.default.homeDirectoryForCurrentUser, + requiresDetailedRecords: Bool = false, + legacyCache: CollectorCache? = nil, + roots: [URL]? = nil + ) throws -> CollectorResult { + let paths = (roots ?? defaultCodexSessionRoots(homeURL: homeURL)) + .flatMap { jsonlFiles(under: $0, modifiedSince: cutoffDate) } + .sorted { $0.path < $1.path } + guard !paths.isEmpty else { + return CollectorResult( + records: [], + source: SourceInfo(status: "missing", files: 0, records: 0) + ) + } + + let store = try CodexIncrementalStore(url: databaseURL) + let storedMetadata = try store.metadataByPath() + let currentPaths = Set(paths.map(\.path)) + let deletedPaths = Set(storedMetadata.keys).subtracting(currentPaths) + var fullyAffectedParentIDs = Set( + deletedPaths.compactMap { storedMetadata[$0]?.sessionID } + ) + var appendedParentAnchorThresholds = [String: TimeInterval]() + var stagedPaths = Set() + try store.beginStaging() + var committed = false + defer { + if !committed { + store.abortStaging() + } + } + + func validatedScan( + at path: URL, + metadata: (size: UInt64, modificationTime: TimeInterval) + ) throws -> PendingCodexSession { + if !forceFullValidation, + let legacyCache, + let scan = cachedCodexScan(for: path, cache: legacyCache), + let fingerprint = contentFingerprint(for: path, size: metadata.size) { + return PendingCodexSession( + path: path, + metadata: metadata, + fingerprint: fingerprint, + scan: scan + ) + } + + guard var stable = stableCodexScan(at: path) else { + throw CodexIncrementalStoreError.unstableSource(path.path) + } + if !stable.isStable, let retry = stableCodexScan(at: path) { + stable = retry + } + guard stable.isStable, + let fingerprint = contentFingerprint(for: path, size: stable.metadata.size) + else { + throw CodexIncrementalStoreError.unstableSource(path.path) + } + return PendingCodexSession( + path: path, + metadata: stable.metadata, + fingerprint: fingerprint, + scan: stable.scan + ) + } + + for path in paths { + guard let metadata = fileMetadata(for: path) else { continue } + let stored = storedMetadata[path.path] + var validatedFullFingerprint: String? + let metadataMatches = stored?.size == metadata.size + && abs((stored?.modificationTime ?? -1) - metadata.modificationTime) < 0.001 + if metadataMatches { + if !forceFullValidation { + continue + } + let fingerprint = contentFingerprint(for: path, size: metadata.size) + if fingerprint == stored?.fingerprint { + guard let fullFingerprint = fullContentFingerprint( + for: path, + size: metadata.size + ), + let afterValidation = fileMetadata(for: path), + UsageCollector.metadata(metadata, matches: afterValidation) + else { + throw CodexIncrementalStoreError.unstableSource(path.path) + } + if fullFingerprint == stored?.validationFingerprint { + continue + } + validatedFullFingerprint = fullFingerprint + } + } + + if !forceFullValidation, + let stored, + metadata.size > stored.size, + contentFingerprint(for: path, size: stored.size) == stored.fingerprint, + let cachedSession = try store.session(path: path.path), + let appended = incrementalCodexAppend(at: path, cached: cachedSession) { + try store.stage(session: appended) + if let earliestNewAnchor = appended.anchors + .dropFirst(cachedSession.anchors.count) + .first?.timestamp { + appendedParentAnchorThresholds[appended.sessionID] = min( + appendedParentAnchorThresholds[appended.sessionID] ?? earliestNewAnchor, + earliestNewAnchor + ) + } + continue + } + + var pending = try validatedScan(at: path, metadata: metadata) + if validatedFullFingerprint != nil { + pending.validationFingerprint = validatedFullFingerprint + } else if forceFullValidation, stored != nil, metadataMatches { + guard let fullFingerprint = fullContentFingerprint( + for: path, + size: pending.metadata.size + ), + let afterValidation = fileMetadata(for: path), + UsageCollector.metadata(pending.metadata, matches: afterValidation) + else { + throw CodexIncrementalStoreError.unstableSource(path.path) + } + pending.validationFingerprint = fullFingerprint + } + try store.stage( + scan: pending, + anchors: codexAnchors(for: pending.scan), + createdAtEpoch: pending.scan.createdAt.flatMap(parseISO)?.timeIntervalSince1970 + ) + stagedPaths.insert(path.path) + fullyAffectedParentIDs.insert(pending.scan.canonicalSessionID) + if let previousID = stored?.sessionID { + fullyAffectedParentIDs.insert(previousID) + } + } + + func stageChild(at childPath: String) throws { + guard currentPaths.contains(childPath), !stagedPaths.contains(childPath) else { return } + let url = URL(fileURLWithPath: childPath) + guard let metadata = fileMetadata(for: url) else { + throw CodexIncrementalStoreError.unstableSource(childPath) + } + var pending = try validatedScan(at: url, metadata: metadata) + if let stored = storedMetadata[childPath], + stored.size == metadata.size, + abs(stored.modificationTime - metadata.modificationTime) < 0.001, + contentFingerprint(for: url, size: metadata.size) == stored.fingerprint { + pending.validationFingerprint = stored.validationFingerprint + } + try store.stage( + scan: pending, + anchors: codexAnchors(for: pending.scan), + createdAtEpoch: pending.scan.createdAt.flatMap(parseISO)?.timeIntervalSince1970 + ) + stagedPaths.insert(childPath) + } + + for parentID in fullyAffectedParentIDs { + for childPath in try store.childPaths(parentSessionID: parentID) { + try stageChild(at: childPath) + } + } + for (parentID, earliestNewAnchor) in appendedParentAnchorThresholds + where !fullyAffectedParentIDs.contains(parentID) { + for childPath in try store.childPaths( + parentSessionID: parentID, + createdAtOnOrAfter: earliestNewAnchor + ) { + try stageChild(at: childPath) + } + } + + for stagedPath in try store.stagedScanPaths() { + guard let item = try store.stagedScan(path: stagedPath) else { + throw CodexIncrementalStoreError.sqlite("missing staged scan for \(stagedPath)") + } + let parentAnchors: [CodexAnchor]? + if let parentID = item.scan.parentSessionID { + if let pendingParent = try store.stagedAnchors(sessionID: parentID) { + parentAnchors = pendingParent + } else { + parentAnchors = try store.anchors(sessionID: parentID) + } + } else { + parentAnchors = nil + } + let childCreatedAt = item.scan.createdAt.flatMap(parseISO)?.timeIntervalSince1970 + let parentAnchor = childCreatedAt.flatMap { timestamp in + parentAnchors.flatMap { codexAnchor(atOrBefore: timestamp, anchors: $0) } + } + var seenRequestIDs = Set() + let result = codexDeltaRecords( + from: item.scan, + parentAnchor: parentAnchor, + seenRequestIDs: &seenRequestIDs + ) + let candidate = CodexCachedSession( + path: item.path.path, + size: item.metadata.size, + modificationTime: item.metadata.modificationTime, + fingerprint: item.fingerprint, + validationFingerprint: item.validationFingerprint, + sessionID: item.scan.canonicalSessionID, + createdAtEpoch: childCreatedAt, + parentSessionID: item.scan.parentSessionID, + anchors: try store.stagedAnchors( + sessionID: item.scan.canonicalSessionID + ) ?? [], + records: result.records, + summaryRecords: summarizeCodexRecords(result.records), + cursor: CodexSessionCursor( + currentModel: item.scan.finalModel ?? item.scan.events.last?.model ?? "unknown", + relevantLineNumber: item.scan.relevantLineCount ?? item.scan.events.count, + hasCumulativeSchema: result.cursor.hasCumulativeSchema, + previousCumulative: result.cursor.previousCumulative, + epoch: result.cursor.epoch + ), + diagnostics: result.diagnostics + ) + if let existing = try store.session(path: item.path.path), + candidate.hasSameStoredAccounting(as: existing) { + if let validationFingerprint = candidate.validationFingerprint, + validationFingerprint != existing.validationFingerprint { + try store.updateValidationFingerprint( + validationFingerprint, + path: item.path.path + ) + } + } else { + try store.stage(session: candidate) + } + } + + try store.commitStaged(deletedPaths: deletedPaths) + committed = true + let cachedSessionCount = try store.sessionCount() + guard cachedSessionCount == paths.count else { + throw CodexIncrementalStoreError.incompleteCache( + expected: paths.count, + actual: cachedSessionCount + ) + } + + var seenRequestIDs = Set() + var records = [UsageRecord]() + var summaries = [CodexSummaryKey: CodexSummaryAccumulator]() + var diagnostics = CodexCollectionDiagnostics() + var sourceRecordCount = 0 + try store.forEachContribution(detailed: requiresDetailedRecords) { contribution in + sourceRecordCount += contribution.recordCount + diagnostics.add(contribution.diagnostics) + for record in contribution.records { + if let requestID = record.requestID, + !seenRequestIDs.insert(requestID).inserted { + diagnostics.duplicateRecords += 1 + continue + } + if requiresDetailedRecords { + records.append(record) + } else { + addCodexSummary(record, to: &summaries) + } + } } - return collectCodexFromSQLite() ?? jsonlResult + if !requiresDetailedRecords { + records = codexSummaryRecords(summaries) + } + return codexCollectorResult( + records: records, + diagnostics: diagnostics, + fileCount: paths.count, + sourceRecordCount: sourceRecordCount + ) } private static func collectCodexFromSQLite() -> CollectorResult? { @@ -261,11 +873,12 @@ enum UsageCollector { scans.map { ($0.canonicalSessionID, $0) }, uniquingKeysWith: { first, _ in first } ) + let anchorsBySessionID = scansBySessionID.mapValues(codexAnchors) var records: [UsageRecord] = [] var diagnostics = CodexCollectionDiagnostics() var seenRequestIDs = Set() for scan in scans.sorted(by: { $0.sourcePath < $1.sourcePath }) { - let parentAnchor = codexForkAnchor(for: scan, scansBySessionID: scansBySessionID) + let parentAnchor = codexForkAnchor(for: scan, anchorsBySessionID: anchorsBySessionID) let result = codexDeltaRecords( from: scan, parentAnchor: parentAnchor, @@ -275,20 +888,32 @@ enum UsageCollector { diagnostics.add(result.diagnostics) } + return codexCollectorResult( + records: records, + diagnostics: diagnostics, + fileCount: paths.count + ) + } + + private static func codexCollectorResult( + records: [UsageRecord], + diagnostics: CodexCollectionDiagnostics, + fileCount: Int, + sourceRecordCount: Int? = nil + ) -> CollectorResult { let breakdown = records.reduce(into: TokenUsageCounts()) { partial, record in partial.add(record.usage) } - return CollectorResult( records: records, source: SourceInfo( status: records.isEmpty ? "missing" : "ok", - files: paths.count, - records: records.count, + files: fileCount, + records: sourceRecordCount ?? records.count, rawRecords: diagnostics.rawRecords, dedupedRecords: diagnostics.duplicateRecords + diagnostics.inheritedRecords, skippedRecords: diagnostics.skippedRecords, - strategy: "total_token_usage_delta_v6_with_legacy_fallback", + strategy: "total_token_usage_delta_v6_with_incremental_cache", exactRecords: diagnostics.exactRecords, legacyRecords: diagnostics.legacyRecords, duplicateRecords: diagnostics.duplicateRecords, @@ -296,7 +921,7 @@ enum UsageCollector { inheritedRecords: diagnostics.inheritedRecords, inheritedTokens: diagnostics.inheritedTokens, unknownBreakdownRecords: diagnostics.unknownBreakdownRecords, - accountingRevision: CollectorCache.currentVersion, + accountingRevision: codexAccountingRevision, tokenBreakdown: SourceTokenBreakdown( processedTokens: breakdown.totalTokens, inputTokens: breakdown.inputTokens, @@ -314,6 +939,47 @@ enum UsageCollector { ) } + private static func summarizeCodexRecords(_ records: [UsageRecord]) -> [UsageRecord] { + var summaries = [CodexSummaryKey: CodexSummaryAccumulator]() + for record in records { + addCodexSummary(record, to: &summaries) + } + return codexSummaryRecords(summaries) + } + + private static func addCodexSummary( + _ record: UsageRecord, + to summaries: inout [CodexSummaryKey: CodexSummaryAccumulator] + ) { + let hour = record.timestampEpoch.map(hour(fromEpoch:)) + ?? hour(fromISO: record.timestamp) + let key = CodexSummaryKey(date: record.date, model: record.model, hour: hour) + summaries[key, default: CodexSummaryAccumulator()].add(record) + } + + private static func codexSummaryRecords( + _ summaries: [CodexSummaryKey: CodexSummaryAccumulator] + ) -> [UsageRecord] { + summaries.map { key, value in + UsageRecord( + date: key.date, + timestamp: value.timestamp, + timestampEpoch: value.timestampEpoch, + tool: "Codex", + model: key.model, + usage: value.usage, + source: .nativeCodex, + dataSource: "codex_incremental_summary", + modelRequestCount: value.modelRequestCount, + toolCallCount: value.toolCallCount + ) + }.sorted { + if $0.date != $1.date { return $0.date < $1.date } + if $0.model != $1.model { return $0.model < $1.model } + return ($0.timestampEpoch ?? -1) < ($1.timestampEpoch ?? -1) + } + } + private static func stableCodexScan( at path: URL ) -> (scan: CodexSessionScan, isStable: Bool, metadata: (size: UInt64, modificationTime: TimeInterval))? { @@ -326,6 +992,246 @@ enum UsageCollector { return (scan, metadata(before, matches: after), after) } + private static func incrementalCodexAppend( + at path: URL, + cached: CodexCachedSession + ) -> CodexCachedSession? { + guard let tail = scanCodexSessionTail( + at: path, + fromOffset: cached.size, + cursor: cached.cursor + ) else { + return nil + } + + var records = cached.records + var diagnostics = cached.diagnostics + var cursor = cached.cursor + diagnostics.rawRecords += tail.events.count + var seenRequestIDs = Set(records.compactMap(\.requestID)) + let scan = CodexSessionScan( + canonicalSessionID: cached.sessionID, + createdAt: cached.createdAtEpoch.map { isoFormatter.string(from: Date(timeIntervalSince1970: $0)) }, + parentSessionID: cached.parentSessionID, + sourcePath: cached.path, + events: tail.events, + finalModel: tail.currentModel, + relevantLineCount: tail.relevantLineNumber + ) + + if cursor.hasCumulativeSchema { + var previous = cursor.previousCumulative + var epoch = cursor.epoch + for index in tail.events.indices { + let event = tail.events[index] + guard event.cumulativePresent else { + diagnostics.skippedRecords += 1 + continue + } + guard let current = event.cumulative, + current.totalTokens > 0, + let day = dayString(for: event) + else { + diagnostics.skippedRecords += 1 + continue + } + + let deltaTotal: Int + let isReset: Bool + if let previous { + if current.totalTokens == previous.totalTokens { + diagnostics.duplicateRecords += 1 + continue + } + if current.totalTokens > previous.totalTokens { + deltaTotal = current.totalTokens - previous.totalTokens + isReset = false + } else if isCodexContextWindowSentinel(event) { + diagnostics.skippedRecords += 1 + continue + } else if isCredibleCodexReset( + at: index, + events: tail.events, + current: current, + previous: previous + ) { + epoch += 1 + diagnostics.counterResets += 1 + deltaTotal = current.totalTokens + isReset = true + } else { + // Re-read the complete session so an ambiguous reset can be + // reconsidered when a following cumulative event arrives. + return nil + } + } else { + deltaTotal = current.totalTokens + isReset = false + } + + guard deltaTotal > 0 else { continue } + let componentResult = codexIncrementUsage( + current: current, + previous: isReset ? nil : previous, + last: event.last, + total: deltaTotal + ) + let requestID = "codex:cumulative:\(cached.sessionID):\(epoch):\(current.totalTokens)" + guard seenRequestIDs.insert(requestID).inserted else { + diagnostics.duplicateRecords += 1 + previous = current + continue + } + records.append( + codexUsageRecord( + scan: scan, + event: event, + day: day, + usage: componentResult.usage, + requestID: requestID, + dataSource: componentResult.hasKnownBreakdown + ? "codex_total_usage_delta" + : "codex_total_usage_delta_unknown_breakdown" + ) + ) + diagnostics.exactRecords += 1 + if !componentResult.hasKnownBreakdown { + diagnostics.unknownBreakdownRecords += 1 + } + previous = current + } + cursor.previousCumulative = previous + cursor.epoch = epoch + } else { + guard !tail.events.contains(where: \.cumulativePresent) else { + return nil + } + for event in tail.events { + guard let usage = event.last, + usage.totalTokens > 0, + let timestamp = event.timestamp, + let day = dayString(for: event) + else { + diagnostics.skippedRecords += 1 + continue + } + let requestID = "codex:legacy:\(cached.sessionID):\(timestamp):\(usage.fingerprint)" + guard seenRequestIDs.insert(requestID).inserted else { + diagnostics.duplicateRecords += 1 + continue + } + records.append( + codexUsageRecord( + scan: scan, + event: event, + day: day, + usage: usage, + requestID: requestID, + dataSource: "codex_last_usage_legacy_estimate" + ) + ) + diagnostics.legacyRecords += 1 + if !isCodexBreakdownConsistent(usage, total: usage.totalTokens) { + diagnostics.unknownBreakdownRecords += 1 + } + } + } + + cursor.currentModel = tail.currentModel + cursor.relevantLineNumber = tail.relevantLineNumber + return CodexCachedSession( + path: cached.path, + size: tail.processedSize, + modificationTime: tail.modificationTime, + fingerprint: tail.fingerprint, + validationFingerprint: nil, + sessionID: cached.sessionID, + createdAtEpoch: cached.createdAtEpoch, + parentSessionID: cached.parentSessionID, + anchors: (cached.anchors + codexAnchors(for: scan)) + .sorted { $0.timestamp < $1.timestamp }, + records: records, + summaryRecords: summarizeCodexRecords(records), + cursor: cursor, + diagnostics: diagnostics + ) + } + + private static func scanCodexSessionTail( + at path: URL, + fromOffset offset: UInt64, + cursor: CodexSessionCursor + ) -> CodexSessionTail? { + guard let metadata = fileMetadata(for: path), metadata.size > offset else { return nil } + + do { + if offset > 0 { + let handle = try FileHandle(forReadingFrom: path) + defer { try? handle.close() } + try handle.seek(toOffset: offset - 1) + guard try handle.read(upToCount: 1)?.first == 0x0A else { return nil } + } + + var currentModel = cursor.currentModel + var relevantLineNumber = cursor.relevantLineNumber + var events = [CodexTokenEvent]() + var encounteredSessionMetadata = false + let processedSize = try forEachCompleteLine( + in: path, + fromOffset: offset, + matchingAny: ["session_meta", "turn_context", "token_count"] + ) { line in + autoreleasepool { + relevantLineNumber += 1 + guard line.utf8.count <= maxRelevantLineBytes, + let obj = jsonObject(line) + else { return } + let type = obj["type"] as? String + let payload = obj["payload"] as? [String: Any] + if type == "session_meta" { + encounteredSessionMetadata = true + return + } + if type == "turn_context" { + currentModel = modelKey(payload?["model"] as? String ?? currentModel) + } + guard type == "event_msg", + payload?["type"] as? String == "token_count", + let info = payload?["info"] as? [String: Any] + else { return } + let timestamp = nonEmptyString(obj["timestamp"] as? String) + events.append( + CodexTokenEvent( + timestamp: timestamp, + timestampEpoch: timestamp.flatMap(parseISO)?.timeIntervalSince1970, + model: currentModel, + cumulativePresent: info.keys.contains("total_token_usage"), + cumulative: (info["total_token_usage"] as? [String: Any]).map(normalizeCodexUsage), + last: (info["last_token_usage"] as? [String: Any]).map(normalizeCodexUsage), + modelContextWindow: integerValue(info["model_context_window"] as Any), + lineNumber: relevantLineNumber + ) + ) + } + } + + guard processedSize > offset, !encounteredSessionMetadata else { return nil } + guard let finalMetadata = fileMetadata(for: path), + let fingerprint = contentFingerprint(for: path, size: processedSize) + else { return nil } + return CodexSessionTail( + events: events, + currentModel: currentModel, + relevantLineNumber: relevantLineNumber, + processedSize: processedSize, + modificationTime: finalMetadata.modificationTime, + fingerprint: fingerprint + ) + } catch { + return nil + } + } + private static func scanCodexSessionFile(at path: URL) -> CodexSessionScan? { guard FileManager.default.isReadableFile(atPath: path.path) else { return nil } var canonicalSessionID: String? @@ -360,12 +1266,14 @@ enum UsageCollector { return } + let timestamp = nonEmptyString(obj["timestamp"] as? String) let cumulativePresent = info.keys.contains("total_token_usage") let cumulative = (info["total_token_usage"] as? [String: Any]).map(normalizeCodexUsage) let last = (info["last_token_usage"] as? [String: Any]).map(normalizeCodexUsage) events.append( CodexTokenEvent( - timestamp: nonEmptyString(obj["timestamp"] as? String), + timestamp: timestamp, + timestampEpoch: timestamp.flatMap(parseISO)?.timeIntervalSince1970, model: currentModel, cumulativePresent: cumulativePresent, cumulative: cumulative, @@ -385,7 +1293,9 @@ enum UsageCollector { createdAt: createdAt, parentSessionID: parentSessionID, sourcePath: path.path, - events: events + events: events, + finalModel: currentModel, + relevantLineCount: relevantLineNumber ) } @@ -404,31 +1314,58 @@ enum UsageCollector { private static func codexForkAnchor( for scan: CodexSessionScan, - scansBySessionID: [String: CodexSessionScan] + anchorsBySessionID: [String: [CodexAnchor]] ) -> TokenUsageCounts? { guard let parentID = scan.parentSessionID, - let parent = scansBySessionID[parentID], - let childCreatedAt = scan.createdAt.flatMap(parseISO) + let anchors = anchorsBySessionID[parentID], + let childCreatedAt = scan.createdAt.flatMap(parseISO)?.timeIntervalSince1970 else { return nil } - return parent.events.last(where: { event in + return codexAnchor(atOrBefore: childCreatedAt, anchors: anchors) + } + + private static func codexAnchors(for scan: CodexSessionScan) -> [CodexAnchor] { + scan.events.compactMap { event in guard event.cumulativePresent, let usage = event.cumulative, usage.totalTokens > 0, - let timestamp = event.timestamp.flatMap(parseISO) + let timestamp = event.timestampEpoch + ?? event.timestamp.flatMap(parseISO)?.timeIntervalSince1970 else { - return false + return nil + } + return CodexAnchor(timestamp: timestamp, usage: usage) + }.sorted { $0.timestamp < $1.timestamp } + } + + private static func codexAnchor( + atOrBefore timestamp: TimeInterval, + anchors: [CodexAnchor] + ) -> TokenUsageCounts? { + var lower = 0 + var upper = anchors.count + while lower < upper { + let middle = lower + (upper - lower) / 2 + if anchors[middle].timestamp <= timestamp { + lower = middle + 1 + } else { + upper = middle } - return timestamp <= childCreatedAt - })?.cumulative + } + guard lower > 0 else { return nil } + return anchors[lower - 1].usage } private static func codexDeltaRecords( from scan: CodexSessionScan, parentAnchor: TokenUsageCounts?, seenRequestIDs: inout Set - ) -> (records: [UsageRecord], diagnostics: CodexCollectionDiagnostics) { + ) -> ( + records: [UsageRecord], + diagnostics: CodexCollectionDiagnostics, + cursor: CodexDeltaCursor + ) { var diagnostics = CodexCollectionDiagnostics(rawRecords: scan.events.count) var records: [UsageRecord] = [] let hasCumulativeSchema = scan.events.contains { $0.cumulativePresent } @@ -438,7 +1375,7 @@ enum UsageCollector { guard let usage = event.last, usage.totalTokens > 0, let timestamp = event.timestamp, - let day = dayString(fromISO: timestamp) + let day = dayString(for: event) else { diagnostics.skippedRecords += 1 continue @@ -463,7 +1400,15 @@ enum UsageCollector { diagnostics.unknownBreakdownRecords += 1 } } - return (records, diagnostics) + return ( + records, + diagnostics, + CodexDeltaCursor( + hasCumulativeSchema: false, + previousCumulative: nil, + epoch: 0 + ) + ) } var startIndex = 0 @@ -488,8 +1433,7 @@ enum UsageCollector { } guard let current = event.cumulative, current.totalTokens > 0, - let timestamp = event.timestamp, - let day = dayString(fromISO: timestamp) + let day = dayString(for: event) else { diagnostics.skippedRecords += 1 continue @@ -558,7 +1502,15 @@ enum UsageCollector { } previous = current } - return (records, diagnostics) + return ( + records, + diagnostics, + CodexDeltaCursor( + hasCumulativeSchema: true, + previousCumulative: previous, + epoch: epoch + ) + ) } private static func codexUsageRecord( @@ -572,6 +1524,7 @@ enum UsageCollector { UsageRecord( date: day, timestamp: event.timestamp, + timestampEpoch: event.timestampEpoch, tool: "Codex", model: event.model, usage: usage, @@ -1524,7 +2477,8 @@ enum UsageCollector { for record in records { let cost = record.costUSD ?? estimateCost(usage: record.usage, tool: record.tool, model: record.model) daily[record.date, default: DailyAccumulator(date: record.date)].add(record: record, cost: cost) - let recordHour = hour(fromISO: record.timestamp) + let recordHour = record.timestampEpoch.map(hour(fromEpoch:)) + ?? hour(fromISO: record.timestamp) if let hour = recordHour { rhythms[record.date, default: RhythmAccumulator(date: record.date)] .add(tokens: record.usage.totalTokens, hour: hour) @@ -1736,10 +2690,12 @@ enum UsageCollector { do { include(withUnsafeBytes(of: size.littleEndian) { Data($0) }) - include(try handle.read(upToCount: chunkSize) ?? Data()) - if size > UInt64(chunkSize) { - try handle.seek(toOffset: size - UInt64(chunkSize)) - include(try handle.read(upToCount: chunkSize) ?? Data()) + let leadingCount = min(chunkSize, Int(clamping: size)) + include(try handle.read(upToCount: leadingCount) ?? Data()) + if size > UInt64(leadingCount) { + let trailingCount = min(chunkSize, Int(clamping: size)) + try handle.seek(toOffset: size - UInt64(trailingCount)) + include(try handle.read(upToCount: trailingCount) ?? Data()) } return String(format: "%016llx", hash) } catch { @@ -1747,6 +2703,30 @@ enum UsageCollector { } } + private static func fullContentFingerprint(for url: URL, size: UInt64) -> String? { + guard let handle = try? FileHandle(forReadingFrom: url) else { return nil } + defer { try? handle.close() } + + var hasher = SHA256() + hasher.update(data: withUnsafeBytes(of: size.littleEndian) { Data($0) }) + var remaining = size + do { + while remaining > 0 { + let requested = min(1_048_576, Int(clamping: remaining)) + guard let chunk = try autoreleasepool(invoking: { + try handle.read(upToCount: requested) + }), !chunk.isEmpty else { + return nil + } + hasher.update(data: chunk) + remaining -= UInt64(chunk.count) + } + return hasher.finalize().map { String(format: "%02x", $0) }.joined() + } catch { + return nil + } + } + private static func loadCache() -> CollectorCacheLoad { loadCache(at: AppPaths.collectorCacheJSON) } @@ -1789,6 +2769,13 @@ enum UsageCollector { at: url.deletingLastPathComponent(), withIntermediateDirectories: true ) + if let attributes = try? FileManager.default.attributesOfItem(atPath: url.path), + let existingSize = (attributes[.size] as? NSNumber)?.intValue, + existingSize == data.count, + let existing = try? Data(contentsOf: url), + existing == data { + return + } try data.write(to: url, options: .atomic) } catch { // Cache misses should never prevent the app from showing fresh usage. @@ -1799,36 +2786,105 @@ enum UsageCollector { calendar.date(byAdding: .day, value: -max(7, historyDays + 1), to: Date()) } - private static func recordsInHistoryWindow( - _ records: [UsageRecord], - historyDays: Int, - now: Date - ) -> [UsageRecord] { - let inclusiveDays = max(1, historyDays) - let today = calendar.startOfDay(for: now) - guard let firstDay = calendar.date( - byAdding: .day, - value: -(inclusiveDays - 1), - to: today - ) else { - return records - } - let firstDayString = dayFormatter.string(from: firstDay) - let todayString = dayFormatter.string(from: today) - return records.filter { - $0.date >= firstDayString && $0.date <= todayString + private static func recordsInHistoryWindow( + _ records: [UsageRecord], + historyDays: Int, + now: Date + ) -> [UsageRecord] { + let inclusiveDays = max(1, historyDays) + let today = calendar.startOfDay(for: now) + guard let firstDay = calendar.date( + byAdding: .day, + value: -(inclusiveDays - 1), + to: today + ) else { + return records + } + let firstDayString = dayFormatter.string(from: firstDay) + let todayString = dayFormatter.string(from: today) + return records.filter { + $0.date >= firstDayString && $0.date <= todayString + } + } + + private static func forEachLine(in url: URL, matchingAny markers: [String] = [], _ body: (String) -> Void) throws { + let handle = try FileHandle(forReadingFrom: url) + defer { try? handle.close() } + + let newline = Data([0x0A]) + let markerData = markers.map { Data($0.utf8) } + var buffer = Data() + buffer.reserveCapacity(128 * 1024) + var discardingOversizedLine = false + + func processLine(_ lineData: Data) { + guard lineMatches(lineData, markers: markerData), + let line = String(data: lineData, encoding: .utf8), + !line.isEmpty + else { + return + } + body(line) + } + + while try autoreleasepool(invoking: { () throws -> Bool in + guard let chunk = try handle.read(upToCount: 64 * 1024), !chunk.isEmpty else { + return false + } + buffer.append(chunk) + + var consumedEnd = buffer.startIndex + var lineStart = buffer.startIndex + var searchRange = buffer.startIndex.. lineStart { + let lineData = buffer.subdata(in: lineStart.. buffer.startIndex { + buffer.removeSubrange(buffer.startIndex.. maxRelevantLineBytes { + discardingOversizedLine = true + buffer.removeAll(keepingCapacity: true) + } + return true + }) {} + + if !discardingOversizedLine, + !buffer.isEmpty, + buffer.count <= maxRelevantLineBytes { + processLine(buffer) } } - private static func forEachLine(in url: URL, matchingAny markers: [String] = [], _ body: (String) -> Void) throws { + @discardableResult + private static func forEachCompleteLine( + in url: URL, + fromOffset offset: UInt64, + matchingAny markers: [String] = [], + _ body: (String) -> Void + ) throws -> UInt64 { let handle = try FileHandle(forReadingFrom: url) defer { try? handle.close() } + try handle.seek(toOffset: offset) let newline = Data([0x0A]) let markerData = markers.map { Data($0.utf8) } var buffer = Data() buffer.reserveCapacity(128 * 1024) var discardingOversizedLine = false + var discardedIncompleteBytes = 0 + var processedSize = offset func processLine(_ lineData: Data) { guard lineMatches(lineData, markers: markerData), @@ -1840,9 +2896,9 @@ enum UsageCollector { body(line) } - while true { + while try autoreleasepool(invoking: { () throws -> Bool in guard let chunk = try handle.read(upToCount: 64 * 1024), !chunk.isEmpty else { - break + return false } buffer.append(chunk) @@ -1854,8 +2910,7 @@ enum UsageCollector { if discardingOversizedLine { discardingOversizedLine = false } else if lineEnd > lineStart { - let lineData = buffer.subdata(in: lineStart.. buffer.startIndex { + let consumedBytes = buffer.distance(from: buffer.startIndex, to: consumedEnd) + processedSize += UInt64(discardedIncompleteBytes + consumedBytes) + discardedIncompleteBytes = 0 buffer.removeSubrange(buffer.startIndex.. maxRelevantLineBytes { discardingOversizedLine = true + discardedIncompleteBytes += buffer.count buffer.removeAll(keepingCapacity: true) } - } + return true + }) {} - if !discardingOversizedLine, - !buffer.isEmpty, - buffer.count <= maxRelevantLineBytes { - processLine(buffer) - } + return processedSize } private static func lineMatches(_ data: Data, markers: [Data]) -> Bool { @@ -1997,6 +3053,17 @@ enum UsageCollector { return dayFormatter.string(from: date) } + private static func dayString(for event: CodexTokenEvent) -> String? { + if let timestamp = event.timestampEpoch { + return dayFormatter.string(from: Date(timeIntervalSince1970: timestamp)) + } + return event.timestamp.flatMap(dayString(fromISO:)) + } + + private static func hour(fromEpoch value: TimeInterval) -> Int { + calendar.component(.hour, from: Date(timeIntervalSince1970: value)) + } + private static func hour(fromISO value: String?) -> Int? { guard let value, let date = parseISO(value) else { return nil } return calendar.component(.hour, from: date) @@ -2265,6 +3332,797 @@ private struct CollectorResult { var source: SourceInfo } +private struct CodexCollectionOutcome { + var result: CollectorResult + var usedIncrementalStore: Bool +} + +private struct PendingCodexSession { + var path: URL + var metadata: (size: UInt64, modificationTime: TimeInterval) + var fingerprint: String + var validationFingerprint: String? = nil + var scan: CodexSessionScan +} + +private struct StoredCodexSessionMetadata { + var size: UInt64 + var modificationTime: TimeInterval + var fingerprint: String + var validationFingerprint: String? + var sessionID: String +} + +private struct CodexCachedSession { + var path: String + var size: UInt64 + var modificationTime: TimeInterval + var fingerprint: String + var validationFingerprint: String? = nil + var sessionID: String + var createdAtEpoch: TimeInterval? + var parentSessionID: String? + var anchors: [CodexAnchor] + var records: [UsageRecord] + var summaryRecords: [UsageRecord] + var cursor: CodexSessionCursor + var diagnostics: CodexCollectionDiagnostics + + func hasSameStoredAccounting(as other: CodexCachedSession) -> Bool { + path == other.path + && size == other.size + && abs(modificationTime - other.modificationTime) < 0.001 + && fingerprint == other.fingerprint + && sessionID == other.sessionID + && createdAtEpoch == other.createdAtEpoch + && parentSessionID == other.parentSessionID + && anchors == other.anchors + && records == other.records + && summaryRecords == other.summaryRecords + && cursor == other.cursor + && diagnostics == other.diagnostics + } +} + +private struct CodexCachedContribution { + var records: [UsageRecord] + var recordCount: Int + var diagnostics: CodexCollectionDiagnostics +} + +private struct CodexSummaryKey: Hashable { + var date: String + var model: String + var hour: Int? +} + +private struct CodexSummaryAccumulator { + var timestamp: String? + var timestampEpoch: TimeInterval? + var usage = TokenUsageCounts() + var modelRequestCount = 0 + var toolCallCount = 0 + + mutating func add(_ record: UsageRecord) { + timestamp = timestamp ?? record.timestamp + timestampEpoch = timestampEpoch ?? record.timestampEpoch + usage.add(record.usage) + modelRequestCount += max(0, record.modelRequestCount) + toolCallCount += max(0, record.toolCallCount) + } +} + +private enum CodexIncrementalStoreError: LocalizedError { + case sqlite(String) + case corruptPayload(String) + case unstableSource(String) + case incompleteCache(expected: Int, actual: Int) + + var errorDescription: String? { + switch self { + case let .sqlite(message): + return "Incremental cache error: \(message)" + case let .corruptPayload(context): + return "Incremental cache payload is corrupt: \(context)" + case .unstableSource: + return "A Codex session changed while it was being collected." + case let .incompleteCache(expected, actual): + return "Incremental cache is incomplete (expected \(expected), got \(actual))." + } + } + + var shouldRebuildCache: Bool { + switch self { + case .corruptPayload: + return true + case let .sqlite(message): + let normalized = message.lowercased() + return normalized.contains("not a database") + || normalized.contains("database disk image is malformed") + || normalized.contains("database malformed") + case .incompleteCache: + return true + case .unstableSource: + return false + } + } +} + +private final class CodexIncrementalStore { + private static let schemaVersion: Int32 = 6 + private static let transient = unsafeBitCast(-1, to: sqlite3_destructor_type.self) + + private var database: OpaquePointer? + private var stagingTransactionActive = false + + static func discardDatabase(at url: URL) { + let fileManager = FileManager.default + for path in [url.path, url.path + "-wal", url.path + "-shm"] { + guard fileManager.fileExists(atPath: path) else { continue } + try? fileManager.removeItem(atPath: path) + } + } + + init(url: URL) throws { + try FileManager.default.createDirectory( + at: url.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + let flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX + guard sqlite3_open_v2(url.path, &database, flags, nil) == SQLITE_OK else { + let message = database.map { String(cString: sqlite3_errmsg($0)) } ?? "open failed" + if let database { sqlite3_close(database) } + database = nil + throw CodexIncrementalStoreError.sqlite(message) + } + do { + sqlite3_busy_timeout(database, 2_000) + try execute("PRAGMA journal_mode=WAL") + try execute("PRAGMA synchronous=NORMAL") + try migrateIfNeeded() + } catch { + if let database { + sqlite3_close(database) + } + database = nil + throw error + } + } + + deinit { + if let database { + sqlite3_close(database) + } + } + + func metadataByPath() throws -> [String: StoredCodexSessionMetadata] { + let statement = try prepare( + """ + SELECT path, size, modification_time, fingerprint, + validation_fingerprint, session_id + FROM codex_sessions + """ + ) + defer { sqlite3_finalize(statement) } + var result = [String: StoredCodexSessionMetadata]() + while sqlite3_step(statement) == SQLITE_ROW { + guard let path = columnText(statement, index: 0), + let fingerprint = columnText(statement, index: 3), + let sessionID = columnText(statement, index: 5) + else { continue } + result[path] = StoredCodexSessionMetadata( + size: UInt64(max(0, sqlite3_column_int64(statement, 1))), + modificationTime: sqlite3_column_double(statement, 2), + fingerprint: fingerprint, + validationFingerprint: columnText(statement, index: 4), + sessionID: sessionID + ) + } + try checkFinalStep(statement) + return result + } + + func childPaths(parentSessionID: String) throws -> [String] { + let statement = try prepare( + "SELECT path FROM codex_sessions WHERE parent_session_id = ? ORDER BY path" + ) + defer { sqlite3_finalize(statement) } + bind(parentSessionID, to: statement, index: 1) + var result = [String]() + while sqlite3_step(statement) == SQLITE_ROW { + if let path = columnText(statement, index: 0) { + result.append(path) + } + } + try checkFinalStep(statement) + return result + } + + func childPaths( + parentSessionID: String, + createdAtOnOrAfter timestamp: TimeInterval + ) throws -> [String] { + let statement = try prepare( + """ + SELECT path FROM codex_sessions + WHERE parent_session_id = ? + AND (created_at_epoch IS NULL OR created_at_epoch >= ?) + ORDER BY path + """ + ) + defer { sqlite3_finalize(statement) } + bind(parentSessionID, to: statement, index: 1) + sqlite3_bind_double(statement, 2, timestamp) + var result = [String]() + while sqlite3_step(statement) == SQLITE_ROW { + if let path = columnText(statement, index: 0) { + result.append(path) + } + } + try checkFinalStep(statement) + return result + } + + func anchors(sessionID: String) throws -> [CodexAnchor]? { + let statement = try prepare( + "SELECT anchors FROM codex_sessions WHERE session_id = ? ORDER BY path LIMIT 1" + ) + defer { sqlite3_finalize(statement) } + bind(sessionID, to: statement, index: 1) + let status = sqlite3_step(statement) + if status == SQLITE_DONE { return nil } + guard status == SQLITE_ROW, + let data = columnData(statement, index: 0) + else { + throw currentError() + } + return try decode([CodexAnchor].self, from: data, context: "anchors") + } + + func session(path: String) throws -> CodexCachedSession? { + let statement = try prepare( + """ + SELECT size, modification_time, fingerprint, validation_fingerprint, + session_id, created_at_epoch, parent_session_id, anchors, + records, COALESCE(summary_records, records), cursor, diagnostics + FROM codex_sessions WHERE path = ? LIMIT 1 + """ + ) + defer { sqlite3_finalize(statement) } + bind(path, to: statement, index: 1) + let status = sqlite3_step(statement) + if status == SQLITE_DONE { return nil } + guard status == SQLITE_ROW, + let fingerprint = columnText(statement, index: 2), + let sessionID = columnText(statement, index: 4), + let anchorsData = columnData(statement, index: 7), + let recordsData = columnData(statement, index: 8), + let summaryData = columnData(statement, index: 9), + let cursorData = columnData(statement, index: 10), + let diagnosticsData = columnData(statement, index: 11) + else { return nil } + return CodexCachedSession( + path: path, + size: UInt64(max(0, sqlite3_column_int64(statement, 0))), + modificationTime: sqlite3_column_double(statement, 1), + fingerprint: fingerprint, + validationFingerprint: columnText(statement, index: 3), + sessionID: sessionID, + createdAtEpoch: sqlite3_column_type(statement, 5) == SQLITE_NULL + ? nil : sqlite3_column_double(statement, 5), + parentSessionID: columnText(statement, index: 6), + anchors: try decode([CodexAnchor].self, from: anchorsData, context: "session anchors"), + records: try decode([UsageRecord].self, from: recordsData, context: "session records"), + summaryRecords: try decode([UsageRecord].self, from: summaryData, context: "session summaries"), + cursor: try decode(CodexSessionCursor.self, from: cursorData, context: "session cursor"), + diagnostics: try decode( + CodexCollectionDiagnostics.self, + from: diagnosticsData, + context: "session diagnostics" + ) + ) + } + + func beginStaging() throws { + guard !stagingTransactionActive else { + throw CodexIncrementalStoreError.sqlite("staging transaction already active") + } + try execute("BEGIN IMMEDIATE TRANSACTION") + do { + try execute("DELETE FROM codex_staged_scans") + try execute("DELETE FROM codex_staged_sessions") + stagingTransactionActive = true + } catch { + try? execute("ROLLBACK") + throw error + } + } + + func abortStaging() { + guard stagingTransactionActive else { return } + try? execute("ROLLBACK") + stagingTransactionActive = false + } + + func updateValidationFingerprint(_ fingerprint: String, path: String) throws { + guard stagingTransactionActive else { + throw CodexIncrementalStoreError.sqlite("staging transaction is not active") + } + let statement = try prepare( + "UPDATE codex_sessions SET validation_fingerprint = ? WHERE path = ?" + ) + defer { sqlite3_finalize(statement) } + bind(fingerprint, to: statement, index: 1) + bind(path, to: statement, index: 2) + try requireDone(statement) + } + + func stage( + scan item: PendingCodexSession, + anchors: [CodexAnchor], + createdAtEpoch: TimeInterval? + ) throws { + guard stagingTransactionActive else { + throw CodexIncrementalStoreError.sqlite("staging transaction is not active") + } + let encoder = PropertyListEncoder() + encoder.outputFormat = .binary + let anchors = try encoder.encode(anchors) + let scan = try encoder.encode(item.scan) + let statement = try prepare( + """ + INSERT OR REPLACE INTO codex_staged_scans ( + path, size, modification_time, fingerprint, validation_fingerprint, + session_id, created_at_epoch, parent_session_id, anchors, scan + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """ + ) + defer { sqlite3_finalize(statement) } + bind(item.path.path, to: statement, index: 1) + sqlite3_bind_int64(statement, 2, sqlite3_int64(item.metadata.size)) + sqlite3_bind_double(statement, 3, item.metadata.modificationTime) + bind(item.fingerprint, to: statement, index: 4) + bind(item.validationFingerprint, to: statement, index: 5) + bind(item.scan.canonicalSessionID, to: statement, index: 6) + bind(createdAtEpoch, to: statement, index: 7) + bind(item.scan.parentSessionID, to: statement, index: 8) + bind(anchors, to: statement, index: 9) + bind(scan, to: statement, index: 10) + try requireDone(statement) + } + + func stagedScanPaths() throws -> [String] { + let statement = try prepare("SELECT path FROM codex_staged_scans ORDER BY path") + defer { sqlite3_finalize(statement) } + var paths = [String]() + while sqlite3_step(statement) == SQLITE_ROW { + if let path = columnText(statement, index: 0) { + paths.append(path) + } + } + try checkFinalStep(statement) + return paths + } + + func stagedScan(path: String) throws -> PendingCodexSession? { + let statement = try prepare( + """ + SELECT size, modification_time, fingerprint, validation_fingerprint, scan + FROM codex_staged_scans WHERE path = ? LIMIT 1 + """ + ) + defer { sqlite3_finalize(statement) } + bind(path, to: statement, index: 1) + let status = sqlite3_step(statement) + if status == SQLITE_DONE { return nil } + guard status == SQLITE_ROW, + let fingerprint = columnText(statement, index: 2), + let scanData = columnData(statement, index: 4) + else { throw currentError() } + return PendingCodexSession( + path: URL(fileURLWithPath: path), + metadata: ( + size: UInt64(max(0, sqlite3_column_int64(statement, 0))), + modificationTime: sqlite3_column_double(statement, 1) + ), + fingerprint: fingerprint, + validationFingerprint: columnText(statement, index: 3), + scan: try decode(CodexSessionScan.self, from: scanData, context: "staged scan") + ) + } + + func stagedAnchors(sessionID: String) throws -> [CodexAnchor]? { + for table in ["codex_staged_scans", "codex_staged_sessions"] { + let statement = try prepare( + "SELECT anchors FROM \(table) WHERE session_id = ? ORDER BY path LIMIT 1" + ) + defer { sqlite3_finalize(statement) } + bind(sessionID, to: statement, index: 1) + let status = sqlite3_step(statement) + if status == SQLITE_DONE { continue } + guard status == SQLITE_ROW, + let data = columnData(statement, index: 0) + else { throw currentError() } + return try decode([CodexAnchor].self, from: data, context: "staged anchors") + } + return nil + } + + func stage(session: CodexCachedSession) throws { + guard stagingTransactionActive else { + throw CodexIncrementalStoreError.sqlite("staging transaction is not active") + } + let encoder = PropertyListEncoder() + encoder.outputFormat = .binary + let anchors = try encoder.encode(session.anchors) + let records = try encoder.encode(session.records) + let summaryRecords = try encoder.encode(session.summaryRecords) + let cursor = try encoder.encode(session.cursor) + let diagnostics = try encoder.encode(session.diagnostics) + let statement = try prepare( + """ + INSERT OR REPLACE INTO codex_staged_sessions ( + path, size, modification_time, fingerprint, validation_fingerprint, + session_id, created_at_epoch, parent_session_id, anchors, records, + summary_records, record_count, cursor, diagnostics + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """ + ) + defer { sqlite3_finalize(statement) } + bind(session.path, to: statement, index: 1) + sqlite3_bind_int64(statement, 2, sqlite3_int64(session.size)) + sqlite3_bind_double(statement, 3, session.modificationTime) + bind(session.fingerprint, to: statement, index: 4) + bind(session.validationFingerprint, to: statement, index: 5) + bind(session.sessionID, to: statement, index: 6) + bind(session.createdAtEpoch, to: statement, index: 7) + bind(session.parentSessionID, to: statement, index: 8) + bind(anchors, to: statement, index: 9) + bind(records, to: statement, index: 10) + bind(summaryRecords, to: statement, index: 11) + sqlite3_bind_int64(statement, 12, sqlite3_int64(session.records.count)) + bind(cursor, to: statement, index: 13) + bind(diagnostics, to: statement, index: 14) + try requireDone(statement) + } + + func commitStaged(deletedPaths: Set) throws { + guard stagingTransactionActive else { + throw CodexIncrementalStoreError.sqlite("staging transaction is not active") + } + do { + let stagedCount = try stagedSessionCount() + let stagedPayloadBytes = try stagedSessionPayloadBytes() + if !deletedPaths.isEmpty { + let statement = try prepare("DELETE FROM codex_sessions WHERE path = ?") + defer { sqlite3_finalize(statement) } + for path in deletedPaths { + sqlite3_reset(statement) + sqlite3_clear_bindings(statement) + bind(path, to: statement, index: 1) + try requireDone(statement) + } + } + if stagedCount > 0 { + try execute( + """ + INSERT OR REPLACE INTO codex_sessions ( + path, size, modification_time, fingerprint, validation_fingerprint, + session_id, created_at_epoch, parent_session_id, anchors, records, + summary_records, record_count, cursor, diagnostics + ) + SELECT path, size, modification_time, fingerprint, + validation_fingerprint, session_id, created_at_epoch, + parent_session_id, anchors, records, summary_records, + record_count, cursor, diagnostics + FROM codex_staged_sessions + """ + ) + } + if stagedCount > 0 || !deletedPaths.isEmpty { + try execute( + """ + INSERT INTO cache_meta(key, value) VALUES ('generation', '1') + ON CONFLICT(key) DO UPDATE SET value = CAST(value AS INTEGER) + 1 + """ + ) + let logicalWriteBytes = stagedPayloadBytes * 2 + try execute( + """ + INSERT INTO cache_meta(key, value) + VALUES ('last_logical_write_bytes', '\(logicalWriteBytes)') + ON CONFLICT(key) DO UPDATE SET value = excluded.value + """ + ) + } + try execute("DELETE FROM codex_staged_scans") + try execute("DELETE FROM codex_staged_sessions") + try execute("COMMIT") + stagingTransactionActive = false + } catch { + try? execute("ROLLBACK") + stagingTransactionActive = false + throw error + } + } + + private func stagedSessionCount() throws -> Int { + let statement = try prepare("SELECT COUNT(*) FROM codex_staged_sessions") + defer { sqlite3_finalize(statement) } + guard sqlite3_step(statement) == SQLITE_ROW else { throw currentError() } + return Int(sqlite3_column_int64(statement, 0)) + } + + private func stagedSessionPayloadBytes() throws -> Int { + let statement = try prepare( + """ + SELECT COALESCE(SUM( + LENGTH(anchors) + LENGTH(records) + LENGTH(summary_records) + + LENGTH(cursor) + LENGTH(diagnostics) + ), 0) + FROM codex_staged_sessions + """ + ) + defer { sqlite3_finalize(statement) } + guard sqlite3_step(statement) == SQLITE_ROW else { throw currentError() } + return Int(sqlite3_column_int64(statement, 0)) + } + + func sessionCount() throws -> Int { + let statement = try prepare("SELECT COUNT(*) FROM codex_sessions") + defer { sqlite3_finalize(statement) } + guard sqlite3_step(statement) == SQLITE_ROW else { throw currentError() } + return Int(sqlite3_column_int64(statement, 0)) + } + + func forEachContribution( + detailed: Bool, + _ body: (CodexCachedContribution) throws -> Void + ) throws { + let statement = try prepare( + detailed + ? "SELECT records, diagnostics, record_count FROM codex_sessions ORDER BY path" + : """ + SELECT CASE + WHEN session_id IN ( + SELECT session_id FROM codex_sessions + GROUP BY session_id HAVING COUNT(*) > 1 + ) THEN records + ELSE COALESCE(summary_records, records) + END, + diagnostics, + record_count + FROM codex_sessions + ORDER BY path + """ + ) + defer { sqlite3_finalize(statement) } + while sqlite3_step(statement) == SQLITE_ROW { + guard let recordsData = columnData(statement, index: 0), + let diagnosticsData = columnData(statement, index: 1) + else { throw currentError() } + try body( + CodexCachedContribution( + records: try decode( + [UsageRecord].self, + from: recordsData, + context: "contribution records" + ), + recordCount: Int(sqlite3_column_int64(statement, 2)), + diagnostics: try decode( + CodexCollectionDiagnostics.self, + from: diagnosticsData, + context: "contribution diagnostics" + ) + ) + ) + } + try checkFinalStep(statement) + } + + func stats() throws -> CodexIncrementalCacheStats { + let statement = try prepare( + """ + SELECT + COALESCE((SELECT CAST(value AS INTEGER) FROM cache_meta WHERE key = 'generation'), 0), + COUNT(*), + COALESCE(SUM(record_count), 0), + COALESCE(( + SELECT CAST(value AS INTEGER) FROM cache_meta + WHERE key = 'last_logical_write_bytes' + ), 0) + FROM codex_sessions + """ + ) + defer { sqlite3_finalize(statement) } + guard sqlite3_step(statement) == SQLITE_ROW else { throw currentError() } + return CodexIncrementalCacheStats( + generation: Int(sqlite3_column_int64(statement, 0)), + sessions: Int(sqlite3_column_int64(statement, 1)), + records: Int(sqlite3_column_int64(statement, 2)), + lastLogicalWriteBytes: Int(sqlite3_column_int64(statement, 3)) + ) + } + + private func migrateIfNeeded() throws { + guard database != nil else { throw CodexIncrementalStoreError.sqlite("database closed") } + let current = userVersion() + guard current >= 0, current <= Self.schemaVersion else { + throw CodexIncrementalStoreError.sqlite("unsupported schema version \(current)") + } + try execute( + """ + CREATE TABLE IF NOT EXISTS cache_meta ( + key TEXT PRIMARY KEY NOT NULL, + value TEXT NOT NULL + ) + """ + ) + if current > 0, current < Self.schemaVersion { + // v0.1.48 is the first public incremental-cache release. Recreate + // older development schemas so interim payloads cannot survive. + try execute("DROP TABLE IF EXISTS codex_sessions") + try execute("DROP TABLE IF EXISTS codex_staged_scans") + try execute("DROP TABLE IF EXISTS codex_staged_sessions") + try execute("DELETE FROM cache_meta") + } + try execute( + """ + CREATE TABLE IF NOT EXISTS codex_sessions ( + path TEXT PRIMARY KEY NOT NULL, + size INTEGER NOT NULL, + modification_time REAL NOT NULL, + fingerprint TEXT NOT NULL, + validation_fingerprint TEXT, + session_id TEXT NOT NULL, + created_at_epoch REAL, + parent_session_id TEXT, + anchors BLOB NOT NULL, + records BLOB NOT NULL, + summary_records BLOB, + record_count INTEGER NOT NULL, + cursor BLOB, + diagnostics BLOB NOT NULL + ) + """ + ) + try execute( + "CREATE INDEX IF NOT EXISTS codex_sessions_session_id ON codex_sessions(session_id)" + ) + try execute( + "CREATE INDEX IF NOT EXISTS codex_sessions_parent_id ON codex_sessions(parent_session_id)" + ) + try execute( + """ + CREATE TABLE IF NOT EXISTS codex_staged_scans ( + path TEXT PRIMARY KEY NOT NULL, + size INTEGER NOT NULL, + modification_time REAL NOT NULL, + fingerprint TEXT NOT NULL, + validation_fingerprint TEXT, + session_id TEXT NOT NULL, + created_at_epoch REAL, + parent_session_id TEXT, + anchors BLOB NOT NULL, + scan BLOB NOT NULL + ) + """ + ) + try execute( + "CREATE INDEX IF NOT EXISTS codex_staged_scans_session_id ON codex_staged_scans(session_id)" + ) + try execute( + """ + CREATE TABLE IF NOT EXISTS codex_staged_sessions ( + path TEXT PRIMARY KEY NOT NULL, + size INTEGER NOT NULL, + modification_time REAL NOT NULL, + fingerprint TEXT NOT NULL, + validation_fingerprint TEXT, + session_id TEXT NOT NULL, + created_at_epoch REAL, + parent_session_id TEXT, + anchors BLOB NOT NULL, + records BLOB NOT NULL, + summary_records BLOB NOT NULL, + record_count INTEGER NOT NULL, + cursor BLOB NOT NULL, + diagnostics BLOB NOT NULL + ) + """ + ) + try execute("PRAGMA user_version = \(Self.schemaVersion)") + } + + private func userVersion() -> Int32 { + guard let statement = try? prepare("PRAGMA user_version") else { return -1 } + defer { sqlite3_finalize(statement) } + guard sqlite3_step(statement) == SQLITE_ROW else { return -1 } + return sqlite3_column_int(statement, 0) + } + + private func execute(_ sql: String) throws { + guard let database else { throw CodexIncrementalStoreError.sqlite("database closed") } + var error: UnsafeMutablePointer? + guard sqlite3_exec(database, sql, nil, nil, &error) == SQLITE_OK else { + let message = error.map { String(cString: $0) } + ?? String(cString: sqlite3_errmsg(database)) + sqlite3_free(error) + throw CodexIncrementalStoreError.sqlite(message) + } + } + + private func prepare(_ sql: String) throws -> OpaquePointer { + guard let database else { throw CodexIncrementalStoreError.sqlite("database closed") } + var statement: OpaquePointer? + guard sqlite3_prepare_v2(database, sql, -1, &statement, nil) == SQLITE_OK, + let statement + else { throw currentError() } + return statement + } + + private func bind(_ value: String?, to statement: OpaquePointer, index: Int32) { + guard let value else { + sqlite3_bind_null(statement, index) + return + } + sqlite3_bind_text(statement, index, value, -1, Self.transient) + } + + private func bind(_ value: TimeInterval?, to statement: OpaquePointer, index: Int32) { + guard let value else { + sqlite3_bind_null(statement, index) + return + } + sqlite3_bind_double(statement, index, value) + } + + private func bind(_ data: Data, to statement: OpaquePointer, index: Int32) { + _ = data.withUnsafeBytes { bytes in + sqlite3_bind_blob(statement, index, bytes.baseAddress, Int32(bytes.count), Self.transient) + } + } + + private func decode( + _ type: T.Type, + from data: Data, + context: String + ) throws -> T { + do { + return try PropertyListDecoder().decode(type, from: data) + } catch { + throw CodexIncrementalStoreError.corruptPayload(context) + } + } + + private func columnText(_ statement: OpaquePointer, index: Int32) -> String? { + guard let value = sqlite3_column_text(statement, index) else { return nil } + return String(cString: value) + } + + private func columnData(_ statement: OpaquePointer, index: Int32) -> Data? { + let count = Int(sqlite3_column_bytes(statement, index)) + guard count >= 0 else { return nil } + if count == 0 { return Data() } + guard let bytes = sqlite3_column_blob(statement, index) else { return nil } + return Data(bytes: bytes, count: count) + } + + private func requireDone(_ statement: OpaquePointer) throws { + guard sqlite3_step(statement) == SQLITE_DONE else { throw currentError() } + } + + private func checkFinalStep(_ statement: OpaquePointer) throws { + let status = sqlite3_errcode(database) + guard status == SQLITE_OK || status == SQLITE_DONE else { throw currentError() } + } + + private func currentError() -> CodexIncrementalStoreError { + guard let database else { return .sqlite("database closed") } + return .sqlite(String(cString: sqlite3_errmsg(database))) + } +} + private struct CollectorCache: Codable { static let currentVersion = UsageCollector.codexAccountingRevision @@ -2292,10 +4150,13 @@ private struct CodexSessionScan: Codable { var parentSessionID: String? var sourcePath: String var events: [CodexTokenEvent] + var finalModel: String? = nil + var relevantLineCount: Int? = nil } private struct CodexTokenEvent: Codable { var timestamp: String? + var timestampEpoch: TimeInterval? = nil var model: String var cumulativePresent: Bool var cumulative: TokenUsageCounts? @@ -2304,7 +4165,35 @@ private struct CodexTokenEvent: Codable { var lineNumber: Int } -private struct CodexCollectionDiagnostics { +private struct CodexAnchor: Codable, Equatable { + var timestamp: TimeInterval + var usage: TokenUsageCounts +} + +private struct CodexDeltaCursor { + var hasCumulativeSchema: Bool + var previousCumulative: TokenUsageCounts? + var epoch: Int +} + +private struct CodexSessionCursor: Codable, Equatable { + var currentModel: String + var relevantLineNumber: Int + var hasCumulativeSchema: Bool + var previousCumulative: TokenUsageCounts? + var epoch: Int +} + +private struct CodexSessionTail { + var events: [CodexTokenEvent] + var currentModel: String + var relevantLineNumber: Int + var processedSize: UInt64 + var modificationTime: TimeInterval + var fingerprint: String +} + +private struct CodexCollectionDiagnostics: Codable, Equatable { var rawRecords = 0 var exactRecords = 0 var legacyRecords = 0 @@ -2328,9 +4217,10 @@ private struct CodexCollectionDiagnostics { } } -private struct UsageRecord: Codable { +private struct UsageRecord: Codable, Equatable { var date: String var timestamp: String? + var timestampEpoch: TimeInterval? = nil var tool: String var model: String var usage: TokenUsageCounts @@ -2348,6 +4238,7 @@ private struct UsageRecord: Codable { enum CodingKeys: String, CodingKey { case date case timestamp + case timestampEpoch case tool case model case usage @@ -2366,6 +4257,7 @@ private struct UsageRecord: Codable { init( date: String, timestamp: String?, + timestampEpoch: TimeInterval? = nil, tool: String, model: String, usage: TokenUsageCounts, @@ -2382,6 +4274,7 @@ private struct UsageRecord: Codable { ) { self.date = date self.timestamp = timestamp + self.timestampEpoch = timestampEpoch self.tool = tool self.model = model self.usage = usage @@ -2401,6 +4294,7 @@ private struct UsageRecord: Codable { let container = try decoder.container(keyedBy: CodingKeys.self) date = try container.decode(String.self, forKey: .date) timestamp = try container.decodeIfPresent(String.self, forKey: .timestamp) + timestampEpoch = try container.decodeIfPresent(TimeInterval.self, forKey: .timestampEpoch) tool = try container.decode(String.self, forKey: .tool) model = try container.decode(String.self, forKey: .model) usage = try container.decode(TokenUsageCounts.self, forKey: .usage) @@ -2417,7 +4311,7 @@ private struct UsageRecord: Codable { } } -private enum UsageRecordSource: String, Codable { +private enum UsageRecordSource: String, Codable, Equatable { case nativeCodex case nativeCodexSQLite case nativeClaudeCode diff --git a/TokenStepSwift/Sources/TokenStepSwift/Stores/AppState.swift b/TokenStepSwift/Sources/TokenStepSwift/Stores/AppState.swift index d917c1f..320b64d 100644 --- a/TokenStepSwift/Sources/TokenStepSwift/Stores/AppState.swift +++ b/TokenStepSwift/Sources/TokenStepSwift/Stores/AppState.swift @@ -26,7 +26,14 @@ final class AppState: ObservableObject { @Published var lastError: String? private var timer: Timer? + private var foregroundTimer: Timer? + private var foregroundRefreshSurfaces = Set() private var pendingRefreshAfterCurrent = false + private var pendingForcedRefresh = false + private var lastQuotaRefreshAttemptAt: Date? + private var lastRankRefreshAttemptAt: Date? + private var lastAutomaticUsageRefreshAttemptAt: Date? + private var lastUsageObservedAt: Date? init() { load() @@ -40,6 +47,7 @@ final class AppState: ObservableObject { deinit { timer?.invalidate() + foregroundTimer?.invalidate() } var today: DailyUsage { @@ -148,34 +156,88 @@ final class AppState: ObservableObject { autostartEnabled = AutostartService.isEnabled } - func refresh() { + func refresh(forceCollection: Bool = true) { guard !isRefreshing else { - pendingRefreshAfterCurrent = true + if forceCollection { + pendingRefreshAfterCurrent = true + pendingForcedRefresh = true + } + return + } + let refreshStartedAt = Date() + if !forceCollection, + EnergyRefreshPolicy.isFresh( + lastAttemptAt: lastAutomaticUsageRefreshAttemptAt, + ttl: EnergyRefreshPolicy.automaticRetryTTL( + requestedSeconds: settings.refreshIntervalSeconds + ), + now: refreshStartedAt + ) { return } + if !forceCollection { + lastAutomaticUsageRefreshAttemptAt = refreshStartedAt + } isRefreshing = true lastError = nil let historyDays = settings.historyDays Task { + var outcome: CollectionRunOutcome = .unchanged + var collectionSucceeded = false do { - try await Task.detached(priority: .utility) { - try DataService.runCollectorInHelper(historyDays: historyDays) + outcome = try await Task.detached(priority: .utility) { + try DataService.runCollectorInHelper( + historyDays: historyDays, + force: forceCollection + ) }.value + collectionSucceeded = true } catch { lastError = error.localizedDescription } - load() + if outcome != .unchanged { + load() + } + if collectionSucceeded, outcome != .updatedWhileSourcesChanged { + lastUsageObservedAt = Date() + } isRefreshing = false - refreshCodexQuota() - refreshTokenRank(force: true) if pendingRefreshAfterCurrent { + let force = pendingForcedRefresh pendingRefreshAfterCurrent = false - refresh() + pendingForcedRefresh = false + refresh(forceCollection: force) } } } - func refreshCodexQuota() { + func refreshForForeground(now: Date = Date()) { + let snapshotDate = UsageSnapshotRefreshPolicy.generatedDate(snapshot.generatedAt) + let freshestObservation = [snapshotDate, lastUsageObservedAt] + .compactMap { $0 } + .max() + if EnergyRefreshPolicy.shouldRefreshForForeground( + generatedAt: freshestObservation, + requestedSeconds: settings.refreshIntervalSeconds, + now: now + ) { + refresh(forceCollection: false) + } + refreshCodexQuota(now: now) + refreshTokenRank() + } + + func setForegroundRefreshSurface(_ identifier: String, visible: Bool) { + if visible { + foregroundRefreshSurfaces.insert(identifier) + refreshForForeground() + } else { + foregroundRefreshSurfaces.remove(identifier) + } + configureForegroundTimer() + } + + func refreshCodexQuota(force: Bool = false, now: Date = Date()) { guard settings.showCodexQuota else { codexQuota = .unavailable claudeQuota = .unavailable @@ -183,6 +245,15 @@ final class AppState: ObservableObject { return } guard !isRefreshingCodexQuota else { return } + if !force, + EnergyRefreshPolicy.isFresh( + lastAttemptAt: lastQuotaRefreshAttemptAt, + ttl: EnergyRefreshPolicy.quotaTTL, + now: now + ) { + return + } + lastQuotaRefreshAttemptAt = now isRefreshingCodexQuota = true Task { let quotas = await Task.detached(priority: .utility) { @@ -268,6 +339,7 @@ final class AppState: ObservableObject { settings.refreshIntervalSeconds = seconds saveSettingsAndReload() configureTimer() + configureForegroundTimer() } func setTheme(_ theme: TokenStepTheme) { @@ -298,7 +370,7 @@ final class AppState: ObservableObject { settings.showCodexQuota = visible saveSettingsAndReload() if visible { - refreshCodexQuota() + refreshCodexQuota(force: true) } else { codexQuota = .unavailable claudeQuota = .unavailable @@ -322,7 +394,7 @@ final class AppState: ObservableObject { refresh() } - func refreshTokenRank(force: Bool = false) { + func refreshTokenRank(force: Bool = false, now: Date = Date()) { guard settings.agentWorkRankVisibility.readsLocalIdentity else { clearTokenRankState() return @@ -333,11 +405,20 @@ final class AppState: ObservableObject { return } guard !isRefreshingTokenRank else { return } - if !force, - let fetchedAt = tokenRank?.fetchedAt, - Date().timeIntervalSince(fetchedAt) < AgentWorkRankService.cacheTTL { - return + if !force { + if EnergyRefreshPolicy.isFresh( + lastAttemptAt: lastRankRefreshAttemptAt, + ttl: EnergyRefreshPolicy.rankTTL, + now: now + ) { + return + } + if let fetchedAt = tokenRank?.fetchedAt, + now.timeIntervalSince(fetchedAt) < AgentWorkRankService.cacheTTL { + return + } } + lastRankRefreshAttemptAt = now agentWorkRankIdentity = AgentWorkRankService.loadLocalIdentity() isRefreshingTokenRank = true @@ -496,13 +577,46 @@ final class AppState: ObservableObject { private func configureTimer() { timer?.invalidate() timer = nil - guard settings.refreshIntervalSeconds > 0 else { return } - timer = Timer.scheduledTimer(withTimeInterval: TimeInterval(settings.refreshIntervalSeconds), repeats: true) { [weak self] _ in + guard let interval = EnergyRefreshPolicy.backgroundInterval( + requestedSeconds: settings.refreshIntervalSeconds, + powerSource: TokenStepPowerState.source, + lowPowerMode: TokenStepPowerState.lowPowerModeEnabled + ) else { + return + } + timer = Timer.scheduledTimer(withTimeInterval: TimeInterval(interval), repeats: false) { [weak self] _ in Task { @MainActor in - self?.refresh() + guard let self else { return } + self.refresh(forceCollection: false) + self.refreshCodexQuota() + self.refreshTokenRank() + self.configureTimer() } } - timer?.tolerance = min(TimeInterval(settings.refreshIntervalSeconds) * 0.1, 30) + timer?.tolerance = min(TimeInterval(interval) * 0.1, 60) + } + + private func configureForegroundTimer() { + foregroundTimer?.invalidate() + foregroundTimer = nil + guard !foregroundRefreshSurfaces.isEmpty, + let interval = EnergyRefreshPolicy.foregroundTickInterval( + requestedSeconds: settings.refreshIntervalSeconds + ) + else { + return + } + foregroundTimer = Timer.scheduledTimer( + withTimeInterval: TimeInterval(interval), + repeats: false + ) { [weak self] _ in + Task { @MainActor in + guard let self else { return } + self.refreshForForeground() + self.configureForegroundTimer() + } + } + foregroundTimer?.tolerance = min(TimeInterval(interval) * 0.1, 10) } private func refreshIfSnapshotIsStale() { @@ -522,7 +636,7 @@ final class AppState: ObservableObject { + "\(UsageCollector.codexAccountingRevision); starting immediate recalibration." ) } - refresh() + refresh(forceCollection: reason != .stale) } private func scheduleDeferredUpdateCheck() { @@ -604,8 +718,7 @@ enum UsageSnapshotRefreshPolicy { guard refreshIntervalSeconds > 0 else { return snapshot.generatedAt == nil ? .missingSnapshotTimestamp : nil } - guard let generatedAt = snapshot.generatedAt, - let generatedDate = parseGeneratedAt(generatedAt) + guard let generatedDate = generatedDate(snapshot.generatedAt) else { return .missingSnapshotTimestamp } @@ -615,7 +728,8 @@ enum UsageSnapshotRefreshPolicy { return nil } - private static func parseGeneratedAt(_ value: String) -> Date? { + static func generatedDate(_ value: String?) -> Date? { + guard let value else { return nil } if let date = generatedAtISOWithFractional.date(from: value) { return date } diff --git a/TokenStepSwift/Sources/TokenStepSwift/Support/AppPaths.swift b/TokenStepSwift/Sources/TokenStepSwift/Support/AppPaths.swift index 8b327fd..a87aef5 100644 --- a/TokenStepSwift/Sources/TokenStepSwift/Support/AppPaths.swift +++ b/TokenStepSwift/Sources/TokenStepSwift/Support/AppPaths.swift @@ -15,6 +15,8 @@ enum AppPaths { static let usageJSON = appSupportRoot.appendingPathComponent("data/usage.json") static let collectorCacheJSON = appSupportRoot.appendingPathComponent("cache/collector-cache.json") + static let collectionCheckpointJSON = appSupportRoot.appendingPathComponent("cache/collection-checkpoint.json") + static let codexIncrementalCacheSQLite = appSupportRoot.appendingPathComponent("cache/codex-incremental.sqlite3") static let claudeQuotaCacheJSON = appSupportRoot.appendingPathComponent("cache/claude-quota-cache.json") static let settingsJSON = appSupportRoot.appendingPathComponent("config/settings.json") static let autostartDefaultMarker = appSupportRoot.appendingPathComponent("config/autostart-default-applied") diff --git a/TokenStepSwift/Sources/TokenStepSwift/Support/EnergyRefreshPolicy.swift b/TokenStepSwift/Sources/TokenStepSwift/Support/EnergyRefreshPolicy.swift new file mode 100644 index 0000000..4f973c4 --- /dev/null +++ b/TokenStepSwift/Sources/TokenStepSwift/Support/EnergyRefreshPolicy.swift @@ -0,0 +1,67 @@ +import Foundation +import IOKit.ps + +enum TokenStepPowerSource: Equatable { + case ac + case battery +} + +enum EnergyRefreshPolicy { + static let acBackgroundFloorSeconds = 15 * 60 + static let batteryBackgroundFloorSeconds = 30 * 60 + static let quotaTTL: TimeInterval = 15 * 60 + static let rankTTL: TimeInterval = 30 * 60 + static let minimumAutomaticRetryTTL: TimeInterval = 60 + static let maximumForegroundTickSeconds = 60 + + static func backgroundInterval( + requestedSeconds: Int, + powerSource: TokenStepPowerSource, + lowPowerMode: Bool + ) -> Int? { + guard requestedSeconds > 0 else { return nil } + let floor = powerSource == .battery || lowPowerMode + ? batteryBackgroundFloorSeconds + : acBackgroundFloorSeconds + return max(requestedSeconds, floor) + } + + static func shouldRefreshForForeground( + generatedAt: Date?, + requestedSeconds: Int, + now: Date + ) -> Bool { + guard requestedSeconds > 0 else { return false } + guard let generatedAt else { return true } + return now.timeIntervalSince(generatedAt) >= TimeInterval(requestedSeconds) + } + + static func isFresh(lastAttemptAt: Date?, ttl: TimeInterval, now: Date) -> Bool { + guard let lastAttemptAt else { return false } + return now.timeIntervalSince(lastAttemptAt) < ttl + } + + static func automaticRetryTTL(requestedSeconds: Int) -> TimeInterval { + max(minimumAutomaticRetryTTL, TimeInterval(max(0, requestedSeconds))) + } + + static func foregroundTickInterval(requestedSeconds: Int) -> Int? { + guard requestedSeconds > 0 else { return nil } + return min(requestedSeconds, maximumForegroundTickSeconds) + } +} + +enum TokenStepPowerState { + static var source: TokenStepPowerSource { + guard let info = IOPSCopyPowerSourcesInfo()?.takeRetainedValue(), + let raw = IOPSGetProvidingPowerSourceType(info)?.takeUnretainedValue() as String? + else { + return .ac + } + return raw == kIOPSBatteryPowerValue ? .battery : .ac + } + + static var lowPowerModeEnabled: Bool { + ProcessInfo.processInfo.isLowPowerModeEnabled + } +} diff --git a/TokenStepSwift/Sources/TokenStepSwift/Support/Localization.swift b/TokenStepSwift/Sources/TokenStepSwift/Support/Localization.swift index 8bc76ce..5153b8a 100644 --- a/TokenStepSwift/Sources/TokenStepSwift/Support/Localization.swift +++ b/TokenStepSwift/Sources/TokenStepSwift/Support/Localization.swift @@ -216,7 +216,7 @@ enum TokenStepLocalization { "token / 天": "tokens / day", "自动刷新": "Auto Refresh", "显示入口": "Display Entry", - "菜单栏、弹层和仪表盘会按这个频率同步更新。": "The menu bar, popover, and dashboard sync at this rhythm.", + "面板可见时按此频率检查；后台会根据供电状态降低频率。": "Visible panels check at this rhythm; background refresh slows down based on power state.", "手动更新": "Manual refresh", "每 %@": "Every %@", "当前节奏": "Current rhythm", @@ -583,7 +583,7 @@ enum TokenStepLocalization { "token / 天": "token / 天", "自动刷新": "自動重新整理", "显示入口": "顯示入口", - "菜单栏、弹层和仪表盘会按这个频率同步更新。": "選單列、浮層和儀表板會按這個頻率同步更新。", + "面板可见时按此频率检查；后台会根据供电状态降低频率。": "面板可見時按此頻率檢查；背景會依供電狀態降低頻率。", "手动更新": "手動更新", "每 %@": "每 %@", "当前节奏": "目前節奏", diff --git a/TokenStepSwift/Sources/TokenStepSwift/Support/MainWindowPresenter.swift b/TokenStepSwift/Sources/TokenStepSwift/Support/MainWindowPresenter.swift index 3ea0ce5..ca29dc2 100644 --- a/TokenStepSwift/Sources/TokenStepSwift/Support/MainWindowPresenter.swift +++ b/TokenStepSwift/Sources/TokenStepSwift/Support/MainWindowPresenter.swift @@ -14,13 +14,15 @@ final class MainWindowNavigation: ObservableObject { } @MainActor -final class MainWindowPresenter { +final class MainWindowPresenter: NSObject, NSWindowDelegate { static let shared = MainWindowPresenter() private var window: NSWindow? + private weak var appState: AppState? private let navigation = MainWindowNavigation() func show(appState: AppState, section: AppSection? = nil) { + self.appState = appState if let section { navigation.select(section) } @@ -31,6 +33,7 @@ final class MainWindowPresenter { closeTransientPanels(except: window) window.makeKeyAndOrderFront(nil) window.orderFrontRegardless() + appState.setForegroundRefreshSurface("main-window", visible: true) } private func makeWindow(appState: AppState) -> NSWindow { @@ -46,6 +49,7 @@ final class MainWindowPresenter { window.titlebarSeparatorStyle = .none window.toolbarStyle = .unifiedCompact window.isReleasedWhenClosed = false + window.delegate = self window.minSize = NSSize(width: 1080, height: 720) window.maxSize = NSSize(width: 1440, height: 980) window.setContentSize(NSSize(width: 1240, height: 820)) @@ -54,6 +58,18 @@ final class MainWindowPresenter { return window } + func windowWillClose(_ notification: Notification) { + appState?.setForegroundRefreshSurface("main-window", visible: false) + } + + func windowDidMiniaturize(_ notification: Notification) { + appState?.setForegroundRefreshSurface("main-window", visible: false) + } + + func windowDidDeminiaturize(_ notification: Notification) { + appState?.setForegroundRefreshSurface("main-window", visible: true) + } + private func closeTransientPanels(except mainWindow: NSWindow) { for window in NSApp.windows where window !== mainWindow && window.title.isEmpty { window.close() diff --git a/TokenStepSwift/Sources/TokenStepSwift/Support/TokenIslandWindowPresenter.swift b/TokenStepSwift/Sources/TokenStepSwift/Support/TokenIslandWindowPresenter.swift index da57e9c..13e67d4 100644 --- a/TokenStepSwift/Sources/TokenStepSwift/Support/TokenIslandWindowPresenter.swift +++ b/TokenStepSwift/Sources/TokenStepSwift/Support/TokenIslandWindowPresenter.swift @@ -54,6 +54,7 @@ final class TokenIslandWindowPresenter { appState.shouldShowTokenIsland, let screen = TokenIslandDisplayDetector.notchedPrimaryScreen else { return } + appState.setForegroundRefreshSurface("token-island", visible: true) let panel = popoverPanel ?? makePopoverPanel(appState: appState) popoverPanel = panel @@ -93,6 +94,7 @@ final class TokenIslandWindowPresenter { hidePopoverTask?.cancel() hidePopoverTask = nil popoverVisible = false + appState?.setForegroundRefreshSurface("token-island", visible: false) guard let panel = popoverPanel else { return } NSAnimationContext.runAnimationGroup { context in context.duration = 0.10 @@ -115,6 +117,7 @@ final class TokenIslandWindowPresenter { ringPanel?.orderOut(nil) popoverPanel?.orderOut(nil) popoverVisible = false + appState.setForegroundRefreshSurface("token-island", visible: false) return } diff --git a/TokenStepSwift/Sources/TokenStepSwift/Views/MainWindowView.swift b/TokenStepSwift/Sources/TokenStepSwift/Views/MainWindowView.swift index 92d9c69..c9ff33f 100644 --- a/TokenStepSwift/Sources/TokenStepSwift/Views/MainWindowView.swift +++ b/TokenStepSwift/Sources/TokenStepSwift/Views/MainWindowView.swift @@ -70,6 +70,9 @@ struct MainWindowView: View { .id(appState.appearanceID) } .background(TokenStepBackdrop().id(appState.appearanceID)) + .onAppear { + appState.refreshForForeground() + } .toolbar { ToolbarItem(placement: .primaryAction) { Button { diff --git a/TokenStepSwift/Sources/TokenStepSwift/Views/PopoverPanelView.swift b/TokenStepSwift/Sources/TokenStepSwift/Views/PopoverPanelView.swift index b90c5e9..0f228a7 100644 --- a/TokenStepSwift/Sources/TokenStepSwift/Views/PopoverPanelView.swift +++ b/TokenStepSwift/Sources/TokenStepSwift/Views/PopoverPanelView.swift @@ -37,6 +37,11 @@ struct PopoverPanelView: View { .frame(width: 412) .background(TokenStepBackdrop()) .id(appState.appearanceID) + .onAppear { + if !isScreenshotRendering { + appState.refreshForForeground() + } + } } private var header: some View { diff --git a/TokenStepSwift/Sources/TokenStepSwift/Views/Settings/SettingsDisplayRefreshCards.swift b/TokenStepSwift/Sources/TokenStepSwift/Views/Settings/SettingsDisplayRefreshCards.swift index 4128470..29c7e3a 100644 --- a/TokenStepSwift/Sources/TokenStepSwift/Views/Settings/SettingsDisplayRefreshCards.swift +++ b/TokenStepSwift/Sources/TokenStepSwift/Views/Settings/SettingsDisplayRefreshCards.swift @@ -52,7 +52,7 @@ struct SettingsRefreshCard: View { var body: some View { SettingsCard(title: L("自动刷新"), symbol: "arrow.triangle.2.circlepath.circle.fill") { VStack(alignment: .leading, spacing: 18) { - Text(L("菜单栏、弹层和仪表盘会按这个频率同步更新。")) + Text(L("面板可见时按此频率检查；后台会根据供电状态降低频率。")) .font(.callout.weight(.semibold)) .foregroundStyle(.secondary) diff --git a/TokenStepSwift/Tests/Fixtures/CCSwitchProxyFixtureCheck.swift b/TokenStepSwift/Tests/Fixtures/CCSwitchProxyFixtureCheck.swift index 4c588ab..f2153f9 100644 --- a/TokenStepSwift/Tests/Fixtures/CCSwitchProxyFixtureCheck.swift +++ b/TokenStepSwift/Tests/Fixtures/CCSwitchProxyFixtureCheck.swift @@ -734,6 +734,25 @@ struct CCSwitchProxyFixtureCheck { try assertEqual(source?.dedupedRecords, 1, "shared-session proxy record is deduplicated") try assertEqual(snapshot.totals.tokens, 35, "shared-session request counts once") try assertEqual(snapshot.totals.cost, 0.45, "shared-session proxy cost enriches native record") + + let incremental = UsageCollector.collectIncrementalCodexAndProxySnapshotForTests( + codexRoots: [root], + cacheURL: root.appendingPathComponent("incremental-cache.sqlite3"), + ccSwitchDatabaseURL: database + ) + let incrementalSource = incremental.sources["CC Switch Proxy"] + try assertEqual( + incrementalSource?.status, + "all_deduped", + "incremental shared-session Codex source status" + ) + try assertEqual( + incrementalSource?.dedupedRecords, + 1, + "incremental cache retains request-level details for proxy dedupe" + ) + try assertEqual(incremental.totals.tokens, 35, "incremental shared-session request counts once") + try assertEqual(incremental.totals.cost, 0.45, "incremental proxy cost enriches native record") } private static func codexLines(sessionID: String, totalTokens: Int) -> [String] { diff --git a/TokenStepSwift/Tests/Fixtures/CodexCumulativeFixtureCheck.swift b/TokenStepSwift/Tests/Fixtures/CodexCumulativeFixtureCheck.swift index 844437c..994ba24 100644 --- a/TokenStepSwift/Tests/Fixtures/CodexCumulativeFixtureCheck.swift +++ b/TokenStepSwift/Tests/Fixtures/CodexCumulativeFixtureCheck.swift @@ -15,7 +15,13 @@ struct CodexCumulativeFixtureCheck { try checkParallelChildren() try checkNestedChild() try checkRescanAppendAndRebuildStability() + try checkFirstFullValidationDoesNotTrustMissingHash() + try checkFullValidationDetectsMiddleRewrite() + try checkFullValidationRescansRewriteThenAppend() try checkParentAnchorCacheDependency() + try checkLateSessionMetadataForcesFullRescan() + try checkCorruptIncrementalCacheSelfHeals() + try checkCorruptIncrementalPayloadSelfHeals() try checkLegacyCacheRevisionTriggersRecalibration() try checkReasoningAndCachedSubsetsAreNotDoublePriced() try checkShanghaiMidnightBoundary() @@ -347,7 +353,7 @@ struct CodexCumulativeFixtureCheck { private static func checkRescanAppendAndRebuildStability() throws { try withFixtureHome("stable-rescan") { home in - let cacheURL = home.appendingPathComponent("fixture-cache/collector-cache-v8.json") + let cacheURL = home.appendingPathComponent("fixture-cache/codex-incremental.sqlite3") let initialLines = [ sessionMeta(id: "stable-session", timestamp: "2026-07-13T11:00:00Z"), turnContext(model: "gpt-5", timestamp: "2026-07-13T11:00:01Z"), @@ -357,13 +363,11 @@ struct CodexCumulativeFixtureCheck { let log = try writeSession(home: home, filename: "stable.jsonl", lines: initialLines) let first = UsageCollector.collectCodexUsageSnapshotForTests(homeURL: home, cacheURL: cacheURL) - let firstCacheData = try Data(contentsOf: cacheURL) + let firstStats = try requireCacheStats(cacheURL) let repeated = UsageCollector.collectCodexUsageSnapshotForTests(homeURL: home, cacheURL: cacheURL) - let repeatedCacheData = try Data(contentsOf: cacheURL) try expectEqual(first.totals.tokens, 160, "initial scan total") try expectEqual(snapshotSignature(repeated), snapshotSignature(first), "unchanged repeated scan is deterministic") - try expectEqual(repeatedCacheData, firstCacheData, "unchanged cache hit is byte-stable") - try expectEqual(try cacheVersion(at: cacheURL), 8, "fixture uses the v8 cache schema") + try expectEqual(try requireCacheStats(cacheURL), firstStats, "unchanged cache hit performs no writes") let originalMetadata = try FileManager.default.attributesOfItem(atPath: log.path) let originalModificationDate = originalMetadata[.modificationDate] as? Date @@ -393,7 +397,8 @@ struct CodexCumulativeFixtureCheck { } let afterSameMetadataRewrite = UsageCollector.collectCodexUsageSnapshotForTests( homeURL: home, - cacheURL: cacheURL + cacheURL: cacheURL, + forceFullValidation: true ) try expectEqual( afterSameMetadataRewrite.totals.tokens, @@ -410,8 +415,11 @@ struct CodexCumulativeFixtureCheck { let afterAppend = UsageCollector.collectCodexUsageSnapshotForTests(homeURL: home, cacheURL: cacheURL) try expectEqual(afterAppend.totals.tokens, 230, "append advances only to final cumulative total") - try FileManager.default.removeItem(at: cacheURL) - let afterCacheRebuild = UsageCollector.collectCodexUsageSnapshotForTests(homeURL: home, cacheURL: cacheURL) + let rebuiltCacheURL = home.appendingPathComponent("fixture-cache/codex-rebuilt.sqlite3") + let afterCacheRebuild = UsageCollector.collectCodexUsageSnapshotForTests( + homeURL: home, + cacheURL: rebuiltCacheURL + ) try expectEqual( snapshotSignature(afterCacheRebuild), snapshotSignature(afterAppend), @@ -432,12 +440,13 @@ struct CodexCumulativeFixtureCheck { private static func checkParentAnchorCacheDependency() throws { try withFixtureHome("parent-cache-dependency") { home in - let cacheURL = home.appendingPathComponent("fixture-cache/collector-cache-v8.json") + let cacheURL = home.appendingPathComponent("fixture-cache/codex-incremental.sqlite3") let parentLines = [ sessionMeta(id: "cache-parent", timestamp: "2026-07-13T00:00:00Z"), turnContext(model: "gpt-5", timestamp: "2026-07-13T00:00:01Z"), tokenCount(timestamp: "2026-07-13T00:01:00Z", cumulative: vector(total: 100), last: vector(total: 100)), - tokenCount(timestamp: "2026-07-13T00:02:00Z", cumulative: vector(total: 200), last: vector(total: 100)) + tokenCount(timestamp: "2026-07-13T00:02:00Z", cumulative: vector(total: 200), last: vector(total: 100)), + String(repeating: "ignored-padding-", count: 512) ] let parentLog = try writeSession(home: home, filename: "99-parent.jsonl", lines: parentLines) try writeSession( @@ -476,6 +485,321 @@ struct CodexCumulativeFixtureCheck { } } + private static func checkFullValidationDetectsMiddleRewrite() throws { + try withFixtureHome("full-validation-middle-rewrite") { home in + let cacheURL = home.appendingPathComponent("fixture-cache/codex-incremental.sqlite3") + let initialLines = [ + String(repeating: "leading-padding-", count: 400), + sessionMeta(id: "middle-rewrite", timestamp: "2026-07-13T11:30:00Z"), + turnContext(model: "gpt-5", timestamp: "2026-07-13T11:30:01Z"), + tokenCount(timestamp: "2026-07-13T11:31:00Z", cumulative: vector(total: 100), last: vector(total: 100)), + tokenCount(timestamp: "2026-07-13T11:32:00Z", cumulative: vector(total: 160), last: vector(total: 60)), + String(repeating: "trailing-padding-", count: 400) + ] + let log = try writeSession(home: home, filename: "middle.jsonl", lines: initialLines) + let initial = UsageCollector.collectCodexUsageSnapshotForTests( + homeURL: home, + cacheURL: cacheURL + ) + let initialStats = try requireCacheStats(cacheURL) + try expectEqual(initial.totals.tokens, 160, "middle rewrite baseline") + + let validated = UsageCollector.collectCodexUsageSnapshotForTests( + homeURL: home, + cacheURL: cacheURL, + forceFullValidation: true + ) + try expectEqual(validated.totals.tokens, 160, "full validation baseline") + try expectEqual( + try requireCacheStats(cacheURL).generation, + initialStats.generation, + "recording a strong validation hash does not rewrite accounting payloads" + ) + + let originalAttributes = try FileManager.default.attributesOfItem(atPath: log.path) + let originalModificationDate = originalAttributes[.modificationDate] as? Date + var rewrittenLines = initialLines + rewrittenLines[4] = tokenCount( + timestamp: "2026-07-13T11:32:00Z", + cumulative: vector(total: 190), + last: vector(total: 90) + ) + let originalSize = try Data(contentsOf: log).count + try (rewrittenLines.joined(separator: "\n") + "\n").write( + to: log, + atomically: true, + encoding: .utf8 + ) + try expectEqual(try Data(contentsOf: log).count, originalSize, "middle rewrite size") + if let originalModificationDate { + try FileManager.default.setAttributes( + [.modificationDate: originalModificationDate], + ofItemAtPath: log.path + ) + } + + let rewritten = UsageCollector.collectCodexUsageSnapshotForTests( + homeURL: home, + cacheURL: cacheURL, + forceFullValidation: true + ) + try expectEqual( + rewritten.totals.tokens, + 190, + "strong validation detects a same-size same-mtime middle rewrite" + ) + } + } + + private static func checkFirstFullValidationDoesNotTrustMissingHash() throws { + try withFixtureHome("first-validation-middle-rewrite") { home in + let cacheURL = home.appendingPathComponent("fixture-cache/codex-incremental.sqlite3") + let initialLines = [ + String(repeating: "leading-padding-", count: 400), + sessionMeta(id: "first-validation", timestamp: "2026-07-13T11:30:00Z"), + turnContext(model: "gpt-5", timestamp: "2026-07-13T11:30:01Z"), + tokenCount(timestamp: "2026-07-13T11:31:00Z", cumulative: vector(total: 100), last: vector(total: 100)), + tokenCount(timestamp: "2026-07-13T11:32:00Z", cumulative: vector(total: 160), last: vector(total: 60)), + String(repeating: "trailing-padding-", count: 400) + ] + let log = try writeSession(home: home, filename: "first-validation.jsonl", lines: initialLines) + let initial = UsageCollector.collectCodexUsageSnapshotForTests( + homeURL: home, + cacheURL: cacheURL + ) + try expectEqual(initial.totals.tokens, 160, "first validation baseline") + + let originalAttributes = try FileManager.default.attributesOfItem(atPath: log.path) + let originalModificationDate = originalAttributes[.modificationDate] as? Date + let originalSize = try Data(contentsOf: log).count + var rewrittenLines = initialLines + rewrittenLines[4] = tokenCount( + timestamp: "2026-07-13T11:32:00Z", + cumulative: vector(total: 190), + last: vector(total: 90) + ) + try (rewrittenLines.joined(separator: "\n") + "\n").write( + to: log, + atomically: true, + encoding: .utf8 + ) + try expectEqual(try Data(contentsOf: log).count, originalSize, "first validation rewrite size") + if let originalModificationDate { + try FileManager.default.setAttributes( + [.modificationDate: originalModificationDate], + ofItemAtPath: log.path + ) + } + + let validated = UsageCollector.collectCodexUsageSnapshotForTests( + homeURL: home, + cacheURL: cacheURL, + forceFullValidation: true + ) + try expectEqual( + validated.totals.tokens, + 190, + "first validation rescans instead of trusting a missing strong-hash baseline" + ) + } + } + + private static func checkFullValidationRescansRewriteThenAppend() throws { + try withFixtureHome("full-validation-rewrite-append") { home in + let cacheURL = home.appendingPathComponent("fixture-cache/codex-incremental.sqlite3") + let freshCacheURL = home.appendingPathComponent("fresh-cache/codex-incremental.sqlite3") + let initialLines = [ + String(repeating: "leading-padding-", count: 400), + sessionMeta(id: "rewrite-append", timestamp: "2026-07-13T11:30:00Z"), + turnContext(model: "gpt-5", timestamp: "2026-07-13T11:30:01Z"), + tokenCount(timestamp: "2026-07-13T11:31:00Z", cumulative: vector(total: 100), last: vector(total: 100)), + tokenCount(timestamp: "2026-07-13T11:32:00Z", cumulative: vector(total: 160), last: vector(total: 60)), + String(repeating: "trailing-padding-", count: 400) + ] + let log = try writeSession(home: home, filename: "rewrite-append.jsonl", lines: initialLines) + _ = UsageCollector.collectCodexUsageSnapshotForTests(homeURL: home, cacheURL: cacheURL) + _ = UsageCollector.collectCodexUsageSnapshotForTests( + homeURL: home, + cacheURL: cacheURL, + forceFullValidation: true + ) + + let originalAttributes = try FileManager.default.attributesOfItem(atPath: log.path) + let originalModificationDate = originalAttributes[.modificationDate] as? Date + let originalSize = try Data(contentsOf: log).count + var rewrittenLines = initialLines + rewrittenLines[4] = tokenCount( + timestamp: "2026-07-13T11:32:00Z", + cumulative: vector(total: 190), + last: vector(total: 90) + ) + try (rewrittenLines.joined(separator: "\n") + "\n").write( + to: log, + atomically: true, + encoding: .utf8 + ) + try expectEqual(try Data(contentsOf: log).count, originalSize, "rewrite-append prefix size") + if let originalModificationDate { + try FileManager.default.setAttributes( + [.modificationDate: originalModificationDate], + ofItemAtPath: log.path + ) + } + try appendLine( + turnContext(model: "gpt-5", timestamp: "2026-07-14T00:00:00Z"), + to: log + ) + try appendLine( + tokenCount( + timestamp: "2026-07-14T00:01:00Z", + cumulative: vector(total: 230), + last: vector(total: 40) + ), + to: log + ) + + let validated = UsageCollector.collectCodexUsageSnapshotForTests( + homeURL: home, + cacheURL: cacheURL, + forceFullValidation: true + ) + try expectEqual(dailyTokens(validated, date: "2026-07-13"), 190, "rewrite-append corrected prior day") + try expectEqual(dailyTokens(validated, date: "2026-07-14"), 40, "rewrite-append keeps only the new-day delta") + + let rebuilt = UsageCollector.collectCodexUsageSnapshotForTests( + homeURL: home, + cacheURL: freshCacheURL + ) + try expectEqual( + snapshotSignature(validated), + snapshotSignature(rebuilt), + "forced rewrite-append validation matches a clean rebuild" + ) + } + } + + private static func checkCorruptIncrementalCacheSelfHeals() throws { + try withFixtureHome("corrupt-incremental-cache") { home in + let cacheURL = home.appendingPathComponent("fixture-cache/codex-incremental.sqlite3") + try FileManager.default.createDirectory( + at: cacheURL.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + try Data("not-a-sqlite-database".utf8).write(to: cacheURL) + try writeSession( + home: home, + filename: "recoverable.jsonl", + lines: [ + sessionMeta(id: "recoverable", timestamp: "2026-07-13T12:00:00Z"), + turnContext(model: "gpt-5", timestamp: "2026-07-13T12:00:01Z"), + tokenCount( + timestamp: "2026-07-13T12:01:00Z", + cumulative: vector(total: 140), + last: vector(total: 140) + ) + ] + ) + + let snapshot = UsageCollector.collectCodexWithIncrementalFallbackForTests( + homeURL: home, + cacheURL: cacheURL + ) + try expectEqual(snapshot.totals.tokens, 140, "corrupt cache rebuild preserves accounting") + try expectEqual(snapshot.sources["Codex"]?.status, "ok", "corrupt cache rebuild status") + let stats = try requireCacheStats(cacheURL) + try expectEqual(stats.sessions, 1, "corrupt cache is replaced with a valid incremental store") + } + } + + private static func checkLateSessionMetadataForcesFullRescan() throws { + try withFixtureHome("late-session-metadata") { home in + let cacheURL = home.appendingPathComponent("fixture-cache/codex-incremental.sqlite3") + try writeSimpleParent(home: home, filename: "99-parent.jsonl", id: "late-meta-parent") + let child = try writeSession( + home: home, + filename: "00-child.jsonl", + lines: ["{\"type\":\"event_msg\",\"payload\":{\"type\":\"noop\"}}"] + ) + + let beforeMetadata = UsageCollector.collectCodexUsageSnapshotForTests( + homeURL: home, + cacheURL: cacheURL + ) + try expectEqual(beforeMetadata.totals.tokens, 300, "leading child log is initially empty") + + for line in replayChildLines( + id: "late-meta-child", + parentID: "late-meta-parent", + includeSecondParentMetadata: false, + terminalTotals: [360] + ) { + try appendLine(line, to: child) + } + let afterMetadata = UsageCollector.collectCodexUsageSnapshotForTests( + homeURL: home, + cacheURL: cacheURL + ) + try expectEqual( + afterMetadata.totals.tokens, + 360, + "an appended first session_meta forces a full fork-aware rescan" + ) + try expectEqual( + afterMetadata.sources["Codex"]?.inheritedTokens, + 300, + "late metadata restores the parent fork anchor" + ) + } + } + + private static func checkCorruptIncrementalPayloadSelfHeals() throws { + try withFixtureHome("corrupt-incremental-payload") { home in + let cacheURL = home.appendingPathComponent("fixture-cache/codex-incremental.sqlite3") + try writeSession( + home: home, + filename: "payload.jsonl", + lines: [ + sessionMeta(id: "payload-session", timestamp: "2026-07-13T13:00:00Z"), + turnContext(model: "gpt-5", timestamp: "2026-07-13T13:00:01Z"), + tokenCount( + timestamp: "2026-07-13T13:01:00Z", + cumulative: vector(total: 175), + last: vector(total: 175) + ) + ] + ) + let initial = UsageCollector.collectCodexUsageSnapshotForTests( + homeURL: home, + cacheURL: cacheURL + ) + try expectEqual(initial.totals.tokens, 175, "payload corruption fixture baseline") + + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/sqlite3") + process.arguments = [ + cacheURL.path, + "UPDATE codex_sessions SET diagnostics = X'00'" + ] + process.standardOutput = Pipe() + process.standardError = Pipe() + try process.run() + process.waitUntilExit() + try expectEqual(process.terminationStatus, 0, "fixture corrupts one cache payload") + + let healed = UsageCollector.collectCodexWithIncrementalFallbackForTests( + homeURL: home, + cacheURL: cacheURL + ) + try expectEqual(healed.totals.tokens, 175, "payload corruption rebuild preserves accounting") + try expectEqual(healed.sources["Codex"]?.status, "ok", "payload corruption rebuild status") + try expectEqual( + try requireCacheStats(cacheURL).sessions, + 1, + "payload corruption is replaced instead of repeatedly falling back" + ) + } + } + private static func checkLegacyCacheRevisionTriggersRecalibration() throws { try withFixtureHome("legacy-cache-revision") { home in let cacheURL = home.appendingPathComponent("fixture-cache/collector-cache-v7.json") @@ -745,10 +1069,11 @@ struct CodexCumulativeFixtureCheck { snapshot.daily.first(where: { $0.date == date })?.totalTokens } - private static func cacheVersion(at url: URL) throws -> Int? { - let data = try Data(contentsOf: url) - let object = try JSONSerialization.jsonObject(with: data) as? [String: Any] - return object?["version"] as? Int + private static func requireCacheStats(_ url: URL) throws -> CodexIncrementalCacheStats { + guard let stats = UsageCollector.codexIncrementalCacheStatsForTests(databaseURL: url) else { + throw FixtureFailure("incremental cache stats are available") + } + return stats } private static func snapshotSignature(_ snapshot: UsageSnapshot) -> SnapshotSignature { diff --git a/TokenStepSwift/Tests/Fixtures/EnergyEfficiencyBenchmark.swift b/TokenStepSwift/Tests/Fixtures/EnergyEfficiencyBenchmark.swift new file mode 100644 index 0000000..bf1ef11 --- /dev/null +++ b/TokenStepSwift/Tests/Fixtures/EnergyEfficiencyBenchmark.swift @@ -0,0 +1,166 @@ +import Darwin +import Foundation + +@main +struct EnergyEfficiencyBenchmark { + static func main() { + do { + let arguments = CommandLine.arguments + guard arguments.count >= 3 else { + throw BenchmarkError.message("Usage: EnergyEfficiencyBenchmark ") + } + let database = URL(fileURLWithPath: arguments[1]) + let mode = arguments[2] + let home = ProcessInfo.processInfo.environment["TOKENSTEP_BENCHMARK_HOME"] + .map { URL(fileURLWithPath: $0, isDirectory: true) } + ?? FileManager.default.homeDirectoryForCurrentUser + let codexStateBefore = UsageCollector.codexCollectionStateForTests(homeURL: home) + + let stateStarted = ContinuousClock.now + let state = UsageCollector.collectionState( + historyDays: 180, + includeExperimentalAgentSources: true, + homeURL: home + ) + let stateElapsed = stateStarted.duration(to: .now) + + let collectStarted = ContinuousClock.now + var migrationComparison: CodexAccountingComparisonDiagnostics? + let snapshot: UsageSnapshot + if mode == "migration-compare" { + let comparison = try UsageCollector.compareLegacyMigrationCodexAccountingForTests( + homeURL: home, + databaseURL: database + ) + migrationComparison = comparison + snapshot = comparison.incrementalSnapshot + } else { + snapshot = UsageCollector.collectCodexUsageSnapshotForTests( + homeURL: home, + cacheURL: database, + forceFullValidation: mode == "validate" + ) + } + let collectElapsed = collectStarted.duration(to: .now) + guard snapshot.sources["Codex"]?.status == "ok", + let stats = UsageCollector.codexIncrementalCacheStatsForTests(databaseURL: database) + else { + throw BenchmarkError.message("incremental collection failed") + } + + print("mode=\(mode)") + print("state_files=\(state.files.count)") + print("state_ms=\(milliseconds(stateElapsed))") + print("collect_ms=\(milliseconds(collectElapsed))") + print("cache_generation=\(stats.generation)") + print("cached_sessions=\(stats.sessions)") + print("cached_records=\(stats.records)") + print("last_logical_write_bytes=\(stats.lastLogicalWriteBytes)") + + if mode == "cache-compare" { + let detailed = UsageCollector.collectCodexUsageSnapshotForTests( + homeURL: home, + cacheURL: database, + requiresDetailedRecords: true + ) + let mismatches = try mismatchSections(snapshot, detailed) + print("mismatch_sections=\(mismatches.joined(separator: ","))") + guard mismatches.isEmpty else { + throw BenchmarkError.message("cached summary differs from cached detailed accounting") + } + print("cache_accounting_match=true") + } + + if mode == "database-compare" { + guard let referencePath = ProcessInfo.processInfo.environment[ + "TOKENSTEP_BENCHMARK_REFERENCE_DATABASE" + ] else { + throw BenchmarkError.message( + "TOKENSTEP_BENCHMARK_REFERENCE_DATABASE is required" + ) + } + let reference = UsageCollector.collectCodexUsageSnapshotForTests( + homeURL: home, + cacheURL: URL(fileURLWithPath: referencePath) + ) + let mismatches = try mismatchSections(snapshot, reference) + print("mismatch_sections=\(mismatches.joined(separator: ","))") + guard mismatches.isEmpty else { + throw BenchmarkError.message("incremental databases disagree") + } + print("database_accounting_match=true") + } + + if mode == "compare" || mode == "migration-compare" { + let referenceStarted = ContinuousClock.now + let comparison = try migrationComparison + ?? UsageCollector.compareIncrementalCodexAccountingForTests( + homeURL: home, + databaseURL: database + ) + let referenceElapsed = referenceStarted.duration(to: .now) + let codexStateAfterReference = UsageCollector.codexCollectionStateForTests( + homeURL: home + ) + guard codexStateBefore == codexStateAfterReference else { + throw BenchmarkError.message("Codex session files changed during accounting comparison") + } + let mismatches = try mismatchSections( + comparison.incrementalSnapshot, + comparison.referenceSnapshot + ) + print("mismatch_sections=\(mismatches.joined(separator: ","))") + print("mismatched_path_count=\(comparison.mismatchedPathHashes.count)") + print("mismatched_path_hashes=\(comparison.mismatchedPathHashes.prefix(12).joined(separator: ","))") + print("record_count_delta=\(comparison.incrementalRecordCount - comparison.referenceRecordCount)") + guard mismatches.isEmpty else { + throw BenchmarkError.message("incremental accounting differs from full rebuild") + } + print("reference_ms=\(milliseconds(referenceElapsed))") + print("accounting_match=true") + } + } catch { + fputs("Energy efficiency benchmark failed: \(error)\n", stderr) + exit(1) + } + } + + private static func milliseconds(_ duration: Duration) -> Int { + let components = duration.components + return Int(components.seconds * 1_000) + + Int(components.attoseconds / 1_000_000_000_000_000) + } + + private static func accountingSignature(_ snapshot: UsageSnapshot) throws -> Data { + var normalized = snapshot + normalized.generatedAt = nil + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + return try encoder.encode(normalized) + } + + private static func mismatchSections( + _ lhs: UsageSnapshot, + _ rhs: UsageSnapshot + ) throws -> [String] { + var mismatches = [String]() + if try canonical(lhs.totals) != canonical(rhs.totals) { mismatches.append("totals") } + if try canonical(lhs.daily) != canonical(rhs.daily) { mismatches.append("daily") } + if try canonical(lhs.rhythms) != canonical(rhs.rhythms) { mismatches.append("rhythms") } + if try canonical(lhs.agentWork) != canonical(rhs.agentWork) { mismatches.append("agent_work") } + if try canonical(lhs.tools) != canonical(rhs.tools) { mismatches.append("tools") } + if try canonical(lhs.models) != canonical(rhs.models) { mismatches.append("models") } + if try canonical(lhs.sources) != canonical(rhs.sources) { mismatches.append("sources") } + return mismatches + } + + private static func canonical(_ value: T) throws -> Data { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + return try encoder.encode(value) + } +} + +private enum BenchmarkError: Error { + case message(String) +} diff --git a/TokenStepSwift/Tests/Fixtures/UsageRecalibrationMigrationFixtureCheck.swift b/TokenStepSwift/Tests/Fixtures/UsageRecalibrationMigrationFixtureCheck.swift index 4a59d2f..f1f9e4b 100644 --- a/TokenStepSwift/Tests/Fixtures/UsageRecalibrationMigrationFixtureCheck.swift +++ b/TokenStepSwift/Tests/Fixtures/UsageRecalibrationMigrationFixtureCheck.swift @@ -7,6 +7,9 @@ struct UsageRecalibrationMigrationFixtureCheck { try? FileManager.default.removeItem(at: root) defer { try? FileManager.default.removeItem(at: root) } + try checkEnergyRefreshPolicy() + try checkCollectionCheckpointPolicy() + let currentRevision = UsageCollector.codexAccountingRevision let legacy = snapshot(accountingRevision: nil, records: 1) let previous = snapshot(accountingRevision: currentRevision - 1, records: 1) @@ -136,6 +139,164 @@ struct UsageRecalibrationMigrationFixtureCheck { print("PASS: usage recalibration migration marker and failure preservation") } + private static func checkCollectionCheckpointPolicy() throws { + let now = Date(timeIntervalSince1970: 1_786_080_000) + let state = UsageCollectionState( + historyDays: 30, + includesExperimentalAgentSources: false, + windowDay: "2026-08-07", + files: [ + UsageCollectionFileState( + path: "/tmp/session.jsonl", + size: 100, + modificationTime: 123 + ) + ] + ) + let fresh = CollectionCheckpoint( + verifiedAt: now.addingTimeInterval(-60), + state: state + ) + try expect( + CollectionCheckpointPolicy.shouldSkipCollection( + force: false, + hasSnapshot: true, + checkpoint: fresh, + state: state, + now: now + ), + "an unchanged fresh source state should skip collection" + ) + try expect( + !CollectionCheckpointPolicy.shouldSkipCollection( + force: true, + hasSnapshot: true, + checkpoint: fresh, + state: state, + now: now + ), + "manual refresh should bypass the checkpoint" + ) + + var changed = state + changed.files[0].size += 1 + try expect( + !CollectionCheckpointPolicy.shouldSkipCollection( + force: false, + hasSnapshot: true, + checkpoint: fresh, + state: changed, + now: now + ), + "a changed source file should invalidate the checkpoint" + ) + let expired = CollectionCheckpoint( + verifiedAt: now.addingTimeInterval(-CollectionCheckpoint.validationTTL), + state: state + ) + try expect( + !CollectionCheckpointPolicy.shouldSkipCollection( + force: false, + hasSnapshot: true, + checkpoint: expired, + state: state, + now: now + ), + "an expired checkpoint should trigger periodic validation" + ) + try expect( + !CollectionCheckpointPolicy.shouldPersist( + beforeCollection: state, + afterCollection: changed + ), + "a source race must not persist a stale checkpoint" + ) + var nextDay = state + nextDay.windowDay = "2026-08-08" + try expect( + !CollectionCheckpointPolicy.shouldSkipCollection( + force: false, + hasSnapshot: true, + checkpoint: fresh, + state: nextDay, + now: now + ), + "crossing Shanghai midnight should roll the history window" + ) + try expect( + CollectionCheckpointPolicy.shouldPersist( + beforeCollection: state, + afterCollection: state + ), + "a stable source scan should persist its checkpoint" + ) + } + + private static func checkEnergyRefreshPolicy() throws { + try expect( + EnergyRefreshPolicy.backgroundInterval( + requestedSeconds: 300, + powerSource: .ac, + lowPowerMode: false + ) == 900, + "AC background refresh should use a fifteen-minute floor" + ) + try expect( + EnergyRefreshPolicy.backgroundInterval( + requestedSeconds: 300, + powerSource: .battery, + lowPowerMode: false + ) == 1_800, + "battery background refresh should use a thirty-minute floor" + ) + try expect( + EnergyRefreshPolicy.backgroundInterval( + requestedSeconds: 300, + powerSource: .ac, + lowPowerMode: true + ) == 1_800, + "low-power mode should use the battery refresh floor" + ) + try expect( + EnergyRefreshPolicy.backgroundInterval( + requestedSeconds: 0, + powerSource: .battery, + lowPowerMode: true + ) == nil, + "manual mode should not schedule background work" + ) + try expect( + EnergyRefreshPolicy.automaticRetryTTL(requestedSeconds: 300) == 300, + "automatic usage failures should honor the requested retry interval" + ) + try expect( + EnergyRefreshPolicy.foregroundTickInterval(requestedSeconds: 300) == 60, + "a visible surface should cheaply recheck freshness once per minute" + ) + try expect( + EnergyRefreshPolicy.foregroundTickInterval(requestedSeconds: 0) == nil, + "manual mode should not run a visible-surface timer" + ) + + let now = Date(timeIntervalSince1970: 20_000) + try expect( + EnergyRefreshPolicy.shouldRefreshForForeground( + generatedAt: now.addingTimeInterval(-301), + requestedSeconds: 300, + now: now + ), + "foreground presentation should refresh stale usage" + ) + try expect( + !EnergyRefreshPolicy.shouldRefreshForForeground( + generatedAt: now.addingTimeInterval(-299), + requestedSeconds: 300, + now: now + ), + "foreground presentation should reuse fresh usage" + ) + } + private static func snapshot( accountingRevision: Int?, records: Int, diff --git a/TokenStepSwift/Tests/TokenStepSwiftTests/EnergyRefreshPolicyTests.swift b/TokenStepSwift/Tests/TokenStepSwiftTests/EnergyRefreshPolicyTests.swift new file mode 100644 index 0000000..5d5aceb --- /dev/null +++ b/TokenStepSwift/Tests/TokenStepSwiftTests/EnergyRefreshPolicyTests.swift @@ -0,0 +1,89 @@ +import Foundation +import XCTest +@testable import TokenStepSwift + +final class EnergyRefreshPolicyTests: XCTestCase { + func testBackgroundRefreshUsesFifteenMinuteFloorOnACPower() { + XCTAssertEqual( + EnergyRefreshPolicy.backgroundInterval( + requestedSeconds: 300, + powerSource: .ac, + lowPowerMode: false + ), + 900 + ) + } + + func testBackgroundRefreshUsesThirtyMinuteFloorOnBatteryOrLowPowerMode() { + XCTAssertEqual( + EnergyRefreshPolicy.backgroundInterval( + requestedSeconds: 300, + powerSource: .battery, + lowPowerMode: false + ), + 1_800 + ) + XCTAssertEqual( + EnergyRefreshPolicy.backgroundInterval( + requestedSeconds: 300, + powerSource: .ac, + lowPowerMode: true + ), + 1_800 + ) + } + + func testManualModeDoesNotScheduleBackgroundRefresh() { + XCTAssertNil( + EnergyRefreshPolicy.backgroundInterval( + requestedSeconds: 0, + powerSource: .battery, + lowPowerMode: true + ) + ) + } + + func testForegroundRefreshStillHonorsRequestedFreshness() { + let now = Date(timeIntervalSince1970: 10_000) + XCTAssertTrue( + EnergyRefreshPolicy.shouldRefreshForForeground( + generatedAt: now.addingTimeInterval(-301), + requestedSeconds: 300, + now: now + ) + ) + XCTAssertFalse( + EnergyRefreshPolicy.shouldRefreshForForeground( + generatedAt: now.addingTimeInterval(-299), + requestedSeconds: 300, + now: now + ) + ) + } + + func testIndependentTTLUsesLastAttemptRatherThanUsageRefresh() { + let now = Date(timeIntervalSince1970: 20_000) + XCTAssertTrue( + EnergyRefreshPolicy.isFresh( + lastAttemptAt: now.addingTimeInterval(-899), + ttl: 900, + now: now + ) + ) + XCTAssertFalse( + EnergyRefreshPolicy.isFresh( + lastAttemptAt: now.addingTimeInterval(-900), + ttl: 900, + now: now + ) + ) + } + + func testAutomaticFailureBackoffAndVisibleTickAreBounded() { + XCTAssertEqual(EnergyRefreshPolicy.automaticRetryTTL(requestedSeconds: 300), 300) + XCTAssertEqual(EnergyRefreshPolicy.automaticRetryTTL(requestedSeconds: 0), 60) + XCTAssertEqual(EnergyRefreshPolicy.foregroundTickInterval(requestedSeconds: 300), 60) + XCTAssertEqual(EnergyRefreshPolicy.foregroundTickInterval(requestedSeconds: 30), 30) + XCTAssertNil(EnergyRefreshPolicy.foregroundTickInterval(requestedSeconds: 0)) + } +} diff --git a/TokenStepSwift/Tests/TokenStepSwiftTests/UsageCollectorCodexTests.swift b/TokenStepSwift/Tests/TokenStepSwiftTests/UsageCollectorCodexTests.swift index bcfe051..b8c50fe 100644 --- a/TokenStepSwift/Tests/TokenStepSwiftTests/UsageCollectorCodexTests.swift +++ b/TokenStepSwift/Tests/TokenStepSwiftTests/UsageCollectorCodexTests.swift @@ -100,7 +100,7 @@ final class UsageCollectorCodexTests: XCTestCase { func testCodexCollectorCacheHitAppendAndRebuildRemainStable() throws { let home = try makeTemporaryHome("cache") let root = home.appendingPathComponent(".codex/sessions/2026/07/13", isDirectory: true) - let cache = home.appendingPathComponent("cache/collector-v8.json") + let cache = home.appendingPathComponent("cache/codex-incremental.sqlite3") try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) let file = root.appendingPathComponent("stable.jsonl") let initial = [ @@ -112,11 +112,18 @@ final class UsageCollectorCodexTests: XCTestCase { try writeCodexSession(initial, to: file) let first = UsageCollector.collectCodexUsageSnapshotForTests(homeURL: home, cacheURL: cache) - let firstCache = try Data(contentsOf: cache) + let firstStats = try XCTUnwrap( + UsageCollector.codexIncrementalCacheStatsForTests(databaseURL: cache) + ) let repeated = UsageCollector.collectCodexUsageSnapshotForTests(homeURL: home, cacheURL: cache) XCTAssertEqual(first.totals.tokens, 160) XCTAssertEqual(repeated.totals.tokens, first.totals.tokens) - XCTAssertEqual(try Data(contentsOf: cache), firstCache) + XCTAssertEqual( + UsageCollector.codexIncrementalCacheStatsForTests(databaseURL: cache), + firstStats + ) + XCTAssertEqual(firstStats.sessions, 1) + XCTAssertEqual(firstStats.records, 2) try writeCodexSession( initial + [ @@ -126,9 +133,18 @@ final class UsageCollectorCodexTests: XCTestCase { ) let appended = UsageCollector.collectCodexUsageSnapshotForTests(homeURL: home, cacheURL: cache) XCTAssertEqual(appended.totals.tokens, 230) - - try FileManager.default.removeItem(at: cache) - let rebuilt = UsageCollector.collectCodexUsageSnapshotForTests(homeURL: home, cacheURL: cache) + let appendedStats = try XCTUnwrap( + UsageCollector.codexIncrementalCacheStatsForTests(databaseURL: cache) + ) + XCTAssertEqual(appendedStats.generation, firstStats.generation + 1) + XCTAssertEqual(appendedStats.sessions, 1) + XCTAssertEqual(appendedStats.records, 3) + + let rebuiltCache = home.appendingPathComponent("cache/codex-rebuilt.sqlite3") + let rebuilt = UsageCollector.collectCodexUsageSnapshotForTests( + homeURL: home, + cacheURL: rebuiltCache + ) XCTAssertEqual(rebuilt.totals.tokens, appended.totals.tokens) XCTAssertEqual(rebuilt.sources["Codex"]?.tokenBreakdown, appended.sources["Codex"]?.tokenBreakdown) } diff --git a/script/benchmark_energy_efficiency.sh b/script/benchmark_energy_efficiency.sh new file mode 100755 index 0000000..f955ceb --- /dev/null +++ b/script/benchmark_energy_efficiency.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SWIFT_DIR="$ROOT_DIR/TokenStepSwift" +BUILD_DIR="${TMPDIR:-/tmp}/tokenstep-energy-benchmark-$UID" +OVERLAY_DIR="$BUILD_DIR/vfs-overlay" +OVERLAY_FILE="$OVERLAY_DIR/overlay.yaml" +EMPTY_MODULEMAP="$OVERLAY_DIR/empty.modulemap" +EXECUTABLE="$BUILD_DIR/energy-efficiency-benchmark" +DATABASE="${TOKENSTEP_BENCHMARK_DATABASE:-$BUILD_DIR/codex-incremental.sqlite3}" + +mkdir -p "$BUILD_DIR" "$OVERLAY_DIR" +cat > "$EMPTY_MODULEMAP" <<'EOF' +// Intentionally empty. +EOF +cat > "$OVERLAY_FILE" <