395 lines · 14.9 KB · Swift
RawDownload
import Foundationimport Testing@testable import CodexBarCore @Suite(.serialized)struct CodexBaselineCharacterizationTests {    private func makeContext(        runtime: ProviderRuntime,        sourceMode: ProviderSourceMode,        env: [String: String] = [:],        settings: ProviderSettingsSnapshot? = nil,        includeCredits: Bool = false,        codexArguments: [String]? = nil) -> ProviderFetchContext    {        let browserDetection = BrowserDetection(cacheTTL: 0)        let fetcher = if let codexArguments {            UsageFetcher(                environment: env,                initializeTimeoutSeconds: 20.0,                requestTimeoutSeconds: 3.0,                codexArguments: codexArguments)        } else {            UsageFetcher(environment: env, initializeTimeoutSeconds: 20.0, requestTimeoutSeconds: 3.0)        }        return ProviderFetchContext(            runtime: runtime,            sourceMode: sourceMode,            includeCredits: includeCredits,            webTimeout: 1,            webDebugDumpHTML: false,            verbose: false,            env: env,            settings: settings,            fetcher: fetcher,            claudeFetcher: ClaudeUsageFetcher(browserDetection: browserDetection),            browserDetection: browserDetection)    }     private func strategyIDs(        runtime: ProviderRuntime,        sourceMode: ProviderSourceMode,        env: [String: String] = [:],        settings: ProviderSettingsSnapshot? = nil) async -> [String]    {        let descriptor = ProviderDescriptorRegistry.descriptor(for: .codex)        let context = self.makeContext(runtime: runtime, sourceMode: sourceMode, env: env, settings: settings)        let strategies = await descriptor.fetchPlan.pipeline.resolveStrategies(context)        return strategies.map(\.id)    }     private func fetchOutcome(        runtime: ProviderRuntime,        sourceMode: ProviderSourceMode,        env: [String: String] = [:],        settings: ProviderSettingsSnapshot? = nil,        includeCredits: Bool = false,        codexArguments: [String]? = nil) async -> ProviderFetchOutcome    {        let descriptor = ProviderDescriptorRegistry.descriptor(for: .codex)        let context = self.makeContext(            runtime: runtime,            sourceMode: sourceMode,            env: env,            settings: settings,            includeCredits: includeCredits,            codexArguments: codexArguments)        return await descriptor.fetchPlan.fetchOutcome(context: context, provider: .codex)    }     private struct StubCodexCLI {        let executable: String        let arguments: [String]    }     private func makeStubCodexCLI() -> StubCodexCLI {        let script = """        if [ -n "${CODEXBAR_STUB_COUNTER:-}" ]; then          printf '%s\\n' start >> "$CODEXBAR_STUB_COUNTER"        fi         while IFS= read -r line; do          case "$line" in            *'"method":"initialized"'*|*'"method": "initialized"'*)              ;;            *'"method":"initialize"'*|*'"method": "initialize"'*)              printf '%s\\n' '{"id":1,"result":{}}'              ;;            *'"method"'*account*rateLimits*read*)              if [ "${CODEXBAR_STUB_CREDITS_ONLY:-}" = "1" ]; then                response='{"id":2,"result":{"rateLimits":{"credits":'                response="${response}"'{"hasCredits":true,"unlimited":false,"balance":"7"}}}}'                printf '%s\\n' "$response"              else                response='{"id":2,"result":{"rateLimits":{"credits":'                response="${response}"'{"hasCredits":true,"unlimited":false,"balance":"7"},'                if [ "${CODEXBAR_STUB_MONTHLY_LIMIT:-}" = "1" ]; then                  response="${response}"'"individualLimit":{"limit":100000,"used":7761,'                  response="${response}"'"remainingPercent":92.239,"resetsAt":1782864000},'                fi                response="${response}"'"primary":{"usedPercent":12,"windowDurationMins":300,"resetsAt":1766948068},'                response="${response}"'"secondary":{"usedPercent":43,"windowDurationMins":10080,'                response="${response}"'"resetsAt":1767407914}}}}'                printf '%s\\n' "$response"              fi              ;;            *'"method"'*account*read*)              response='{"id":3,"result":{"account":{"type":"chatgpt","email":"stub@example.com",'              response="${response}"'"planType":"pro"},"requiresOpenaiAuth":false}}'              printf '%s\\n' "$response"              ;;          esac        done        """        return StubCodexCLI(executable: "/bin/sh", arguments: ["-c", script])    }     private func makeEmptyCodexHome() throws -> URL {        let homeURL = FileManager.default.temporaryDirectory            .appendingPathComponent("codex-empty-home-\(UUID().uuidString)", isDirectory: true)        try FileManager.default.createDirectory(at: homeURL, withIntermediateDirectories: true)        return homeURL    }     private func makeUnavailableOAuthHome() throws -> URL {        let homeURL = FileManager.default.temporaryDirectory            .appendingPathComponent("codex-oauth-home-\(UUID().uuidString)", isDirectory: true)        try FileManager.default.createDirectory(at: homeURL, withIntermediateDirectories: true)         let credentials = CodexOAuthCredentials(            accessToken: "access-token",            refreshToken: "refresh-token",            idToken: nil,            accountId: "account-id",            lastRefresh: Date())        try CodexOAuthCredentialsStore.save(credentials, env: ["CODEX_HOME": homeURL.path])         let configURL = homeURL.appendingPathComponent("config.toml")        try "chatgpt_base_url = \"http://127.0.0.1:9\"".write(to: configURL, atomically: true, encoding: .utf8)         return homeURL    }     @Test    func `app auto pipeline order is OAuth then CLI without web`() async {        let strategyIDs = await self.strategyIDs(runtime: .app, sourceMode: .auto)        #expect(strategyIDs == ["codex.oauth", "codex.cli"])    }     @Test    func `CLI auto pipeline order is OAuth then CLI without web`() async {        let strategyIDs = await self.strategyIDs(runtime: .cli, sourceMode: .auto)        #expect(strategyIDs == ["codex.oauth", "codex.cli"])    }     @Test    func `explicit fetch plan modes keep single Codex strategy selection`() async {        let appCases: [(ProviderSourceMode, [String])] = [            (.oauth, ["codex.oauth"]),            (.cli, ["codex.cli"]),            (.web, ["codex.web.dashboard"]),        ]         for (sourceMode, expected) in appCases {            let strategyIDs = await self.strategyIDs(runtime: .app, sourceMode: sourceMode)            #expect(strategyIDs == expected)        }         for (sourceMode, expected) in appCases {            let strategyIDs = await self.strategyIDs(runtime: .cli, sourceMode: sourceMode)            #expect(strategyIDs == expected)        }    }     @Test    func `app auto records unavailable OAuth before successful CLI fallback`() async throws {        let stubCLI = self.makeStubCodexCLI()        let codexHome = try self.makeEmptyCodexHome()        defer { try? FileManager.default.removeItem(at: codexHome) }        let env = [            "CODEX_CLI_PATH": stubCLI.executable,            "CODEX_HOME": codexHome.path,        ]         let outcome = await self.fetchOutcome(            runtime: .app,            sourceMode: .auto,            env: env,            codexArguments: stubCLI.arguments)         #expect(outcome.attempts.map(\.strategyID) == ["codex.oauth", "codex.cli"])        #expect(outcome.attempts.map(\.wasAvailable) == [false, true])         switch outcome.result {        case let .success(result):            #expect(result.sourceLabel == "codex-cli")            #expect(result.usage.accountEmail(for: .codex) == "stub@example.com")            #expect(result.usage.loginMethod(for: .codex) == "pro")        case let .failure(error):            Issue.record("Unexpected failure: \(error)")        }    }     @Test    func `app auto does not fall back from non auth failing OAuth`() async throws {        let stubCLI = self.makeStubCodexCLI()        let oauthHome = try self.makeUnavailableOAuthHome()        defer { try? FileManager.default.removeItem(at: oauthHome) }         let env = [            "CODEX_CLI_PATH": stubCLI.executable,            "CODEX_HOME": oauthHome.path,        ]         let outcome = await self.fetchOutcome(            runtime: .app,            sourceMode: .auto,            env: env,            codexArguments: stubCLI.arguments)         #expect(outcome.attempts.map(\.strategyID) == ["codex.oauth"])        #expect(outcome.attempts.map(\.wasAvailable) == [true])        #expect(outcome.attempts[0].errorDescription?.isEmpty == false)         switch outcome.result {        case .success:            Issue.record("Expected non-auth OAuth failure to stop before CLI fallback")        case let .failure(error as CodexOAuthFetchError):            switch error {            case .networkError:                break            default:                Issue.record("Expected network error, got \(error)")            }        case let .failure(error):            Issue.record("Unexpected failure: \(error)")        }    }     @Test    func `Codex CLI strategy fetches usage and credits with one app-server process`() async {        let stubCLI = self.makeStubCodexCLI()        let counterURL = FileManager.default.temporaryDirectory            .appendingPathComponent("codex-stub-counter-\(UUID().uuidString)", isDirectory: false)        defer { try? FileManager.default.removeItem(at: counterURL) }         let env = [            "CODEX_CLI_PATH": stubCLI.executable,            "CODEXBAR_STUB_COUNTER": counterURL.path,        ]         let outcome = await self.fetchOutcome(            runtime: .app,            sourceMode: .cli,            env: env,            includeCredits: true,            codexArguments: stubCLI.arguments)         switch outcome.result {        case let .success(result):            #expect(result.sourceLabel == "codex-cli")            #expect(result.usage.primary?.usedPercent == 12)            #expect(result.credits?.remaining == 7)        case let .failure(error):            Issue.record("Unexpected failure: \(error)")        }         let count = (try? String(contentsOf: counterURL, encoding: .utf8))?            .split(whereSeparator: \.isNewline)            .count ?? 0        #expect(count == 1)    }     @Test    func `Codex CLI strategy keeps credits when rate limit windows are absent`() async {        let stubCLI = self.makeStubCodexCLI()         let outcome = await self.fetchOutcome(            runtime: .app,            sourceMode: .cli,            env: [                "CODEX_CLI_PATH": stubCLI.executable,                "CODEXBAR_STUB_CREDITS_ONLY": "1",            ],            includeCredits: true,            codexArguments: stubCLI.arguments)         switch outcome.result {        case let .success(result):            #expect(result.sourceLabel == "codex-cli")            #expect(result.usage.primary == nil)            #expect(result.usage.secondary == nil)            #expect(result.usage.accountEmail(for: .codex) == "stub@example.com")            #expect(result.credits?.remaining == 7)        case let .failure(error):            Issue.record("Unexpected failure: \(error)")        }    }     @Test    func `Codex CLI strategy maps monthly credit limit`() async {        let stubCLI = self.makeStubCodexCLI()         let outcome = await self.fetchOutcome(            runtime: .app,            sourceMode: .cli,            env: [                "CODEX_CLI_PATH": stubCLI.executable,                "CODEXBAR_STUB_MONTHLY_LIMIT": "1",            ],            includeCredits: true,            codexArguments: stubCLI.arguments)         switch outcome.result {        case let .success(result):            let limit = try? #require(result.credits?.codexCreditLimit)            #expect(limit?.limit == 100_000)            #expect(limit?.used == 7761)            #expect(limit?.remaining == 92239)            #expect(limit?.remainingPercent == 92.239)            #expect(limit?.resetsAt == Date(timeIntervalSince1970: 1_782_864_000))        case let .failure(error):            Issue.record("Unexpected failure: \(error)")        }    }     @Test    func `CLI auto records unavailable OAuth before successful CLI`() async throws {        let stubCLI = self.makeStubCodexCLI()        let codexHome = try self.makeEmptyCodexHome()        defer { try? FileManager.default.removeItem(at: codexHome) }        let settings = ProviderSettingsSnapshot.make(            codex: .init(                usageDataSource: .auto,                cookieSource: .auto,                manualCookieHeader: nil,                managedAccountStoreUnreadable: true))         let outcome = await self.fetchOutcome(            runtime: .cli,            sourceMode: .auto,            env: [                "CODEX_CLI_PATH": stubCLI.executable,                "CODEX_HOME": codexHome.path,            ],            settings: settings,            codexArguments: stubCLI.arguments)         #expect(outcome.attempts.map(\.strategyID) == ["codex.oauth", "codex.cli"])        #expect(outcome.attempts.map(\.wasAvailable) == [false, true])         switch outcome.result {        case let .success(result):            #expect(result.sourceLabel == "codex-cli")            #expect(result.usage.accountEmail(for: .codex) == "stub@example.com")        case let .failure(error):            Issue.record("Unexpected failure: \(error)")        }    }     @Test    func `CLI auto tries OAuth before missing CLI fallback`() async throws {        let oauthHome = try self.makeUnavailableOAuthHome()        defer { try? FileManager.default.removeItem(at: oauthHome) }        let settings = ProviderSettingsSnapshot.make(            codex: .init(                usageDataSource: .auto,                cookieSource: .auto,                manualCookieHeader: nil,                managedAccountStoreUnreadable: true))         let outcome = await self.fetchOutcome(            runtime: .cli,            sourceMode: .auto,            env: [                "CODEX_CLI_PATH": "/missing/codex",                "CODEX_HOME": oauthHome.path,            ],            settings: settings)         #expect(outcome.attempts.map(\.strategyID) == ["codex.oauth"])        #expect(outcome.attempts.map(\.wasAvailable) == [true])         switch outcome.result {        case .success:            Issue.record("Expected unavailable OAuth endpoint to fail before CLI fallback")        case let .failure(error as CodexOAuthFetchError):            if case .networkError = error {                break            }            Issue.record("Expected network error, got \(error)")        case let .failure(error):            Issue.record("Unexpected failure: \(error)")        }    }}