Files
steipete--codexbar/Tests/CodexBarTests/ProviderPluginRuntimeTests.swift
Alec Gutman, Chip d353ace608 Account for all connected providers in Overview spend (#3054)
* Fix OpenRouter spend in Overview

* Refactor plugin cost usage mapping

* Secure OpenRouter management key routing

* Update OpenRouter detail parity fixtures

* Narrow management auth to OpenRouter

* Pin OpenRouter management auth in broker

* Avoid reserved provider test pollution

* Rerun CI after test isolation fix

* Wire OpenRouter app settings snapshot

* Expose OpenRouter management key setting

* Expose OpenRouter settings key to app tests

* Classify invalid OpenRouter activity rows

* Document provider-specific spend paths

* Include OpenRouter in cost source contract

---------

Co-authored-by: Alec Gutman <chipagosfinest@users.noreply.github.com>
2026-08-18 16:13:19 -07:00

904 lines
36 KiB
Swift

import Foundation
#if canImport(FoundationNetworking)
import FoundationNetworking
#endif
import Testing
@testable import CodexBarCore
struct ProviderPluginRuntimeTests {
@Test
func `automatic engine defaults to QuickJS`() {
#expect(ProviderPluginRuntime.resolveEngineKind(
.automatic,
environment: [:],
useJavaScriptCoreRollback: false) == .quickJS)
}
@Test
func `explicit engine selection bypasses automatic policy`() {
#expect(ProviderPluginRuntime.resolveEngineKind(
.quickJS,
environment: [ProviderPluginRuntime.engineEnvironmentKey: "jsc"],
useJavaScriptCoreRollback: true) == .quickJS)
}
#if canImport(JavaScriptCore)
@Test
func `JavaScriptCore rollback supports environment and debug defaults`() {
#expect(ProviderPluginRuntime.resolveEngineKind(
.automatic,
environment: [ProviderPluginRuntime.engineEnvironmentKey: "jsc"],
useJavaScriptCoreRollback: false) == .javaScriptCore)
#expect(ProviderPluginRuntime.resolveEngineKind(
.automatic,
environment: [:],
useJavaScriptCoreRollback: true) == .javaScriptCore)
#expect(ProviderPluginRuntime.resolveEngineKind(
.automatic,
environment: [ProviderPluginRuntime.engineEnvironmentKey: "quickjs"],
useJavaScriptCoreRollback: true) == .quickJS)
}
#endif
@Test
func `missing resource bundle throws a provider load error`() {
#expect(throws: ProviderPluginError.load(CodexBarCoreResources.missingBundleMessage)) {
_ = try ProviderPluginRuntime(bundledPlugin: "openrouter", resourceBundle: nil)
}
}
@Test
func `date now uses the injected fetch clock`() async throws {
let expected = Date(timeIntervalSince1970: 1_800_000_000)
let runtime = try ProviderPluginRuntime(source: Self.plugin(fetchBody: """
return { primary: { usedPercent: 1, resetsAt: ctx.date.now() } };
"""))
let snapshot = try await runtime.fetchUsage(secrets: ["TEST_KEY": "test-key"], now: expected)
#expect(snapshot.primary?.resetsAt == expected)
}
@Test
func `date nowMillis uses the injected fetch clock`() async throws {
let expected = Date(timeIntervalSince1970: 1_800_000_000)
let runtime = try ProviderPluginRuntime(source: Self.plugin(fetchBody: """
return { primary: { usedPercent: 1, resetDescription: String(ctx.date.nowMillis()) } };
"""))
let snapshot = try await runtime.fetchUsage(secrets: ["TEST_KEY": "test-key"], now: expected)
#expect(snapshot.primary?.resetDescription == "1800000000000")
}
@Test
func `context exposes no browser or timer globals`() throws {
let runtime = try ProviderPluginRuntime(source: Self.plugin())
#expect(try runtime.globalType(of: "fetch") == "undefined")
#expect(try runtime.globalType(of: "XMLHttpRequest") == "undefined")
#expect(try runtime.globalType(of: "setTimeout") == "undefined")
#expect(try runtime.globalType(of: "setInterval") == "undefined")
#expect(try runtime.globalType(of: "ctx") == "undefined")
}
@Test
func `origin allowlist rejects before issuing request`() async throws {
let requests = RequestRecorder()
let runtime = try ProviderPluginRuntime(
source: Self.plugin(fetchBody: """
const response = await ctx.http.getJSON("https://other.example/usage");
return { primary: { usedPercent: response.status } };
"""),
transport: Self.transport(recorder: requests))
await #expect(throws: ProviderPluginError.self) {
_ = try await runtime.fetchUsage(secrets: ["TEST_KEY": "secret"])
}
#expect(await requests.isEmpty)
}
@Test
func `HTTP broker injects auth and returns JSON`() async throws {
let requests = RequestRecorder()
let runtime = try ProviderPluginRuntime(
source: Self.plugin(fetchBody: """
const response = await ctx.http.getJSON("https://api.example.test/usage", {
headers: { "X-Client": "plugin-test" },
});
return { primary: { usedPercent: response.json.used } };
"""),
transport: Self.transport(recorder: requests, body: #"{"used":42}"#))
let snapshot = try await runtime.fetchUsage(secrets: ["TEST_KEY": "secret-value"])
#expect(snapshot.primary?.usedPercent == 42)
let request = try #require(await requests.first)
#expect(request.value(forHTTPHeaderField: "Authorization") == "Bearer secret-value")
#expect(request.value(forHTTPHeaderField: "X-Client") == "plugin-test")
}
@Test
func `HTTP broker rejects OpenRouter management auth outside OpenRouter plugin`() async throws {
let runtime = try ProviderPluginRuntime(source: Self.plugin(
settings: """
{ key: "TEST_KEY", title: "API key", type: "secure" },
{ key: "OPENROUTER_MANAGEMENT_API_KEY", title: "Management key", type: "secure" }
""",
fetchBody: """
await ctx.http.getJSON("https://api.example.test/usage", { openRouterManagementAuth: true });
return { primary: { usedPercent: 1 } };
"""))
await #expect(throws: ProviderPluginError.self) {
_ = try await runtime.fetchUsage(secrets: [
"TEST_KEY": "secret",
"OPENROUTER_MANAGEMENT_API_KEY": "management-secret",
])
}
}
@Test
func `HTTP request deadline defaults to fifteen seconds and accepts bounded override`() async throws {
let requests = RequestRecorder()
let runtime = try ProviderPluginRuntime(
source: Self.plugin(fetchBody: """
await ctx.http.getJSON("https://api.example.test/default");
const response = await ctx.http.getJSON("https://api.example.test/override", { timeoutSeconds: 7.5 });
return { primary: { usedPercent: response.json.used } };
"""),
transport: Self.transport(recorder: requests, body: #"{"used":11}"#))
let snapshot = try await runtime.fetchUsage(secrets: ["TEST_KEY": "secret-value"])
#expect(snapshot.primary?.usedPercent == 11)
let recorded = await requests.all
#expect(recorded.map(\.timeoutInterval) == [15, 7.5])
}
@Test(arguments: ["0", "0.5", "31", #""slow""#])
func `HTTP request deadline rejects values outside one through thirty seconds`(value: String) async throws {
let requests = RequestRecorder()
let runtime = try ProviderPluginRuntime(
source: Self.plugin(fetchBody: """
await ctx.http.getJSON("https://api.example.test/usage", { timeoutSeconds: \(value) });
return { primary: { usedPercent: 1 } };
"""),
transport: Self.transport(recorder: requests))
await #expect(throws: ProviderPluginError.self) {
_ = try await runtime.fetchUsage(secrets: ["TEST_KEY": "secret-value"])
}
#expect(await requests.isEmpty)
}
@Test
func `HTTP request deadline cancels a transport that exceeds it`() async throws {
let cancellation = TransportCancellationProbe()
let runtime = try ProviderPluginRuntime(
source: Self.plugin(fetchBody: """
await ctx.http.getJSON("https://api.example.test/slow", { timeoutSeconds: 1 });
return { primary: { usedPercent: 1 } };
"""),
transport: ProviderHTTPTransportHandler { request in
do {
try await Task.sleep(for: .seconds(5))
} catch is CancellationError {
await cancellation.markCancelled()
throw CancellationError()
}
let response = try #require(HTTPURLResponse(
url: request.url!,
statusCode: 200,
httpVersion: nil,
headerFields: ["Content-Type": "application/json"]))
return (Data(#"{"used":1}"#.utf8), response)
})
do {
_ = try await runtime.fetchUsage(secrets: ["TEST_KEY": "secret-value"])
Issue.record("Expected the request deadline to reject the plugin fetch")
} catch let error as ProviderPluginError {
guard case .script = error else {
Issue.record("Expected a request deadline failure, received \(error)")
return
}
await cancellation.waitUntilCancelled()
} catch {
Issue.record("Unexpected error: \(error)")
}
}
@Test
func `settings split enforces kind and only secrets are redacted`() async throws {
let runtime = try ProviderPluginRuntime(source: Self.plugin(
settings: """
{ key: "TEST_KEY", title: "API key", type: "secure" },
{ key: "PLAIN_VALUE", title: "Plain value", type: "plain" }
""",
fetchBody: """
const plain = ctx.settings.get("PLAIN_VALUE");
const secret = ctx.settings.getSecret("TEST_KEY");
throw new Error(`${plain}:${secret}`);
"""))
do {
_ = try await runtime.fetchUsage(
settings: ["PLAIN_VALUE": "visible-setting"],
secrets: ["TEST_KEY": "hidden-secret"])
Issue.record("Expected rejection")
} catch {
#expect(error.localizedDescription.contains("visible-setting"))
#expect(!error.localizedDescription.contains("hidden-secret"))
#expect(error.localizedDescription.contains("<redacted>"))
}
}
@Test
func `authorization scheme injects bounded token prefix`() async throws {
let requests = RequestRecorder()
let runtime = try ProviderPluginRuntime(
source: Self.plugin(
auth: #"{ type: "authorization-scheme", scheme: "Token", secret: "TEST_KEY" }"#,
fetchBody: """
const response = await ctx.http.getJSON("https://api.example.test/usage");
return { primary: { usedPercent: response.status } };
"""),
transport: Self.transport(recorder: requests))
_ = try await runtime.fetchUsage(secrets: ["TEST_KEY": "fixture-key"])
#expect(await requests.first?.value(forHTTPHeaderField: "Authorization") == "Token fixture-key")
#expect(throws: ProviderPluginError.self) {
_ = try ProviderPluginRuntime(source: Self.plugin(
auth: #"{ type: "authorization-scheme", scheme: "bad scheme", secret: "TEST_KEY" }"#))
}
}
@Test
func `postJSON sends serialized body without relaxing headers`() async throws {
let requests = RequestRecorder()
let runtime = try ProviderPluginRuntime(
source: Self.plugin(fetchBody: """
const response = await ctx.http.postJSON("https://api.example.test/usage", {
body: { team: "fixture", count: 2 },
headers: { "X-Client": "plugin-test" },
});
return { primary: { usedPercent: response.json.used } };
"""),
transport: Self.transport(recorder: requests, body: #"{"used":17}"#))
let snapshot = try await runtime.fetchUsage(secrets: ["TEST_KEY": "fixture-key"])
let request = try #require(await requests.first)
#expect(snapshot.primary?.usedPercent == 17)
#expect(request.httpMethod == "POST")
#expect(request.value(forHTTPHeaderField: "Content-Type") == "application/json")
let body = try #require(request.httpBody)
let object = try #require(JSONSerialization.jsonObject(with: body) as? [String: Any])
#expect(object["team"] as? String == "fixture")
#expect(object["count"] as? Int == 2)
}
@Test
func `setting endpoint resolves at fetch time and permits loopback HTTP only by policy`() async throws {
let requests = RequestRecorder()
let runtime = try ProviderPluginRuntime(
source: Self.plugin(
endpoints: #"[{ setting: "BASE_URL", policy: "https-or-loopback-http" }]"#,
settings: """
{ key: "TEST_KEY", title: "API key", type: "secure" },
{ key: "BASE_URL", title: "Base URL", type: "plain" }
""",
fetchBody: """
const base = ctx.settings.get("BASE_URL");
const response = await ctx.http.getJSON(`${base}/usage`);
return { primary: { usedPercent: response.json.used } };
"""),
transport: Self.transport(recorder: requests, body: #"{"used":23}"#))
let snapshot = try await runtime.fetchUsage(
settings: ["BASE_URL": "http://127.0.0.1:8787"],
secrets: ["TEST_KEY": "fixture-key"])
#expect(snapshot.primary?.usedPercent == 23)
#expect(await requests.first?.url?.absoluteString == "http://127.0.0.1:8787/usage")
let rejectedRequests = RequestRecorder()
let rejected = try ProviderPluginRuntime(
source: Self.plugin(
endpoints: #"[{ setting: "BASE_URL", policy: "https-or-loopback-http" }]"#,
settings: """
{ key: "TEST_KEY", title: "API key", type: "secure" },
{ key: "BASE_URL", title: "Base URL", type: "plain" }
""",
fetchBody: """
await ctx.http.getJSON(`${ctx.settings.get("BASE_URL")}/usage`);
return { primary: { usedPercent: 1 } };
"""),
transport: Self.transport(recorder: rejectedRequests))
await #expect(throws: ProviderPluginError.self) {
_ = try await rejected.fetchUsage(
settings: ["BASE_URL": "http://example.test"],
secrets: ["TEST_KEY": "fixture-key"])
}
#expect(await rejectedRequests.isEmpty)
}
@Test(arguments: [
"http://localhost:4000",
"http://10.0.0.8:4000",
"http://172.16.0.8:4000",
"http://192.168.1.8:4000",
"http://169.254.1.8:4000",
"http://[fc00::8]:4000",
"http://[fe80::8]:4000",
"http://gateway.local:4000",
"https://gateway.example:4000",
])
func `private network endpoint policy accepts the native gateway origin set`(origin: String) async throws {
let requests = RequestRecorder()
let runtime = try ProviderPluginRuntime(
source: Self.plugin(
id: "llmproxy",
endpoints: #"[{ setting: "BASE_URL", policy: "https-or-private-network-http" }]"#,
settings: """
{ key: "TEST_KEY", title: "API key", type: "secure" },
{ key: "BASE_URL", title: "Base URL", type: "plain" }
""",
fetchBody: """
const response = await ctx.http.getJSON(`${ctx.settings.get("BASE_URL")}/usage`);
return { primary: { usedPercent: response.json.used } };
"""),
transport: Self.transport(recorder: requests, body: #"{"used":4}"#))
let snapshot = try await runtime.fetchUsage(
settings: ["BASE_URL": origin],
secrets: ["TEST_KEY": "fixture-key"])
#expect(snapshot.primary?.usedPercent == 4)
#expect(await requests.first?.url?.absoluteString == "\(origin)/usage")
}
@Test
func `private network endpoint policy rejects public HTTP before transport`() async throws {
let requests = RequestRecorder()
let runtime = try ProviderPluginRuntime(
source: Self.plugin(
id: "litellm",
endpoints: #"[{ setting: "BASE_URL", policy: "https-or-private-network-http" }]"#,
settings: """
{ key: "TEST_KEY", title: "API key", type: "secure" },
{ key: "BASE_URL", title: "Base URL", type: "plain" }
""",
fetchBody: """
await ctx.http.getJSON(`${ctx.settings.get("BASE_URL")}/usage`);
return { primary: { usedPercent: 1 } };
"""),
transport: Self.transport(recorder: requests))
await #expect(throws: ProviderPluginError.self) {
_ = try await runtime.fetchUsage(
settings: ["BASE_URL": "http://gateway.example"],
secrets: ["TEST_KEY": "fixture-key"])
}
#expect(await requests.isEmpty)
}
@Test
func `bundled private network policy is limited to providers with native parity`() throws {
_ = try ProviderPluginRuntime(source: Self.plugin(
id: "llmproxy",
endpoints: #"[{ setting: "BASE_URL", policy: "https-or-private-network-http" }]"#,
auth: "null",
settings: #"{ key: "BASE_URL", title: "Base URL", type: "plain" }"#))
#expect(throws: ProviderPluginError.self) {
_ = try ProviderPluginRuntime(source: Self.plugin(
endpoints: #"[{ setting: "BASE_URL", policy: "https-or-private-network-http" }]"#,
auth: "null",
settings: #"{ key: "BASE_URL", title: "Base URL", type: "plain" }"#))
}
}
@Test
func `browser cookie access is declared domain only and cookie values are redacted`() async throws {
let access = CookieAccessRecorder(header: "session=secret-cookie-value")
let runtime = try ProviderPluginRuntime(source: Self.plugin(
capabilities: #"capabilities: ["browser-cookies"], cookieDomains: ["example.test"],"#,
fetchBody: """
const cookie = await ctx.browser.cookieHeader("example.test");
throw new Error(`cookie was ${cookie}`);
"""))
do {
_ = try await runtime.fetchUsage(
secrets: ["TEST_KEY": "fixture-key"],
cookieResolver: { provider, domain in try await access.resolve(provider: provider, domain: domain) })
Issue.record("Expected rejection")
} catch {
#expect(!error.localizedDescription.contains("secret-cookie-value"))
#expect(error.localizedDescription.contains("<redacted>"))
}
#expect(await access.domains == ["example.test"])
let rejected = try ProviderPluginRuntime(source: Self.plugin(
capabilities: #"capabilities: ["browser-cookies"], cookieDomains: ["example.test"],"#,
fetchBody: """
await ctx.browser.cookieHeader("other.test");
return { primary: { usedPercent: 1 } };
"""))
await #expect(throws: ProviderPluginError.self) {
_ = try await rejected.fetchUsage(
secrets: ["TEST_KEY": "fixture-key"],
cookieResolver: { provider, domain in try await access.resolve(provider: provider, domain: domain) })
}
#expect(await access.domains == ["example.test"])
}
@Test
func `HTML helpers extract meta content and first regex capture`() async throws {
let runtime = try ProviderPluginRuntime(source: Self.plugin(fetchBody: """
const html = '<meta content="fixture-token" name="csrf"><div>credits: 37</div>';
const meta = ctx.html.metaContent(html, "csrf");
const credits = ctx.html.matchFirst(html, "credits:\\\\s*(\\\\d+)", "i");
return { primary: { usedPercent: meta === "fixture-token" ? Number(credits) : 0 } };
"""))
let snapshot = try await runtime.fetchUsage(secrets: ["TEST_KEY": "fixture-key"])
#expect(snapshot.primary?.usedPercent == 37)
}
@Test
func `undeclared secret access fails`() async throws {
let runtime = try ProviderPluginRuntime(source: Self.plugin(fetchBody: """
ctx.settings.getSecret("OTHER_KEY");
return { primary: { usedPercent: 1 } };
"""))
await #expect(throws: ProviderPluginError.self) {
_ = try await runtime.fetchUsage(secrets: ["TEST_KEY": "secret"])
}
}
@Test(arguments: [
"defineProvider({",
"defineProvider({ id: 'synthetic' });",
"""
defineProvider({
id: "not-a-provider",
name: "Bad",
endpoints: ["https://api.example.test"],
auth: { type: "bearer", secret: "TEST_KEY" },
settings: [{ key: "TEST_KEY", title: "Key" }],
fetchUsage: async () => ({ primary: { usedPercent: 0 } }),
});
""",
])
func `malformed plugins have descriptive load errors`(source: String) {
#expect(throws: ProviderPluginError.self) {
_ = try ProviderPluginRuntime(source: source)
}
}
@Test
func `wrong typed snapshot field fails`() async throws {
let runtime = try ProviderPluginRuntime(source: Self.plugin(fetchBody: """
return { primary: { usedPercent: "42" } };
"""))
await #expect(throws: ProviderPluginError.self) {
_ = try await runtime.fetchUsage(secrets: ["TEST_KEY": "secret"])
}
}
@Test(arguments: [
("exact", UsageDataConfidence.exact),
("estimated", .estimated),
("percentOnly", .percentOnly),
("unknown", .unknown),
])
func `data confidence maps validated values`(rawValue: String, expected: UsageDataConfidence) async throws {
let runtime = try ProviderPluginRuntime(source: Self.plugin(fetchBody: """
return { primary: { usedPercent: 1 }, dataConfidence: "\(rawValue)" };
"""))
let snapshot = try await runtime.fetchUsage(secrets: ["TEST_KEY": "secret"])
#expect(snapshot.dataConfidence == expected)
}
@Test(arguments: [#""certain""#, "42"])
func `invalid data confidence fails the snapshot`(value: String) async throws {
let runtime = try ProviderPluginRuntime(source: Self.plugin(fetchBody: """
return { primary: { usedPercent: 1 }, dataConfidence: \(value) };
"""))
await #expect(throws: ProviderPluginError.self) {
_ = try await runtime.fetchUsage(secrets: ["TEST_KEY": "secret"])
}
}
@Test
func `identity only snapshot preserves a successful sparse provider state`() async throws {
let runtime = try ProviderPluginRuntime(source: Self.plugin(fetchBody: """
return { identity: { organization: "Moonshot", loginMethod: "Balance: $0.00" } };
"""))
let snapshot = try await runtime.fetchUsage(secrets: ["TEST_KEY": "secret"])
#expect(snapshot.primary == nil)
#expect(snapshot.providerCost == nil)
#expect(snapshot.identity?.accountOrganization == "Moonshot")
#expect(snapshot.identity?.loginMethod == "Balance: $0.00")
}
@Test(arguments: [
#"return {};"#,
#"return { identity: {} };"#,
#"return { identity: { email: " " } };"#,
#"return { dataConfidence: "exact" };"#,
#"return { subscriptionRenewsAt: "2026-09-01T00:00:00Z" };"#,
])
func `empty and metadata only snapshots remain invalid`(fetchBody: String) async throws {
let runtime = try ProviderPluginRuntime(source: Self.plugin(fetchBody: fetchBody))
await #expect(throws: ProviderPluginError.self) {
_ = try await runtime.fetchUsage(secrets: ["TEST_KEY": "secret"])
}
}
@Test
func `details map strictly and trim display strings`() async throws {
let runtime = try ProviderPluginRuntime(source: Self.plugin(fetchBody: """
return {
details: [{
title: " Summary ",
rows: [{ label: " Requests ", value: " 42 ", secondaryValue: " Today " }],
chart: {
kind: "line",
title: " Daily ",
unit: " tokens ",
points: [{ label: " Mon ", value: 12.5 }],
},
}],
};
"""))
let snapshot = try await runtime.fetchUsage(secrets: ["TEST_KEY": "secret"])
let section = try #require(snapshot.details.first)
#expect(section.title == "Summary")
#expect(try section.rows == [ProviderDetailSection.Row(
label: "Requests",
value: "42",
secondaryValue: "Today")])
let expectedChart = try ProviderDetailSection.Chart(
kind: .line,
title: "Daily",
unit: "tokens",
points: [ProviderDetailSection.Chart.Point(label: "Mon", value: 12.5)])
#expect(section.chart == expectedChart)
}
@Test(arguments: [
#"details: {}"#,
#"details: [{ rows: [{ label: "ok", value: 1 }] }]"#,
#"details: [{ rows: [], chart: { kind: "pie", points: [] } }]"#,
#"details: [{ rows: [], chart: { kind: "bars", points: [{ label: "x", value: NaN }] } }]"#,
#"""
details: [{
rows: [],
chart: { kind: "bars", points: Array.from({length: 121}, (_, i) => ({label: String(i), value: i})) },
}]
"""#,
])
func `present invalid details fail the fetch`(body: String) async throws {
let runtime = try ProviderPluginRuntime(source: Self.plugin(fetchBody: "return { \(body) };"))
await #expect(throws: ProviderPluginError.self) {
_ = try await runtime.fetchUsage(secrets: ["TEST_KEY": "secret"])
}
}
@Test
func `promise rejection preserves message`() async throws {
let runtime = try ProviderPluginRuntime(source: Self.plugin(fetchBody: """
throw new Error("fixture rejected");
"""))
do {
_ = try await runtime.fetchUsage(secrets: ["TEST_KEY": "secret"])
Issue.record("Expected rejection")
} catch {
#expect(error.localizedDescription.contains("fixture rejected"))
}
}
@Test(arguments: ProviderFetchClassifiedError.Kind.allCases)
func `classified failures preserve kind and message`(kind: ProviderFetchClassifiedError.Kind) async throws {
let apiName = switch kind {
case .authenticationExpired: "authenticationExpired"
case .missingCredential: "missingCredential"
case .permissionDenied: "permissionDenied"
case .rateLimited: "rateLimited"
case .providerUnavailable: "providerUnavailable"
case .parseFailure: "parseFailure"
case .networkFailure: "networkFailure"
case .apiFailure: "apiFailure"
}
let runtime = try ProviderPluginRuntime(source: Self.plugin(fetchBody: """
throw ctx.fail.\(apiName)("classified fixture");
"""))
do {
_ = try await runtime.fetchUsage(secrets: ["TEST_KEY": "secret"])
Issue.record("Expected classified failure")
} catch let error as ProviderFetchClassifiedError {
#expect(error.kind == kind)
#expect(error.message == "classified fixture")
} catch {
Issue.record("Unexpected error: \(error)")
}
}
@Test
func `transient classified failure preserves capped retry delay`() async throws {
let runtime = try ProviderPluginRuntime(source: Self.plugin(fetchBody: """
throw ctx.fail.rateLimited("retry later", { retryAfterSeconds: 30 });
"""))
do {
_ = try await runtime.fetchUsage(secrets: ["TEST_KEY": "secret"])
Issue.record("Expected classified failure")
} catch let error as ProviderFetchClassifiedError {
#expect(error.kind == .rateLimited)
#expect(error.message == "retry later")
#expect(error.retryAfterSeconds == 10)
} catch {
Issue.record("Unexpected error: \(error)")
}
}
@Test
func `non transient classified failure rejects retry options`() async throws {
let runtime = try ProviderPluginRuntime(source: Self.plugin(fetchBody: """
throw ctx.fail.parseFailure("bad payload", { retryAfterSeconds: 1 });
"""))
do {
_ = try await runtime.fetchUsage(secrets: ["TEST_KEY": "secret"])
Issue.record("Expected script failure")
} catch let error as ProviderPluginError {
#expect(error.localizedDescription.contains("retry options are supported only for transient failures"))
} catch {
Issue.record("Unexpected error: \(error)")
}
}
@Test
func `unclassified failures retain generic script mapping`() async throws {
let runtime = try ProviderPluginRuntime(source: Self.plugin(fetchBody: """
throw new Error("ordinary fixture");
"""))
do {
_ = try await runtime.fetchUsage(secrets: ["TEST_KEY": "secret"])
Issue.record("Expected script failure")
} catch let error as ProviderPluginError {
#expect(error == .script("ordinary fixture"))
} catch {
Issue.record("Unexpected error: \(error)")
}
}
@Test
func `script errors redact known secrets`() async throws {
let secret = "super-secret-fixture-value"
let runtime = try ProviderPluginRuntime(source: Self.plugin(fetchBody: """
throw new Error(`leaked: ${ctx.settings.getSecret("TEST_KEY")}`);
"""))
do {
_ = try await runtime.fetchUsage(secrets: ["TEST_KEY": secret])
Issue.record("Expected rejection")
} catch {
#expect(!error.localizedDescription.contains(secret))
#expect(error.localizedDescription.contains("<redacted>"))
}
}
@Test
func `QuickJS engine supports bounded recursion on its dedicated stack`() async throws {
let runtime = try ProviderPluginRuntime(
source: Self.plugin(fetchBody: """
function recurse(depth) {
return depth <= 0 ? 0 : 1 + recurse(depth - 1);
}
return { primary: { usedPercent: recurse(100) } };
"""),
engine: .quickJS)
let snapshot = try await runtime.fetchUsage(secrets: ["TEST_KEY": "secret"])
#expect(snapshot.primary?.usedPercent == 100)
}
@Test
func `QuickJS engine reports recursion beyond its JavaScript stack limit`() async throws {
// Deliberately starved worker geometry: with a fixed JS stack limit this crashed the process
// (guard fires with too little native left to build the RangeError — the quickjs-ng/zipline#1130
// class). The derived limit (a quarter of the worker stack) makes the invariant hold at any size,
// so even a 512 KiB thread throws a clean script error instead of overrunning the guard page.
let engine = try Self.quickJSEngine(
source: Self.plugin(fetchBody: """
function recurse(depth) {
return depth <= 0 ? 0 : 1 + recurse(depth - 1);
}
return { primary: { usedPercent: recurse(100000) } };
"""),
workerStackSizeBytes: 512 * 1024)
do {
_ = try await Self.fetchUsage(engine: engine)
Issue.record("Expected stack overflow")
} catch let error as ProviderPluginError {
guard case let .script(message) = error else {
Issue.record("Unexpected plugin error: \(error)")
return
}
let normalizedMessage = message.lowercased()
#expect(normalizedMessage.contains("stack"))
#expect(normalizedMessage.contains("overflow") || normalizedMessage.contains("exceeded"))
} catch {
Issue.record("Unexpected error: \(error)")
}
}
@Test
func `hung script times out and next fetch uses a fresh context`() async throws {
let runtime = try ProviderPluginRuntime(
source: Self.plugin(fetchBody: """
if (ctx.settings.getSecret("TEST_KEY") === "hang") while (true) {}
return { primary: { usedPercent: 7 } };
"""),
timeout: 5)
let start = Date()
await #expect(throws: ProviderPluginError.self) {
_ = try await runtime.fetchUsage(secrets: ["TEST_KEY": "hang"])
}
// This ceiling only proves the watchdog interrupted the infinite loop instead of hanging
// forever. The runtime timeout itself leaves fresh-worker compilation ample scheduler headroom.
#expect(Date().timeIntervalSince(start) < 30)
let recovered = try await runtime.fetchUsage(secrets: ["TEST_KEY": "ok"])
#expect(recovered.primary?.usedPercent == 7)
}
private static func plugin(
id: String = "synthetic",
endpoints: String = #"["https://api.example.test"]"#,
auth: String = #"{ type: "bearer", secret: "TEST_KEY" }"#,
settings: String = #"{ key: "TEST_KEY", title: "API key", type: "secure" }"#,
capabilities: String = "",
fetchBody: String = "return { primary: { usedPercent: 1 } };") -> String
{
"""
defineProvider({
id: "\(id)",
name: "Fixture",
endpoints: \(endpoints),
auth: \(auth),
settings: [\(settings)],
\(capabilities)
async fetchUsage(ctx) {
\(fetchBody)
},
});
"""
}
private static func quickJSEngine(
source: String,
workerStackSizeBytes: Int) throws -> QuickJSProviderPluginEngine
{
let bundle = try #require(CodexBarCoreResources.bundle)
let preludeURL = try #require(bundle.url(
forResource: "provider-plugin-prelude",
withExtension: "js"))
let preludeSource = try String(contentsOf: preludeURL, encoding: .utf8)
return try QuickJSProviderPluginEngine.make(
source: source,
preludeSource: preludeSource,
transport: ProviderHTTPTransportHandler { _ in throw URLError(.unsupportedURL) },
timeout: ProviderPluginRuntime.defaultTimeout,
responseSizeLimit: ProviderPluginRuntime.maximumResponseBytes,
enforcesUserResponsePolicy: false,
allowsDynamicID: false,
workerStackSizeBytes: workerStackSizeBytes)
}
private static func fetchUsage(engine: QuickJSProviderPluginEngine) async throws -> UsageSnapshot {
let result = await withCheckedContinuation { continuation in
engine.fetch(
settings: [:],
secrets: ["TEST_KEY": "secret"],
now: Date(),
timeZone: .current,
contextOptions: .production,
cookieResolver: nil,
instanceCookieResolver: nil)
{ continuation.resume(returning: $0) }
}
return try result.get()
}
private static func transport(
recorder: RequestRecorder,
body: String = #"{"ok":true}"#) -> ProviderHTTPTransportHandler
{
ProviderHTTPTransportHandler { request in
await recorder.append(request)
let response = try #require(HTTPURLResponse(
url: request.url!,
statusCode: 200,
httpVersion: nil,
headerFields: ["Content-Type": "application/json"]))
return (Data(body.utf8), response)
}
}
}
private actor TransportCancellationProbe {
private var cancelled = false
private var waiters: [CheckedContinuation<Void, Never>] = []
func markCancelled() {
self.cancelled = true
for waiter in self.waiters {
waiter.resume()
}
self.waiters.removeAll()
}
func waitUntilCancelled() async {
if self.cancelled { return }
await withCheckedContinuation { continuation in
self.waiters.append(continuation)
}
}
}
private actor CookieAccessRecorder {
let header: String
private(set) var domains: [String] = []
init(header: String) {
self.header = header
}
func resolve(provider _: UsageProvider, domain: String) throws -> String {
self.domains.append(domain)
return self.header
}
}
private actor RequestRecorder {
private var requests: [URLRequest] = []
var isEmpty: Bool {
self.requests.isEmpty
}
var first: URLRequest? {
self.requests.first
}
var all: [URLRequest] {
self.requests
}
func append(_ request: URLRequest) {
self.requests.append(request)
}
}