diff --git a/.github/workflows/sync-models.yml b/.github/workflows/sync-models.yml index f52656138..7edd03242 100644 --- a/.github/workflows/sync-models.yml +++ b/.github/workflows/sync-models.yml @@ -95,6 +95,7 @@ jobs: run: bun validate - name: Report changes + id: report env: GH_TOKEN: ${{ steps.committer.outputs.token }} BRANCH: automation/sync-models-${{ matrix.provider }} @@ -119,9 +120,20 @@ jobs: git checkout -B "$BRANCH" git add models providers git commit -m "$TITLE" - git push --force-with-lease origin "$BRANCH" + bun sync:auto-merge HEAD^ HEAD + safe="$(sed -n 's/^safe=//p' "$GITHUB_OUTPUT" | tail -1)" pr_number="$(gh pr list --head "$BRANCH" --base dev --json number --jq '.[0].number')" + if [ "$safe" != "true" ] && [ -n "$pr_number" ]; then + gh pr merge "$pr_number" --disable-auto || true + if [ "$(gh pr view "$pr_number" --json autoMergeRequest --jq '.autoMergeRequest == null')" != "true" ]; then + echo "Failed to disable auto-merge for unsafe sync PR #$pr_number." + exit 1 + fi + fi + + git push --force-with-lease origin "$BRANCH" + if [ -n "$pr_number" ]; then gh pr edit "$pr_number" --title "$TITLE" --body-file .sync/model-sync-report.md for label in "${labels[@]}"; do @@ -129,4 +141,12 @@ jobs: done else gh pr create --base dev --head "$BRANCH" --title "$TITLE" --body-file .sync/model-sync-report.md "${label_args[@]}" + pr_number="$(gh pr list --head "$BRANCH" --base dev --json number --jq '.[0].number')" + fi + + if [ "$safe" = "true" ]; then + gh pr merge "$pr_number" --auto --squash + elif [ "$(gh pr view "$pr_number" --json autoMergeRequest --jq '.autoMergeRequest == null')" != "true" ]; then + echo "Unsafe sync PR #$pr_number still has auto-merge enabled." + exit 1 fi diff --git a/package.json b/package.json index cea23def3..a58fbffc0 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,8 @@ "digitalocean:sync": "bun ./packages/core/script/sync-models.ts digitalocean", "ambient:sync": "bun ./packages/core/script/sync-models.ts ambient", "models:sync": "bun ./packages/core/script/sync-models.ts", - "sync:models": "bun ./packages/core/script/sync-models.ts" + "sync:models": "bun ./packages/core/script/sync-models.ts", + "sync:auto-merge": "bun ./packages/core/script/check-sync-auto-merge.ts" }, "dependencies": { "@cloudflare/workers-types": "^4.20260424.1", diff --git a/packages/core/script/check-sync-auto-merge.ts b/packages/core/script/check-sync-auto-merge.ts new file mode 100644 index 000000000..aacfa69d7 --- /dev/null +++ b/packages/core/script/check-sync-auto-merge.ts @@ -0,0 +1,22 @@ +import { appendFile } from "node:fs/promises"; + +import { classifyAutoMerge, parseNameStatus } from "../src/sync/auto-merge.js"; + +const base = process.argv[2] ?? "HEAD^"; +const head = process.argv[3] ?? "HEAD"; +const diff = Bun.spawnSync(["git", "diff", "--name-status", "--no-renames", base, head], { + stdout: "pipe", + stderr: "inherit", +}); + +if (diff.exitCode !== 0) process.exit(diff.exitCode ?? 1); + +const decision = await classifyAutoMerge(parseNameStatus(diff.stdout.toString())); +const summary = decision.safe + ? `Safe to auto-merge: ${decision.created} created, ${decision.updated} updated, ${decision.deleted} deleted.` + : `Manual review required: ${decision.reasons.join("; ")}.`; + +console.log(summary); +if (process.env.GITHUB_OUTPUT) { + await appendFile(process.env.GITHUB_OUTPUT, `safe=${decision.safe}\nsummary=${summary}\n`); +} diff --git a/packages/core/src/sync/auto-merge.ts b/packages/core/src/sync/auto-merge.ts new file mode 100644 index 000000000..f26126723 --- /dev/null +++ b/packages/core/src/sync/auto-merge.ts @@ -0,0 +1,84 @@ +import { readFile } from "node:fs/promises"; + +export const MAX_CREATED_MODELS = 10; +export const MAX_DELETED_MODELS = 10; +export const MAX_MODEL_CHURN = 15; +const REVIEWED_REASONING_PROVIDERS = new Set(["openrouter"]); + +export interface CatalogChange { + status: "created" | "updated" | "deleted"; + path: string; +} + +export interface AutoMergeDecision { + safe: boolean; + created: number; + updated: number; + deleted: number; + reasons: string[]; +} + +function isModel(path: string) { + return path.endsWith(".toml") && (path.startsWith("models/") || path.includes("/models/")); +} + +function isProviderModel(path: string) { + return path.endsWith(".toml") && path.startsWith("providers/") && path.includes("/models/"); +} + +export async function classifyAutoMerge( + changes: CatalogChange[], + load = (path: string) => readFile(path, "utf8"), +): Promise { + const models = changes.filter((change) => isModel(change.path)); + const created = models.filter((change) => change.status === "created").length; + const updated = models.filter((change) => change.status === "updated").length; + const deleted = models.filter((change) => change.status === "deleted").length; + const reasons: string[] = []; + + if (created > MAX_CREATED_MODELS) reasons.push(`${created} models created (limit ${MAX_CREATED_MODELS})`); + if (deleted > MAX_DELETED_MODELS) reasons.push(`${deleted} models deleted (limit ${MAX_DELETED_MODELS})`); + if (created + deleted > MAX_MODEL_CHURN) { + reasons.push(`${created + deleted} models created or deleted (limit ${MAX_MODEL_CHURN})`); + } + + for (const change of models) { + if (change.status === "deleted" || !isProviderModel(change.path)) continue; + + const model = Bun.TOML.parse(await load(change.path)) as Record; + let reasoning = model.reasoning; + if (reasoning === undefined && typeof model.base_model === "string") { + const base = Bun.TOML.parse(await load(`models/${model.base_model}.toml`)) as Record; + reasoning = base.reasoning; + } + + if (reasoning === true) { + if (!Object.hasOwn(model, "reasoning_options")) { + reasons.push(`${change.path} is a reasoning model without explicit reasoning_options`); + } else if (!REVIEWED_REASONING_PROVIDERS.has(change.path.split("/")[1]!)) { + reasons.push(`${change.path} is a reasoning model that requires manual review`); + } + } + } + + return { safe: reasons.length === 0, created, updated, deleted, reasons }; +} + +export function parseNameStatus(output: string): CatalogChange[] { + return output.trim().split("\n").filter(Boolean).flatMap((line) => { + const [code, ...paths] = line.split("\t"); + const path = paths.at(-1); + if (!code || !path) throw new Error(`Invalid git diff entry: ${line}`); + if (code.startsWith("R")) { + if (paths.length !== 2) throw new Error(`Invalid git rename entry: ${line}`); + return [ + { status: "deleted", path: paths[0]! }, + { status: "created", path: paths[1]! }, + ]; + } + return { + status: code.startsWith("A") ? "created" : code.startsWith("D") ? "deleted" : "updated", + path, + }; + }); +} diff --git a/packages/core/test/auto-merge.test.ts b/packages/core/test/auto-merge.test.ts new file mode 100644 index 000000000..c2d0ae74b --- /dev/null +++ b/packages/core/test/auto-merge.test.ts @@ -0,0 +1,81 @@ +import { expect, test } from "bun:test"; + +import { classifyAutoMerge, parseNameStatus } from "../src/sync/auto-merge.js"; + +const fullModel = (reasoning: boolean, options?: string) => ` +name = "Test" +description = "Test model" +reasoning = ${reasoning} +${options ?? ""} +`; + +test("allows unlimited updates and bounded model churn", async () => { + const changes = Array.from({ length: 30 }, (_, index) => ({ + status: "updated" as const, + path: `providers/test/models/model-${index}.toml`, + })); + const decision = await classifyAutoMerge(changes, async () => fullModel(false)); + + expect(decision.safe).toBe(true); + expect(decision.updated).toBe(30); +}); + +test("requires manual review for bulk additions", async () => { + const changes = Array.from({ length: 11 }, (_, index) => ({ + status: "created" as const, + path: `providers/test/models/model-${index}.toml`, + })); + const decision = await classifyAutoMerge(changes, async () => fullModel(false)); + + expect(decision.safe).toBe(false); + expect(decision.reasons).toContain("11 models created (limit 10)"); +}); + +test("requires manual review for reasoning provider models", async () => { + const withoutOptions = await classifyAutoMerge( + [{ status: "updated", path: "providers/test/models/reasoner.toml" }], + async () => fullModel(true), + ); + const withOptions = await classifyAutoMerge( + [{ status: "updated", path: "providers/test/models/reasoner.toml" }], + async () => fullModel(true, "reasoning_options = []"), + ); + + expect(withoutOptions.safe).toBe(false); + expect(withOptions.safe).toBe(false); +}); + +test("allows reviewed providers with explicit reasoning options", async () => { + const decision = await classifyAutoMerge( + [{ status: "updated", path: "providers/openrouter/models/reasoner.toml" }], + async () => fullModel(true, "reasoning_options = []"), + ); + + expect(decision.safe).toBe(true); +}); + +test("resolves reasoning from base model", async () => { + const decision = await classifyAutoMerge( + [{ status: "created", path: "providers/test/models/reasoner.toml" }], + async (path) => path.startsWith("models/") ? fullModel(true) : 'base_model = "lab/reasoner"\n', + ); + + expect(decision.safe).toBe(false); +}); + +test("parses additions, modifications, and deletions", () => { + expect(parseNameStatus("A\tmodels/a.toml\nM\tmodels/b.toml\nD\tmodels/c.toml\n")) + .toEqual([ + { status: "created", path: "models/a.toml" }, + { status: "updated", path: "models/b.toml" }, + { status: "deleted", path: "models/c.toml" }, + ]); +}); + +test("counts unexpected renames as a deletion and creation", () => { + expect(parseNameStatus("R100\tmodels/old.toml\tmodels/new.toml\n")) + .toEqual([ + { status: "deleted", path: "models/old.toml" }, + { status: "created", path: "models/new.toml" }, + ]); +});