fix(sync): preserve comments, skip unavailable stubs, unblock fable-5

The OpenRouter sync round-trips TOML through Bun.TOML.parse and a hand-rolled serializer, so any rewrite silently dropped authored header comments and could degrade a model when OpenRouter served a deprecated route as a stub (negative pricing / empty supported_parameters), flipping capability flags to false and stripping reasoning_options. - Preserve the leading comment block on every model and metadata rewrite - Skip unavailable OpenRouter stubs, retaining the authored file untouched - Remove the fable-5 blacklist now that the route is healthy again; the stub guard covers the outage case that motivated it - Add tests for comment preservation and unavailable-stub handling
2026-07-01 14:57:19 -05:00
parent f3fc692299
commit f07ac11ffe
3 changed files with 161 additions and 13 deletions
@@ -63,7 +63,10 @@ export interface SyncProvider<SourceModel> {
  parseModels(raw: unknown): SourceModel[];
  translateModel(
    model: SourceModel,
-    context: { existing(id: string): ExistingModel | undefined },
+    context: {
+      existing(id: string): ExistingModel | undefined;
+      authored(id: string): ExistingModel | undefined;
+    },
  ): { id: string; model: SyncedModel; metadata?: { id: string; model: SyncedMetadata } } | undefined;
 }

@@ -141,6 +144,9 @@ export async function syncProvider<SourceModel>(
      existing(id) {
        return existing.get(`${id}.toml`)?.toml;
      },
+      authored(id) {
+        return existing.get(`${id}.toml`)?.authored;
+      },
    });
    if (translated === undefined) {
      if (provider.sourceID !== undefined) skippedRemote.push(provider.sourceID(sourceModel));
@@ -205,7 +211,7 @@ export async function syncProvider<SourceModel>(

    desired.set(relativePath, {
      model: parsed.data,
-      content: formatToml(parsed.data),
+      content: (existing.get(relativePath)?.header ?? "") + formatToml(parsed.data),
    });
  }

@@ -216,10 +222,11 @@ export async function syncProvider<SourceModel>(
  for (const [relativePath, file] of desiredMetadata) {
    const filePath = path.join(metadataDir, relativePath);
    const currentFile = Bun.file(filePath);
-    const current = await currentFile.exists()
+    const currentText = await currentFile.exists() ? await currentFile.text() : undefined;
+    const current = currentText !== undefined
      ? ModelMetadata.safeParse({
          id: relativePath.slice(0, -5),
-          ...Bun.TOML.parse(await currentFile.text()) as Record<string, unknown>,
+          ...Bun.TOML.parse(currentText) as Record<string, unknown>,
        })
      : undefined;
    if (current?.success && stable(current.data) === stable(file.model)) continue;
@@ -228,7 +235,7 @@ export async function syncProvider<SourceModel>(
      console.log(`Would ${current === undefined ? "create" : "update"} metadata ${relativePath}`);
    } else {
      await mkdir(path.dirname(filePath), { recursive: true });
-      await Bun.write(filePath, file.content);
+      await Bun.write(filePath, (currentText !== undefined ? leadingComments(currentText) : "") + file.content);
    }
  }

@@ -394,6 +401,7 @@ async function readExisting(modelsDir: string) {
  const existing = new Map<string, {
    authored: ExistingModel;
    toml: ExistingModel;
+    header: string;
    symlink: boolean;
  }>();
  const brokenSymlinks = new Set<string>();
@@ -425,7 +433,7 @@ async function readExisting(modelsDir: string) {
      ? authored
      : resolveBaseModel(authored, modelMetadata ?? {}, filePath);

-    existing.set(file, { authored, toml, symlink });
+    existing.set(file, { authored, toml, header: leadingComments(text), symlink });
  }

  return { models: existing, brokenSymlinks, modelMetadata };
@@ -675,6 +683,23 @@ function quote(value: string) {
  return `"${value.replaceAll("\\", "\\\\").replaceAll('"', '\\"')}"`;
 }

+// Preserve the leading comment block (header) authored at the top of a TOML file.
+// `Bun.TOML.parse` discards comments, so the serializer must re-attach them or
+// every rewrite would silently delete hand-authored documentation.
+function leadingComments(text: string) {
+  const header: string[] = [];
+  for (const line of text.split("\n")) {
+    const trimmed = line.trim();
+    if (trimmed === "" || trimmed.startsWith("#")) {
+      header.push(line);
+    } else {
+      break;
+    }
+  }
+  while (header.length > 0 && header[header.length - 1]?.trim() === "") header.pop();
+  return header.length > 0 ? `${header.join("\n")}\n` : "";
+}
+
 function formatInteger(n: number) {
  return String(n).replace(/\B(?=(\d{3})+(?!\d))/g, "_");
 }
@@ -7,7 +7,6 @@ import type { ExistingModel, SyncProvider, SyncedFullModel, SyncedModel } from "

 const API_ENDPOINT = "https://openrouter.ai/api/v1/models";
 const MODELS_DIR = path.join(import.meta.dirname, "..", "..", "..", "..", "..", "models");
-const MODEL_NAME_BLACKLIST = ["fable-5"];
 const modelMetadataByID = new Map<string, Record<string, unknown>>();
 const modelMetadataFilesByProvider = new Map<string, Set<string>>();

@@ -91,12 +90,18 @@ export const openrouter = {
    return response.json();
  },
  parseModels(raw) {
-    return OpenRouterResponse.parse(raw).data.filter((model) => {
-      const name = `${model.id} ${model.name}`.toLowerCase();
-      return MODEL_NAME_BLACKLIST.every((value) => !name.includes(value));
-    });
+    return OpenRouterResponse.parse(raw).data;
  },
  translateModel(model, context) {
+    // OpenRouter serves deprecated/unavailable routes as degraded stubs:
+    // negative pricing (`"-1"`) and an empty `supported_parameters` array. Syncing
+    // those would wrongly flip `reasoning`/`tool_call`/`structured_output` to false
+    // and strip `reasoning_options`. Leave the authored file untouched instead, and
+    // skip the model entirely when we have nothing to preserve.
+    if (isUnavailable(model)) {
+      const authored = context.authored(model.id);
+      return authored === undefined ? undefined : { id: model.id, model: authored as SyncedModel };
+    }
    return {
      id: model.id,
      model: buildOpenRouterModel(model, context.existing(model.id)),
@@ -104,6 +109,14 @@ export const openrouter = {
  },
 } satisfies SyncProvider<OpenRouterModel>;

+function isUnavailable(model: OpenRouterModel) {
+  return (
+    model.supported_parameters.length === 0 ||
+    Number(model.pricing.prompt) < 0 ||
+    Number(model.pricing.completion) < 0
+  );
+}
+
 function dateFromTimestamp(timestamp: number) {
  return new Date(timestamp * 1000).toISOString().slice(0, 10);
 }
@@ -1,7 +1,10 @@
 import { expect, test } from "bun:test";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";

-import { formatToml, preserveReasoningOptions } from "../src/sync/index.js";
-import { buildOpenRouterModel, type OpenRouterModel } from "../src/sync/providers/openrouter.js";
+import { formatToml, preserveReasoningOptions, syncProvider, type SyncProvider } from "../src/sync/index.js";
+import { buildOpenRouterModel, openrouter, type OpenRouterModel } from "../src/sync/providers/openrouter.js";

 test("formats interleaved as a root field before reasoning option tables", () => {
  const content = formatToml({
@@ -148,6 +151,113 @@ test("upgrades empty OpenRouter reasoning options from model metadata", () => {
  });
 });

+test("preserves the authored header comment block when rewriting a changed model", async () => {
+  const dir = await mkdtemp(path.join(tmpdir(), "sync-header-"));
+  const modelsDir = path.join(dir, "providers", "example", "models");
+  await Bun.write(path.join(modelsDir, "example-model.toml"), [
+    "# Documented quirk: this route needs a manual note.",
+    "# https://example.com/docs (accessed 2026-06-25)",
+    'name = "Example Model"',
+    'release_date = "2026-01-01"',
+    'last_updated = "2026-01-01"',
+    "attachment = false",
+    "reasoning = false",
+    "tool_call = true",
+    "open_weights = false",
+    "",
+    "[cost]",
+    "input = 1",
+    "output = 2",
+    "",
+    "[limit]",
+    "context = 1_000",
+    "output = 100",
+    "",
+    "[modalities]",
+    'input = ["text"]',
+    'output = ["text"]',
+    "",
+  ].join("\n"));
+
+  const provider: SyncProvider<{ id: string }> = {
+    id: "example",
+    name: "Example",
+    modelsDir,
+    deleteMissing: false,
+    async fetchModels() {
+      return [{ id: "example-model" }];
+    },
+    parseModels(raw) {
+      return raw as { id: string }[];
+    },
+    translateModel(model) {
+      return {
+        id: model.id,
+        model: {
+          name: "Example Model",
+          release_date: "2026-01-01",
+          last_updated: "2026-01-01",
+          attachment: false,
+          reasoning: false,
+          tool_call: true,
+          open_weights: false,
+          cost: { input: 3, output: 9 },
+          limit: { context: 1_000, output: 100 },
+          modalities: { input: ["text"], output: ["text"] },
+        },
+      };
+    },
+  };
+
+  try {
+    const result = await syncProvider(provider);
+    expect(result.updated).toBe(1);
+    const written = await readFile(path.join(modelsDir, "example-model.toml"), "utf8");
+    expect(written).toStartWith(
+      "# Documented quirk: this route needs a manual note.\n# https://example.com/docs (accessed 2026-06-25)\n",
+    );
+    expect(written).toContain("input = 3");
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("retains authored data when OpenRouter reports an unavailable stub", () => {
+  const authored = {
+    name: "Claude Fable Latest",
+    reasoning: true as const,
+    reasoning_options: [{ type: "effort" as const, values: ["low", "high"] as const }],
+    tool_call: true as const,
+    structured_output: true as const,
+  };
+  const translated = openrouter.translateModel(unavailableStub(), {
+    existing: () => undefined,
+    authored: () => authored as never,
+  });
+
+  expect(translated).toEqual({ id: "~anthropic/claude-fable-latest", model: authored as never });
+});
+
+test("skips an unavailable OpenRouter stub with no authored file", () => {
+  const translated = openrouter.translateModel(unavailableStub(), {
+    existing: () => undefined,
+    authored: () => undefined,
+  });
+
+  expect(translated).toBeUndefined();
+});
+
+function unavailableStub(): OpenRouterModel {
+  return openRouterModel({
+    id: "~anthropic/claude-fable-latest",
+    name: "Anthropic: Claude Fable Latest",
+    supported_parameters: [],
+    pricing: { prompt: "-1", completion: "-1" },
+    reasoning: { mandatory: true },
+    top_provider: { context_length: null, max_completion_tokens: null },
+  });
+}
+
 function openRouterModel(overrides: Partial<OpenRouterModel> = {}): OpenRouterModel {
  return {
    id: "anthropic/claude-sonnet-5",