chore(sync): allow CrossModel reasoning auto-merge (#5078)
Co-authored-by: rekram1-node <rekram1-node@users.noreply.github.com>
This commit is contained in:
committed by
GitHub
parent
57a8362627
commit
734d20ea43
@@ -5,6 +5,7 @@ export const MAX_CREATED_MODELS = 10;
|
||||
export const MAX_DELETED_MODELS = 10;
|
||||
export const MAX_MODEL_CHURN = 15;
|
||||
const REVIEWED_REASONING_PROVIDERS = new Set([
|
||||
"crossmodel",
|
||||
"edenai",
|
||||
"empiriolabs",
|
||||
"hyper",
|
||||
|
||||
@@ -22,11 +22,13 @@ function baseModelExists(modelID: string): boolean {
|
||||
// CROSSMODEL_MODELS_URL overrides the endpoint (e.g. a local backend) for testing.
|
||||
const API_ENDPOINT = process.env.CROSSMODEL_MODELS_URL ?? "https://www.crossmodel.ai/api/models";
|
||||
|
||||
const REASONING_EFFORTS = ["none", "minimal", "low", "medium", "high", "xhigh", "max", "default"] as const;
|
||||
|
||||
const ReasoningCapability = z
|
||||
.object({
|
||||
supported: z.boolean().optional(),
|
||||
toggle: z.boolean().nullish().transform((value) => value ?? undefined),
|
||||
effort: z.array(z.string()).nullish().transform((value) => value ?? undefined),
|
||||
effort: z.array(z.enum(REASONING_EFFORTS)).nullish().transform((value) => value ?? undefined),
|
||||
budget_tokens: z
|
||||
.object({ min: z.number().optional(), max: z.number().optional() })
|
||||
.nullish()
|
||||
@@ -160,15 +162,6 @@ function modalities(values: string[] | undefined, fallback: Modality[]): Modalit
|
||||
return [...new Set(result.length > 0 ? result : fallback)];
|
||||
}
|
||||
|
||||
// models.dev's reasoning_options effort enum (schema.ts ReasoningEffortValue).
|
||||
// Guarding against it means an unexpected upstream value is dropped instead of
|
||||
// silently producing a TOML that fails `validate`.
|
||||
const REASONING_EFFORTS = ["none", "minimal", "low", "medium", "high", "xhigh", "max", "default"] as const;
|
||||
type ReasoningEffort = (typeof REASONING_EFFORTS)[number];
|
||||
function isReasoningEffort(value: string): value is ReasoningEffort {
|
||||
return (REASONING_EFFORTS as readonly string[]).includes(value);
|
||||
}
|
||||
|
||||
// Project CrossModel's capabilities.reasoning onto models.dev reasoning_options.
|
||||
// reasoning absent -> undefined (non-reasoning model; option omitted)
|
||||
// reasoning === {} -> [] (model reasons, no verified user-selectable control)
|
||||
@@ -179,8 +172,7 @@ function reasoningOptions(model: CrossModelModel): SyncedModel["reasoning_option
|
||||
const options: NonNullable<SyncedModel["reasoning_options"]> = [];
|
||||
if (reasoning.toggle === true) options.push({ type: "toggle" });
|
||||
if (reasoning.effort !== undefined) {
|
||||
const values = reasoning.effort.filter(isReasoningEffort);
|
||||
if (values.length > 0) options.push({ type: "effort", values });
|
||||
if (reasoning.effort.length > 0) options.push({ type: "effort", values: reasoning.effort });
|
||||
}
|
||||
if (reasoning.budget_tokens !== undefined) {
|
||||
const budget: { type: "budget_tokens"; min?: number; max?: number } = { type: "budget_tokens" };
|
||||
|
||||
@@ -82,7 +82,7 @@ test("does not inspect deleted models", async () => {
|
||||
});
|
||||
|
||||
test("allows reviewed providers with explicit reasoning options", async () => {
|
||||
for (const provider of ["edenai", "empiriolabs", "hyper", "kilo", "llmgateway", "merge-gateway", "nano-gpt", "openrouter", "venice"]) {
|
||||
for (const provider of ["crossmodel", "edenai", "empiriolabs", "hyper", "kilo", "llmgateway", "merge-gateway", "nano-gpt", "openrouter", "venice"]) {
|
||||
const decision = await classifyAutoMerge(
|
||||
[{ status: "updated", path: `providers/${provider}/models/reasoner.toml` }],
|
||||
async () => fullModel(true, 'reasoning_options = [{ type = "toggle" }]'),
|
||||
|
||||
@@ -467,6 +467,46 @@ test("parses CrossModel's nullable reasoning controls", () => {
|
||||
});
|
||||
});
|
||||
|
||||
test("syncs CrossModel's explicit reasoning controls", () => {
|
||||
const model = buildCrossModel(
|
||||
crossModelModel({
|
||||
capabilities: {
|
||||
json: true,
|
||||
reasoning: {
|
||||
supported: true,
|
||||
toggle: true,
|
||||
effort: ["low", "high", "max"],
|
||||
budget_tokens: { min: 1_024, max: 32_000 },
|
||||
},
|
||||
},
|
||||
}),
|
||||
undefined,
|
||||
);
|
||||
|
||||
expect(model).toMatchObject({
|
||||
reasoning_options: [
|
||||
{ type: "toggle" },
|
||||
{ type: "effort", values: ["low", "high", "max"] },
|
||||
{ type: "budget_tokens", min: 1_024, max: 32_000 },
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
test("rejects unknown CrossModel reasoning efforts", () => {
|
||||
expect(() =>
|
||||
CrossModelResponse.parse({
|
||||
data: [
|
||||
{
|
||||
...crossModelModel(),
|
||||
capabilities: {
|
||||
reasoning: { supported: true, effort: ["unexpected"] },
|
||||
},
|
||||
},
|
||||
],
|
||||
})
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
test("syncs NanoGPT's verified reasoning, pricing, limits, and open-weight metadata", () => {
|
||||
const model = buildNanoGptModel(nanoGptModel({
|
||||
pricing: {
|
||||
|
||||
Reference in New Issue
Block a user