From 8f9db533507817c923cf0bae38a34ea781c503f3 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:40:41 +0100 Subject: [PATCH] feat(supervisor): configurable tolerations for run pods (#4491) ## Summary Self-hosted Kubernetes deployments can now add tolerations to run pods, so runs can schedule onto tainted nodes. Previously the only way to do this was to patch the supervisor. `KUBERNETES_RUNNER_TOLERATIONS` takes a comma separated list of `key=value:effect`, or `key:effect` to tolerate any value. It applies to every run pod, and for runs from a schedule tree it merges with the existing `KUBERNETES_SCHEDULED_RUN_TOLERATIONS`. Left unset, nothing changes: no tolerations are added and the pod spec leaves the field off entirely. The Helm chart takes it as a list: ```yaml supervisor: config: kubernetes: runnerTolerations: - dedicated=runs:NoSchedule - spot:NoExecute ``` ## Naming The issue proposed `KUBERNETES_WORKER_TOLERATIONS`. This ships as `KUBERNETES_RUNNER_TOLERATIONS` instead, because `RUNNER_*` is already the prefix for run pod settings (`RUNNER_HEARTBEAT_INTERVAL_SECONDS`, `RUNNER_ADDITIONAL_ENV_VARS`, and `DOCKER_RUNNER_NETWORKS` for the Docker equivalent), whereas "worker" refers to the supervisor itself throughout this app. ## Validation Keys and values are checked against the Kubernetes naming rules when the supervisor starts, so `dedicated=prod runs:NoSchedule` fails immediately with a message naming the offending entry. Without that check a bad value is accepted at startup and then rejected by the API server on every pod create, which stops all runs with the cause buried in an API error. `KUBERNETES_WORKER_NODETYPE_LABEL` is trimmed and validated for the same reason: surrounding whitespace is not valid in a label value, so a padded value fails every pod create today. ## Node selector off switch `KUBERNETES_WORKER_NODETYPE_LABEL` accepts an empty string to skip the node selector entirely, so runs schedule on any node. This already worked and the Helm chart has always shipped it empty, but it was not documented. It is now. The issue also asked for general node affinity configuration. That is not included: the node selector off switch plus tolerations covers the reported problem, and a free form affinity setting is a much larger config surface to commit to. Fixes #4458 --- .../supervisor-run-pod-tolerations.md | 6 + apps/supervisor/src/env.ts | 65 +-------- apps/supervisor/src/envUtil.test.ts | 127 ++++++++++++++++- apps/supervisor/src/envUtil.ts | 130 ++++++++++++++++++ .../src/workloadManager/kubernetes.test.ts | 42 ++++++ .../src/workloadManager/kubernetes.ts | 28 ++-- .../src/workloadManager/kubernetesPodSpec.ts | 30 ++++ docs/self-hosting/env/supervisor.mdx | 3 +- hosting/k8s/helm/templates/supervisor.yaml | 4 + hosting/k8s/helm/values.yaml | 3 +- 10 files changed, 359 insertions(+), 79 deletions(-) create mode 100644 .server-changes/supervisor-run-pod-tolerations.md diff --git a/.server-changes/supervisor-run-pod-tolerations.md b/.server-changes/supervisor-run-pod-tolerations.md new file mode 100644 index 000000000..885b2b596 --- /dev/null +++ b/.server-changes/supervisor-run-pod-tolerations.md @@ -0,0 +1,6 @@ +--- +area: supervisor +type: feature +--- + +Self-hosted Kubernetes deployments can now add tolerations to run pods, so runs are allowed onto tainted nodes. An invalid toleration now stops the supervisor at startup instead of failing every run pod, so check existing values before upgrading. diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts index 2d96cb1e4..8184004d9 100644 --- a/apps/supervisor/src/env.ts +++ b/apps/supervisor/src/env.ts @@ -1,7 +1,7 @@ import { randomUUID } from "crypto"; import { env as stdEnv } from "std-env"; import { z } from "zod"; -import { AdditionalEnvVars, BoolEnv } from "./envUtil.js"; +import { AdditionalEnvVars, BoolEnv, NodeLabelValue, Tolerations } from "./envUtil.js"; export const Env = z .object({ @@ -173,7 +173,7 @@ export const Env = z // Kubernetes settings KUBERNETES_FORCE_ENABLED: BoolEnv.default(false), KUBERNETES_NAMESPACE: z.string().default("default"), - KUBERNETES_WORKER_NODETYPE_LABEL: z.string().default("v4-worker"), + KUBERNETES_WORKER_NODETYPE_LABEL: NodeLabelValue.default("v4-worker"), KUBERNETES_IMAGE_PULL_SECRETS: z.string().optional(), // csv KUBERNETES_EPHEMERAL_STORAGE_SIZE_LIMIT: z.string().default("10Gi"), KUBERNETES_EPHEMERAL_STORAGE_SIZE_REQUEST: z.string().default("2Gi"), @@ -256,65 +256,8 @@ export const Env = z .max(100) .default(20), - // Schedule toleration settings - scheduled runs tolerate taints on the dedicated pool - // Comma-separated list of tolerations in the format: key=value:effect - // For Exists operator (no value): key:effect - KUBERNETES_SCHEDULED_RUN_TOLERATIONS: z - .string() - .transform((val, ctx) => { - const tolerations = val - .split(",") - .map((entry) => entry.trim()) - .filter((entry) => entry.length > 0) - .map((entry) => { - const colonIdx = entry.lastIndexOf(":"); - if (colonIdx === -1) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: `Invalid toleration format (missing effect): "${entry}"`, - }); - return z.NEVER; - } - - const effect = entry.slice(colonIdx + 1); - const validEffects = ["NoSchedule", "NoExecute", "PreferNoSchedule"]; - if (!validEffects.includes(effect)) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: `Invalid toleration effect "${effect}" in "${entry}". Must be one of: ${validEffects.join( - ", " - )}`, - }); - return z.NEVER; - } - - const keyValue = entry.slice(0, colonIdx); - const eqIdx = keyValue.indexOf("="); - const key = eqIdx === -1 ? keyValue : keyValue.slice(0, eqIdx); - - if (!key) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: `Invalid toleration format (empty key): "${entry}"`, - }); - return z.NEVER; - } - - if (eqIdx === -1) { - return { key, operator: "Exists" as const, effect }; - } - - return { - key, - operator: "Equal" as const, - value: keyValue.slice(eqIdx + 1), - effect, - }; - }); - - return tolerations; - }) - .optional(), + KUBERNETES_RUNNER_TOLERATIONS: Tolerations.optional(), // every run pod + KUBERNETES_SCHEDULED_RUN_TOLERATIONS: Tolerations.optional(), // schedule-tree runs only // Placement tags settings PLACEMENT_TAGS_ENABLED: BoolEnv.default(false), diff --git a/apps/supervisor/src/envUtil.test.ts b/apps/supervisor/src/envUtil.test.ts index c3d35758f..378830f8a 100644 --- a/apps/supervisor/src/envUtil.test.ts +++ b/apps/supervisor/src/envUtil.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest"; -import { BoolEnv, AdditionalEnvVars } from "./envUtil.js"; +import { BoolEnv, AdditionalEnvVars, NodeLabelValue, Tolerations } from "./envUtil.js"; describe("BoolEnv", () => { it("should parse string 'true' as true", () => { @@ -78,3 +78,128 @@ describe("AdditionalEnvVars", () => { }); }); }); + +describe("NodeLabelValue", () => { + it("should keep a clean value untouched", () => { + expect(NodeLabelValue.parse("v4-worker")).toBe("v4-worker"); + }); + + it("should trim surrounding whitespace, which Kubernetes would reject", () => { + expect(NodeLabelValue.parse(" v4-worker ")).toBe("v4-worker"); + expect(NodeLabelValue.parse("\tv4-worker\n")).toBe("v4-worker"); + }); + + it("should treat a whitespace-only value as the empty off-switch", () => { + expect(NodeLabelValue.parse("")).toBe(""); + expect(NodeLabelValue.parse(" ")).toBe(""); + }); + + it("should still apply a default only when unset", () => { + const withDefault = NodeLabelValue.default("v4-worker"); + expect(withDefault.parse(undefined)).toBe("v4-worker"); + expect(withDefault.parse("")).toBe(""); + }); + + it("should reject a value Kubernetes would reject, rather than 422 every pod create", () => { + for (const invalid of ["my worker", "-bad-", "bad.", "a".repeat(64)]) { + expect(NodeLabelValue.safeParse(invalid).success).toBe(false); + } + }); +}); + +describe("Tolerations", () => { + it("should parse key=value entries as Equal", () => { + expect(Tolerations.parse("dedicated=runs:NoSchedule")).toEqual([ + { key: "dedicated", operator: "Equal", value: "runs", effect: "NoSchedule" }, + ]); + }); + + it("should parse entries without a value as Exists", () => { + expect(Tolerations.parse("scheduled-runs:NoExecute")).toEqual([ + { key: "scheduled-runs", operator: "Exists", effect: "NoExecute" }, + ]); + }); + + it("should keep an empty value as an exact match for a valueless taint", () => { + expect(Tolerations.parse("dedicated=:NoSchedule")).toEqual([ + { key: "dedicated", operator: "Equal", value: "", effect: "NoSchedule" }, + ]); + + expect(Tolerations.parse("dedicated:NoSchedule")).toEqual([ + { key: "dedicated", operator: "Exists", effect: "NoSchedule" }, + ]); + }); + + it("should parse an empty string as no tolerations", () => { + expect(Tolerations.parse("")).toEqual([]); + expect(Tolerations.parse(" ")).toEqual([]); + }); + + it("should skip blank entries and trim whitespace", () => { + expect(Tolerations.parse(" a=b:NoSchedule , ,")).toEqual([ + { key: "a", operator: "Equal", value: "b", effect: "NoSchedule" }, + ]); + }); + + it("should reject a missing effect, an unknown effect, and an empty key", () => { + for (const invalid of ["dedicated=runs", "dedicated=runs:Nope", "=runs:NoSchedule"]) { + expect(Tolerations.safeParse(invalid).success).toBe(false); + } + }); + + it("should accept a hyphenated key, a digit-suffixed key, and every effect", () => { + expect( + Tolerations.parse("capacity-1=true:PreferNoSchedule,spot:NoExecute,gpu=a10:NoSchedule") + ).toEqual([ + { key: "capacity-1", operator: "Equal", value: "true", effect: "PreferNoSchedule" }, + { key: "spot", operator: "Exists", effect: "NoExecute" }, + { key: "gpu", operator: "Equal", value: "a10", effect: "NoSchedule" }, + ]); + }); + + it("should accept a DNS-subdomain prefixed key", () => { + expect( + Tolerations.parse("node.cluster.x-k8s.io/machinepool=scheduled-runs:NoSchedule") + ).toEqual([ + { + key: "node.cluster.x-k8s.io/machinepool", + operator: "Equal", + value: "scheduled-runs", + effect: "NoSchedule", + }, + ]); + }); + + it("should reject a key or value that Kubernetes would reject at pod create", () => { + for (const invalid of [ + "dedicated=prod runs:NoSchedule", + "ded icated=runs:NoSchedule", + "dedicated=-runs:NoSchedule", + `dedicated=${"r".repeat(64)}:NoSchedule`, + `${"a".repeat(64)}=runs:NoSchedule`, + `example.com/${"a".repeat(64)}=runs:NoSchedule`, + "a/b/c=runs:NoSchedule", + "Example.com/pool=runs:NoSchedule", + ]) { + expect(Tolerations.safeParse(invalid).success).toBe(false); + } + }); + + it("should bound the prefix and the name separately, as Kubernetes does", () => { + const longestPrefix = `${"a".repeat(63)}.${"b".repeat(63)}.${"c".repeat(63)}.${"d".repeat(61)}`; + expect(longestPrefix.length).toBe(253); + + expect(Tolerations.parse(`${longestPrefix}/${"n".repeat(63)}=runs:NoSchedule`)).toHaveLength(1); + expect(Tolerations.safeParse(`${longestPrefix}a/pool=runs:NoSchedule`).success).toBe(false); + }); + + it("should tolerate whitespace around the separators", () => { + expect(Tolerations.parse("dedicated = runs : NoSchedule")).toEqual([ + { key: "dedicated", operator: "Equal", value: "runs", effect: "NoSchedule" }, + ]); + }); + + it("should reject a stray extra effect instead of folding it into the value", () => { + expect(Tolerations.safeParse("dedicated=runs:NoSchedule:NoExecute").success).toBe(false); + }); +}); diff --git a/apps/supervisor/src/envUtil.ts b/apps/supervisor/src/envUtil.ts index 917f984cc..67811f76f 100644 --- a/apps/supervisor/src/envUtil.ts +++ b/apps/supervisor/src/envUtil.ts @@ -16,6 +16,136 @@ export const BoolEnv = baseBoolEnv as Omit & { default: (value: boolean) => z.ZodDefault; }; +const QUALIFIED_NAME = /^[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$/; +const DNS_SUBDOMAIN = /^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$/; +const LABEL_VALUE = /^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$/; +const QUALIFIED_NAME_MAX = 63; +const DNS_SUBDOMAIN_MAX = 253; +const LABEL_VALUE_MAX = 63; + +/** + * isLabelValue mirrors the Kubernetes label value rules. Empty is valid upstream. + */ +function isLabelValue(value: string): boolean { + return value.length <= LABEL_VALUE_MAX && LABEL_VALUE.test(value); +} + +/** + * isQualifiedName mirrors the Kubernetes qualified name rules used for taint and + * label keys: an optional DNS subdomain prefix before the slash, then the name. + * The two halves have different length limits and different case rules, so a + * single pattern with one overall bound gets both ends wrong. + */ +function isQualifiedName(key: string): boolean { + const slashIdx = key.indexOf("/"); + + if (slashIdx === -1) { + return key.length <= QUALIFIED_NAME_MAX && QUALIFIED_NAME.test(key); + } + + const prefix = key.slice(0, slashIdx); + const name = key.slice(slashIdx + 1); + + return ( + prefix.length <= DNS_SUBDOMAIN_MAX && + DNS_SUBDOMAIN.test(prefix) && + name.length <= QUALIFIED_NAME_MAX && + QUALIFIED_NAME.test(name) + ); +} + +/** + * A node label value. Trimmed because Kubernetes rejects surrounding whitespace + * outright, so a padded value fails every pod create. Deliberately no `min(1)`: + * empty is the off-switch, and the Helm chart ships empty by default. + */ +export const NodeLabelValue = z.string().trim().refine(isLabelValue, { + message: + "Must be a Kubernetes label value: alphanumeric, with dashes, underscores and dots inside, at most 63 characters", +}); + +/** + * Comma-separated pod tolerations in the format `key=value:effect`, or `key:effect` + * for the Exists operator. Keys and values are checked against the Kubernetes + * naming rules here so a typo fails at startup, rather than 422ing every single + * pod create with the cause buried in an API server message. + */ +export const Tolerations = z.string().transform((val, ctx) => { + return val + .split(",") + .map((entry) => entry.trim()) + .filter((entry) => entry.length > 0) + .map((entry) => { + const colonIdx = entry.lastIndexOf(":"); + if (colonIdx === -1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Invalid toleration format (missing effect): "${entry}"`, + }); + return z.NEVER; + } + + const effect = entry.slice(colonIdx + 1).trim(); + const validEffects = ["NoSchedule", "NoExecute", "PreferNoSchedule"]; + if (!validEffects.includes(effect)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Invalid toleration effect "${effect}" in "${entry}". Must be one of: ${validEffects.join( + ", " + )}`, + }); + return z.NEVER; + } + + const keyValue = entry.slice(0, colonIdx); + const eqIdx = keyValue.indexOf("="); + const key = (eqIdx === -1 ? keyValue : keyValue.slice(0, eqIdx)).trim(); + + if (!key) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Invalid toleration format (empty key): "${entry}"`, + }); + return z.NEVER; + } + + if (!isQualifiedName(key)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Invalid toleration key "${key}" in "${entry}". Must be a Kubernetes taint key, optionally prefixed with a DNS subdomain.`, + }); + return z.NEVER; + } + + if (eqIdx === -1) { + return { key, operator: "Exists" as const, effect }; + } + + const value = keyValue.slice(eqIdx + 1).trim(); + if (!value) { + logger.warn( + 'Toleration has an empty value, so it matches only a taint whose value is also empty. Drop the "=" to tolerate any value of this key.', + { entry, key } + ); + } + + if (!isLabelValue(value)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Invalid toleration value "${value}" in "${entry}". Must be a Kubernetes label value: alphanumeric, with dashes, underscores and dots inside.`, + }); + return z.NEVER; + } + + return { + key, + operator: "Equal" as const, + value, + effect, + }; + }); +}); + export const AdditionalEnvVars = z.preprocess((val) => { if (typeof val !== "string") { return val; diff --git a/apps/supervisor/src/workloadManager/kubernetes.test.ts b/apps/supervisor/src/workloadManager/kubernetes.test.ts index 85ad3cbeb..bb15c23e9 100644 --- a/apps/supervisor/src/workloadManager/kubernetes.test.ts +++ b/apps/supervisor/src/workloadManager/kubernetes.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it } from "vitest"; import { BLOCK_IO_URING_SECCOMP_PROFILE, + nodetypeNodeSelector, + runPodTolerations, withBlockIoUringSeccompProfile, } from "./kubernetesPodSpec.js"; @@ -14,6 +16,46 @@ const basePodSpec = { }, }; +describe("nodetypeNodeSelector", () => { + it("omits the nodeSelector entirely when the label is empty or unset", () => { + for (const label of ["", undefined]) { + expect(nodetypeNodeSelector(label)).toEqual({}); + } + }); + + it("pins to nodetype=