feat(supervisor): configurable tolerations for run pods (#4491)

## Summary

Self-hosted Kubernetes deployments can now add tolerations to run pods,
so runs
can schedule onto tainted nodes. Previously the only way to do this was
to patch
the supervisor.

`KUBERNETES_RUNNER_TOLERATIONS` takes a comma separated list of
`key=value:effect`, or `key:effect` to tolerate any value. It applies to
every
run pod, and for runs from a schedule tree it merges with the existing
`KUBERNETES_SCHEDULED_RUN_TOLERATIONS`. Left unset, nothing changes: no
tolerations are added and the pod spec leaves the field off entirely.

The Helm chart takes it as a list:

```yaml
supervisor:
  config:
    kubernetes:
      runnerTolerations:
        - dedicated=runs:NoSchedule
        - spot:NoExecute
```

## Naming

The issue proposed `KUBERNETES_WORKER_TOLERATIONS`. This ships as
`KUBERNETES_RUNNER_TOLERATIONS` instead, because `RUNNER_*` is already
the prefix
for run pod settings (`RUNNER_HEARTBEAT_INTERVAL_SECONDS`,
`RUNNER_ADDITIONAL_ENV_VARS`, and `DOCKER_RUNNER_NETWORKS` for the
Docker
equivalent), whereas "worker" refers to the supervisor itself throughout
this app.

## Validation

Keys and values are checked against the Kubernetes naming rules when the
supervisor starts, so `dedicated=prod runs:NoSchedule` fails immediately
with a
message naming the offending entry. Without that check a bad value is
accepted at
startup and then rejected by the API server on every pod create, which
stops all
runs with the cause buried in an API error.
`KUBERNETES_WORKER_NODETYPE_LABEL` is
trimmed and validated for the same reason: surrounding whitespace is not
valid in
a label value, so a padded value fails every pod create today.

## Node selector off switch

`KUBERNETES_WORKER_NODETYPE_LABEL` accepts an empty string to skip the
node
selector entirely, so runs schedule on any node. This already worked and
the Helm
chart has always shipped it empty, but it was not documented. It is now.

The issue also asked for general node affinity configuration. That is
not
included: the node selector off switch plus tolerations covers the
reported
problem, and a free form affinity setting is a much larger config
surface to
commit to.

Fixes #4458

.server-changes/supervisor-run-pod-tolerations.md

@@ -0,0 +1,6 @@
+---
+area: supervisor
+type: feature
+---
+
+Self-hosted Kubernetes deployments can now add tolerations to run pods, so runs are allowed onto tainted nodes. An invalid toleration now stops the supervisor at startup instead of failing every run pod, so check existing values before upgrading.

apps/supervisor/src/env.ts

@@ -1,7 +1,7 @@
 import { randomUUID } from "crypto";
 import { env as stdEnv } from "std-env";
 import { z } from "zod";
-import { AdditionalEnvVars, BoolEnv } from "./envUtil.js";
+import { AdditionalEnvVars, BoolEnv, NodeLabelValue, Tolerations } from "./envUtil.js";
 
 export const Env = z
   .object({
@@ -173,7 +173,7 @@ export const Env = z
     // Kubernetes settings
     KUBERNETES_FORCE_ENABLED: BoolEnv.default(false),
     KUBERNETES_NAMESPACE: z.string().default("default"),
-    KUBERNETES_WORKER_NODETYPE_LABEL: z.string().default("v4-worker"),
+    KUBERNETES_WORKER_NODETYPE_LABEL: NodeLabelValue.default("v4-worker"),
     KUBERNETES_IMAGE_PULL_SECRETS: z.string().optional(), // csv
     KUBERNETES_EPHEMERAL_STORAGE_SIZE_LIMIT: z.string().default("10Gi"),
     KUBERNETES_EPHEMERAL_STORAGE_SIZE_REQUEST: z.string().default("2Gi"),
@@ -256,65 +256,8 @@ export const Env = z
       .max(100)
       .default(20),
 
-    // Schedule toleration settings - scheduled runs tolerate taints on the dedicated pool
-    // Comma-separated list of tolerations in the format: key=value:effect
-    // For Exists operator (no value): key:effect
-    KUBERNETES_SCHEDULED_RUN_TOLERATIONS: z
-      .string()
-      .transform((val, ctx) => {
-        const tolerations = val
-          .split(",")
-          .map((entry) => entry.trim())
-          .filter((entry) => entry.length > 0)
-          .map((entry) => {
-            const colonIdx = entry.lastIndexOf(":");
-            if (colonIdx === -1) {
-              ctx.addIssue({
-                code: z.ZodIssueCode.custom,
-                message: `Invalid toleration format (missing effect): "${entry}"`,
-              });
-              return z.NEVER;
-            }
-
-            const effect = entry.slice(colonIdx + 1);
-            const validEffects = ["NoSchedule", "NoExecute", "PreferNoSchedule"];
-            if (!validEffects.includes(effect)) {
-              ctx.addIssue({
-                code: z.ZodIssueCode.custom,
-                message: `Invalid toleration effect "${effect}" in "${entry}". Must be one of: ${validEffects.join(
-                  ", "
-                )}`,
-              });
-              return z.NEVER;
-            }
-
-            const keyValue = entry.slice(0, colonIdx);
-            const eqIdx = keyValue.indexOf("=");
-            const key = eqIdx === -1 ? keyValue : keyValue.slice(0, eqIdx);
-
-            if (!key) {
-              ctx.addIssue({
-                code: z.ZodIssueCode.custom,
-                message: `Invalid toleration format (empty key): "${entry}"`,
-              });
-              return z.NEVER;
-            }
-
-            if (eqIdx === -1) {
-              return { key, operator: "Exists" as const, effect };
-            }
-
-            return {
-              key,
-              operator: "Equal" as const,
-              value: keyValue.slice(eqIdx + 1),
-              effect,
-            };
-          });
-
-        return tolerations;
-      })
-      .optional(),
+    KUBERNETES_RUNNER_TOLERATIONS: Tolerations.optional(), // every run pod
+    KUBERNETES_SCHEDULED_RUN_TOLERATIONS: Tolerations.optional(), // schedule-tree runs only
 
     // Placement tags settings
     PLACEMENT_TAGS_ENABLED: BoolEnv.default(false),

apps/supervisor/src/envUtil.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect } from "vitest";
-import { BoolEnv, AdditionalEnvVars } from "./envUtil.js";
+import { BoolEnv, AdditionalEnvVars, NodeLabelValue, Tolerations } from "./envUtil.js";
 
 describe("BoolEnv", () => {
   it("should parse string 'true' as true", () => {
@@ -78,3 +78,128 @@ describe("AdditionalEnvVars", () => {
     });
   });
 });
+
+describe("NodeLabelValue", () => {
+  it("should keep a clean value untouched", () => {
+    expect(NodeLabelValue.parse("v4-worker")).toBe("v4-worker");
+  });
+
+  it("should trim surrounding whitespace, which Kubernetes would reject", () => {
+    expect(NodeLabelValue.parse(" v4-worker ")).toBe("v4-worker");
+    expect(NodeLabelValue.parse("\tv4-worker\n")).toBe("v4-worker");
+  });
+
+  it("should treat a whitespace-only value as the empty off-switch", () => {
+    expect(NodeLabelValue.parse("")).toBe("");
+    expect(NodeLabelValue.parse("   ")).toBe("");
+  });
+
+  it("should still apply a default only when unset", () => {
+    const withDefault = NodeLabelValue.default("v4-worker");
+    expect(withDefault.parse(undefined)).toBe("v4-worker");
+    expect(withDefault.parse("")).toBe("");
+  });
+
+  it("should reject a value Kubernetes would reject, rather than 422 every pod create", () => {
+    for (const invalid of ["my worker", "-bad-", "bad.", "a".repeat(64)]) {
+      expect(NodeLabelValue.safeParse(invalid).success).toBe(false);
+    }
+  });
+});
+
+describe("Tolerations", () => {
+  it("should parse key=value entries as Equal", () => {
+    expect(Tolerations.parse("dedicated=runs:NoSchedule")).toEqual([
+      { key: "dedicated", operator: "Equal", value: "runs", effect: "NoSchedule" },
+    ]);
+  });
+
+  it("should parse entries without a value as Exists", () => {
+    expect(Tolerations.parse("scheduled-runs:NoExecute")).toEqual([
+      { key: "scheduled-runs", operator: "Exists", effect: "NoExecute" },
+    ]);
+  });
+
+  it("should keep an empty value as an exact match for a valueless taint", () => {
+    expect(Tolerations.parse("dedicated=:NoSchedule")).toEqual([
+      { key: "dedicated", operator: "Equal", value: "", effect: "NoSchedule" },
+    ]);
+
+    expect(Tolerations.parse("dedicated:NoSchedule")).toEqual([
+      { key: "dedicated", operator: "Exists", effect: "NoSchedule" },
+    ]);
+  });
+
+  it("should parse an empty string as no tolerations", () => {
+    expect(Tolerations.parse("")).toEqual([]);
+    expect(Tolerations.parse("  ")).toEqual([]);
+  });
+
+  it("should skip blank entries and trim whitespace", () => {
+    expect(Tolerations.parse(" a=b:NoSchedule , ,")).toEqual([
+      { key: "a", operator: "Equal", value: "b", effect: "NoSchedule" },
+    ]);
+  });
+
+  it("should reject a missing effect, an unknown effect, and an empty key", () => {
+    for (const invalid of ["dedicated=runs", "dedicated=runs:Nope", "=runs:NoSchedule"]) {
+      expect(Tolerations.safeParse(invalid).success).toBe(false);
+    }
+  });
+
+  it("should accept a hyphenated key, a digit-suffixed key, and every effect", () => {
+    expect(
+      Tolerations.parse("capacity-1=true:PreferNoSchedule,spot:NoExecute,gpu=a10:NoSchedule")
+    ).toEqual([
+      { key: "capacity-1", operator: "Equal", value: "true", effect: "PreferNoSchedule" },
+      { key: "spot", operator: "Exists", effect: "NoExecute" },
+      { key: "gpu", operator: "Equal", value: "a10", effect: "NoSchedule" },
+    ]);
+  });
+
+  it("should accept a DNS-subdomain prefixed key", () => {
+    expect(
+      Tolerations.parse("node.cluster.x-k8s.io/machinepool=scheduled-runs:NoSchedule")
+    ).toEqual([
+      {
+        key: "node.cluster.x-k8s.io/machinepool",
+        operator: "Equal",
+        value: "scheduled-runs",
+        effect: "NoSchedule",
+      },
+    ]);
+  });
+
+  it("should reject a key or value that Kubernetes would reject at pod create", () => {
+    for (const invalid of [
+      "dedicated=prod runs:NoSchedule",
+      "ded icated=runs:NoSchedule",
+      "dedicated=-runs:NoSchedule",
+      `dedicated=${"r".repeat(64)}:NoSchedule`,
+      `${"a".repeat(64)}=runs:NoSchedule`,
+      `example.com/${"a".repeat(64)}=runs:NoSchedule`,
+      "a/b/c=runs:NoSchedule",
+      "Example.com/pool=runs:NoSchedule",
+    ]) {
+      expect(Tolerations.safeParse(invalid).success).toBe(false);
+    }
+  });
+
+  it("should bound the prefix and the name separately, as Kubernetes does", () => {
+    const longestPrefix = `${"a".repeat(63)}.${"b".repeat(63)}.${"c".repeat(63)}.${"d".repeat(61)}`;
+    expect(longestPrefix.length).toBe(253);
+
+    expect(Tolerations.parse(`${longestPrefix}/${"n".repeat(63)}=runs:NoSchedule`)).toHaveLength(1);
+    expect(Tolerations.safeParse(`${longestPrefix}a/pool=runs:NoSchedule`).success).toBe(false);
+  });
+
+  it("should tolerate whitespace around the separators", () => {
+    expect(Tolerations.parse("dedicated = runs : NoSchedule")).toEqual([
+      { key: "dedicated", operator: "Equal", value: "runs", effect: "NoSchedule" },
+    ]);
+  });
+
+  it("should reject a stray extra effect instead of folding it into the value", () => {
+    expect(Tolerations.safeParse("dedicated=runs:NoSchedule:NoExecute").success).toBe(false);
+  });
+});

apps/supervisor/src/envUtil.ts

@@ -16,6 +16,136 @@ export const BoolEnv = baseBoolEnv as Omit<typeof baseBoolEnv, "default"> & {
   default: (value: boolean) => z.ZodDefault<typeof baseBoolEnv>;
 };
 
+const QUALIFIED_NAME = /^[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$/;
+const DNS_SUBDOMAIN = /^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$/;
+const LABEL_VALUE = /^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$/;
+const QUALIFIED_NAME_MAX = 63;
+const DNS_SUBDOMAIN_MAX = 253;
+const LABEL_VALUE_MAX = 63;
+
+/**
+ * isLabelValue mirrors the Kubernetes label value rules. Empty is valid upstream.
+ */
+function isLabelValue(value: string): boolean {
+  return value.length <= LABEL_VALUE_MAX && LABEL_VALUE.test(value);
+}
+
+/**
+ * isQualifiedName mirrors the Kubernetes qualified name rules used for taint and
+ * label keys: an optional DNS subdomain prefix before the slash, then the name.
+ * The two halves have different length limits and different case rules, so a
+ * single pattern with one overall bound gets both ends wrong.
+ */
+function isQualifiedName(key: string): boolean {
+  const slashIdx = key.indexOf("/");
+
+  if (slashIdx === -1) {
+    return key.length <= QUALIFIED_NAME_MAX && QUALIFIED_NAME.test(key);
+  }
+
+  const prefix = key.slice(0, slashIdx);
+  const name = key.slice(slashIdx + 1);
+
+  return (
+    prefix.length <= DNS_SUBDOMAIN_MAX &&
+    DNS_SUBDOMAIN.test(prefix) &&
+    name.length <= QUALIFIED_NAME_MAX &&
+    QUALIFIED_NAME.test(name)
+  );
+}
+
+/**
+ * A node label value. Trimmed because Kubernetes rejects surrounding whitespace
+ * outright, so a padded value fails every pod create. Deliberately no `min(1)`:
+ * empty is the off-switch, and the Helm chart ships empty by default.
+ */
+export const NodeLabelValue = z.string().trim().refine(isLabelValue, {
+  message:
+    "Must be a Kubernetes label value: alphanumeric, with dashes, underscores and dots inside, at most 63 characters",
+});
+
+/**
+ * Comma-separated pod tolerations in the format `key=value:effect`, or `key:effect`
+ * for the Exists operator. Keys and values are checked against the Kubernetes
+ * naming rules here so a typo fails at startup, rather than 422ing every single
+ * pod create with the cause buried in an API server message.
+ */
+export const Tolerations = z.string().transform((val, ctx) => {
+  return val
+    .split(",")
+    .map((entry) => entry.trim())
+    .filter((entry) => entry.length > 0)
+    .map((entry) => {
+      const colonIdx = entry.lastIndexOf(":");
+      if (colonIdx === -1) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Invalid toleration format (missing effect): "${entry}"`,
+        });
+        return z.NEVER;
+      }
+
+      const effect = entry.slice(colonIdx + 1).trim();
+      const validEffects = ["NoSchedule", "NoExecute", "PreferNoSchedule"];
+      if (!validEffects.includes(effect)) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Invalid toleration effect "${effect}" in "${entry}". Must be one of: ${validEffects.join(
+            ", "
+          )}`,
+        });
+        return z.NEVER;
+      }
+
+      const keyValue = entry.slice(0, colonIdx);
+      const eqIdx = keyValue.indexOf("=");
+      const key = (eqIdx === -1 ? keyValue : keyValue.slice(0, eqIdx)).trim();
+
+      if (!key) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Invalid toleration format (empty key): "${entry}"`,
+        });
+        return z.NEVER;
+      }
+
+      if (!isQualifiedName(key)) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Invalid toleration key "${key}" in "${entry}". Must be a Kubernetes taint key, optionally prefixed with a DNS subdomain.`,
+        });
+        return z.NEVER;
+      }
+
+      if (eqIdx === -1) {
+        return { key, operator: "Exists" as const, effect };
+      }
+
+      const value = keyValue.slice(eqIdx + 1).trim();
+      if (!value) {
+        logger.warn(
+          'Toleration has an empty value, so it matches only a taint whose value is also empty. Drop the "=" to tolerate any value of this key.',
+          { entry, key }
+        );
+      }
+
+      if (!isLabelValue(value)) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Invalid toleration value "${value}" in "${entry}". Must be a Kubernetes label value: alphanumeric, with dashes, underscores and dots inside.`,
+        });
+        return z.NEVER;
+      }
+
+      return {
+        key,
+        operator: "Equal" as const,
+        value,
+        effect,
+      };
+    });
+});
+
 export const AdditionalEnvVars = z.preprocess((val) => {
   if (typeof val !== "string") {
     return val;

apps/supervisor/src/workloadManager/kubernetes.test.ts

@@ -1,6 +1,8 @@
 import { describe, expect, it } from "vitest";
 import {
   BLOCK_IO_URING_SECCOMP_PROFILE,
+  nodetypeNodeSelector,
+  runPodTolerations,
   withBlockIoUringSeccompProfile,
 } from "./kubernetesPodSpec.js";
 
@@ -14,6 +16,46 @@ const basePodSpec = {
   },
 };
 
+describe("nodetypeNodeSelector", () => {
+  it("omits the nodeSelector entirely when the label is empty or unset", () => {
+    for (const label of ["", undefined]) {
+      expect(nodetypeNodeSelector(label)).toEqual({});
+    }
+  });
+
+  it("pins to nodetype=<label> when set", () => {
+    expect(nodetypeNodeSelector("v4-worker")).toEqual({ nodeSelector: { nodetype: "v4-worker" } });
+  });
+});
+
+describe("runPodTolerations", () => {
+  const worker = [{ key: "dedicated", operator: "Equal", value: "runs", effect: "NoSchedule" }];
+  const scheduled = [{ key: "scheduled-runs", operator: "Exists", effect: "NoSchedule" }];
+
+  it("leaves tolerations unset when neither is configured", () => {
+    expect(runPodTolerations(undefined, undefined, false)).toBeUndefined();
+    expect(runPodTolerations(undefined, undefined, true)).toBeUndefined();
+    expect(runPodTolerations([], [], true)).toBeUndefined();
+  });
+
+  it("applies the worker tolerations to every run", () => {
+    expect(runPodTolerations(worker, undefined, false)).toEqual(worker);
+    expect(runPodTolerations(worker, undefined, true)).toEqual(worker);
+  });
+
+  it("applies the scheduled-run tolerations on their own, as before this option existed", () => {
+    expect(runPodTolerations(undefined, scheduled, true)).toEqual(scheduled);
+    expect(runPodTolerations(undefined, scheduled, false)).toBeUndefined();
+    expect(runPodTolerations([], scheduled, true)).toEqual(scheduled);
+  });
+
+  it("adds the scheduled-run tolerations only for scheduled runs", () => {
+    expect(runPodTolerations(worker, scheduled, false)).toEqual(worker);
+    expect(runPodTolerations(worker, [], true)).toEqual(worker);
+    expect(runPodTolerations(worker, scheduled, true)).toEqual([...worker, ...scheduled]);
+  });
+});
+
 describe("withBlockIoUringSeccompProfile", () => {
   it("adds the Localhost io_uring profile for node-24 and above, preserving pod security defaults", () => {
     for (const runtime of ["node-24", "node-26", "node-30", "experimental-node-24"]) {

apps/supervisor/src/workloadManager/kubernetes.ts

@@ -14,7 +14,11 @@ import { PlacementTagProcessor } from "@trigger.dev/core/v3/serverOnly";
 import { env } from "../env.js";
 import { type K8sApi, createK8sApi, type k8s } from "../clients/kubernetes.js";
 import { getRunnerId } from "../util.js";
-import { withBlockIoUringSeccompProfile } from "./kubernetesPodSpec.js";
+import {
+  nodetypeNodeSelector,
+  runPodTolerations,
+  withBlockIoUringSeccompProfile,
+} from "./kubernetesPodSpec.js";
 
 type ResourceQuantities = {
   [K in "cpu" | "memory" | "ephemeral-storage"]?: string;
@@ -127,7 +131,7 @@ export class KubernetesWorkloadManager implements WorkloadManager {
           spec: {
             ...podSpec,
             affinity: this.#getAffinity(opts),
-            tolerations: this.#getScheduleTolerations(this.#isScheduledRun(opts)),
+            tolerations: this.#getTolerations(this.#isScheduledRun(opts)),
             terminationGracePeriodSeconds: 60 * 60,
             containers: [
               {
@@ -329,13 +333,7 @@ export class KubernetesWorkloadManager implements WorkloadManager {
             schedulerName: env.KUBERNETES_SCHEDULER_NAME,
           }
         : {}),
-      ...(env.KUBERNETES_WORKER_NODETYPE_LABEL
-        ? {
-            nodeSelector: {
-              nodetype: env.KUBERNETES_WORKER_NODETYPE_LABEL,
-            },
-          }
-        : {}),
+      ...nodetypeNodeSelector(env.KUBERNETES_WORKER_NODETYPE_LABEL),
       ...(env.KUBERNETES_POD_DNS_NDOTS_OVERRIDE_ENABLED
         ? {
             dnsConfig: {
@@ -557,12 +555,12 @@ export class KubernetesWorkloadManager implements WorkloadManager {
     };
   }
 
-  #getScheduleTolerations(isScheduledRun: boolean): k8s.V1Toleration[] | undefined {
-    if (!isScheduledRun || !env.KUBERNETES_SCHEDULED_RUN_TOLERATIONS?.length) {
-      return undefined;
-    }
-
-    return env.KUBERNETES_SCHEDULED_RUN_TOLERATIONS;
+  #getTolerations(isScheduledRun: boolean): k8s.V1Toleration[] | undefined {
+    return runPodTolerations(
+      env.KUBERNETES_RUNNER_TOLERATIONS,
+      env.KUBERNETES_SCHEDULED_RUN_TOLERATIONS,
+      isScheduledRun
+    );
   }
 
   #getProjectPodAffinity(projectId: string): k8s.V1PodAffinity | undefined {

apps/supervisor/src/workloadManager/kubernetesPodSpec.ts

@@ -6,6 +6,36 @@ import type { k8s } from "../clients/kubernetes.js";
  */
 export const BLOCK_IO_URING_SECCOMP_PROFILE = "profiles/block-io-uring.json";
 
+/**
+ * An empty label is the documented off-switch, leaving the pod unpinned. The Helm
+ * chart ships an empty value, so don't collapse this into a fallback default -
+ * that would pin every chart install to a label its nodes don't carry.
+ */
+export function nodetypeNodeSelector(
+  label: string | undefined
+): Pick<k8s.V1PodSpec, "nodeSelector"> {
+  return label ? { nodeSelector: { nodetype: label } } : {};
+}
+
+/**
+ * Tolerations for a run pod: the cluster-wide set, plus the scheduled-run set when the
+ * run came from a schedule tree. Not reconciled - Kubernetes matches tolerations as an
+ * any-match set, so a broad entry in one set can subsume a narrower one in the other.
+ * Returns undefined rather than an empty array to leave the field unset.
+ */
+export function runPodTolerations(
+  runnerTolerations: k8s.V1Toleration[] | undefined,
+  scheduledRunTolerations: k8s.V1Toleration[] | undefined,
+  isScheduledRun: boolean
+): k8s.V1Toleration[] | undefined {
+  const tolerations = [
+    ...(runnerTolerations ?? []),
+    ...(isScheduledRun ? (scheduledRunTolerations ?? []) : []),
+  ];
+
+  return tolerations.length > 0 ? tolerations : undefined;
+}
+
 /**
  * Node >= 24 always creates io_uring fds, which can't be checkpointed. Blocking
  * io_uring_setup makes libuv fall back to epoll. Other runtimes don't need this,

docs/self-hosting/env/supervisor.mdx

@@ -46,7 +46,8 @@ mode: "wide"
 | **Kubernetes settings**                     |          |             |                                                             |
 | `KUBERNETES_FORCE_ENABLED`                  | No       | false       | Force Kubernetes mode.                                      |
 | `KUBERNETES_NAMESPACE`                      | No       | default     | The namespace that runs should be in.                       |
-| `KUBERNETES_WORKER_NODETYPE_LABEL`          | No       | v4-worker   | Nodes for runs need this label, e.g. `nodetype=v4-worker`.  |
+| `KUBERNETES_WORKER_NODETYPE_LABEL`          | No       | v4-worker   | Nodes for runs need `nodetype=<this>`. Empty: any node.     |
+| `KUBERNETES_RUNNER_TOLERATIONS`             | No       | —           | Run pod tolerations. CSV: `key=value:effect`/`key:effect`.  |
 | `KUBERNETES_IMAGE_PULL_SECRETS`             | No       | —           | Image pull secrets (CSV).                                   |
 | `KUBERNETES_EPHEMERAL_STORAGE_SIZE_LIMIT`   | No       | 10Gi        | Ephemeral storage size limit. Applies to all runs.          |
 | `KUBERNETES_EPHEMERAL_STORAGE_SIZE_REQUEST` | No       | 2Gi         | Ephemeral storage size request. Applies to all runs.        |

hosting/k8s/helm/templates/supervisor.yaml

@@ -170,6 +170,10 @@ spec:
               value: {{ .Values.supervisor.config.kubernetes.forceEnabled | quote }}
             - name: KUBERNETES_WORKER_NODETYPE_LABEL
               value: {{ .Values.supervisor.config.kubernetes.workerNodetypeLabel | quote }}
+            {{- with .Values.supervisor.config.kubernetes.runnerTolerations }}
+            - name: KUBERNETES_RUNNER_TOLERATIONS
+              value: {{ join "," . | quote }}
+            {{- end }}
             {{- $registryAuthEnabled := false }}
             {{- if .Values.registry.deploy }}
               {{- $registryAuthEnabled = .Values.registry.auth.enabled }}

hosting/k8s/helm/values.yaml

@@ -296,6 +296,7 @@ supervisor:
       forceEnabled: true
       namespace: "" # Default: uses release namespace
       workerNodetypeLabel: "" # When set, runs will only be scheduled on nodes with "nodetype=<label>"
+      runnerTolerations: [] # Run pod tolerations, e.g. ["dedicated=runs:NoSchedule"]
       ephemeralStorageSizeLimit: "" # Default: 10Gi
       ephemeralStorageSizeRequest: "" # Default: 2Gi´
     podCleaner:
@@ -386,7 +387,7 @@ supervisor:
         key: ""
 
   nodeSelector: {}
-  tolerations: []
+  tolerations: [] # For the supervisor pod itself, not run pods
   affinity: {}
 
 # PostgreSQL configuration