refactor(supervisor): demote per-run logs to debug/verbose for quieter prod output
This commit is contained in:
@@ -198,13 +198,13 @@ class ManagedSupervisor {
|
||||
}
|
||||
|
||||
this.workerSession.on("runNotification", async ({ time, run }) => {
|
||||
this.logger.log("runNotification", { time, run });
|
||||
this.logger.verbose("runNotification", { time, run });
|
||||
|
||||
this.workloadServer.notifyRun({ run });
|
||||
});
|
||||
|
||||
this.workerSession.on("runQueueMessage", async ({ time, message, dequeueResponseMs, pollingIntervalMs }) => {
|
||||
this.logger.log(`Received message with timestamp ${time.toLocaleString()}`, message);
|
||||
this.logger.verbose(`Received message with timestamp ${time.toLocaleString()}`, message);
|
||||
|
||||
if (message.completedWaitpoints.length > 0) {
|
||||
this.logger.debug("Run has completed waitpoints", {
|
||||
@@ -221,7 +221,7 @@ class ManagedSupervisor {
|
||||
const { checkpoint, ...rest } = message;
|
||||
|
||||
if (checkpoint) {
|
||||
this.logger.log("Restoring run", { runId: message.run.id });
|
||||
this.logger.debug("Restoring run", { runId: message.run.id });
|
||||
|
||||
if (this.isComputeMode && this.computeManager && env.COMPUTE_SNAPSHOTS_ENABLED) {
|
||||
try {
|
||||
@@ -244,7 +244,7 @@ class ManagedSupervisor {
|
||||
});
|
||||
|
||||
if (didRestore) {
|
||||
this.logger.log("Compute restore successful", { runId: message.run.id, runnerId });
|
||||
this.logger.debug("Compute restore successful", { runId: message.run.id, runnerId });
|
||||
} else {
|
||||
this.logger.error("Compute restore failed", { runId: message.run.id, runnerId });
|
||||
}
|
||||
@@ -271,7 +271,7 @@ class ManagedSupervisor {
|
||||
});
|
||||
|
||||
if (didRestore) {
|
||||
this.logger.log("Restore successful", { runId: message.run.id });
|
||||
this.logger.debug("Restore successful", { runId: message.run.id });
|
||||
} else {
|
||||
this.logger.error("Restore failed", { runId: message.run.id });
|
||||
}
|
||||
@@ -282,14 +282,14 @@ class ManagedSupervisor {
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.log("Scheduling run", { runId: message.run.id });
|
||||
this.logger.debug("Scheduling run", { runId: message.run.id });
|
||||
|
||||
const warmStartStart = performance.now();
|
||||
const didWarmStart = await this.tryWarmStart(message);
|
||||
const warmStartCheckMs = Math.round(performance.now() - warmStartStart);
|
||||
|
||||
if (didWarmStart) {
|
||||
this.logger.log("Warm start successful", { runId: message.run.id });
|
||||
this.logger.debug("Warm start successful", { runId: message.run.id });
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -151,7 +151,7 @@ export class FailedPodHandler {
|
||||
}
|
||||
|
||||
private async onPodCompleted(pod: V1Pod) {
|
||||
this.logger.info("pod-completed", this.podSummary(pod));
|
||||
this.logger.debug("pod-completed", this.podSummary(pod));
|
||||
this.informerEventsTotal.inc({ namespace: this.namespace, verb: "add" });
|
||||
|
||||
if (!pod.metadata?.name) {
|
||||
@@ -165,7 +165,7 @@ export class FailedPodHandler {
|
||||
}
|
||||
|
||||
if (pod.metadata?.deletionTimestamp) {
|
||||
this.logger.info("pod-completed: pod is being deleted", this.podSummary(pod));
|
||||
this.logger.verbose("pod-completed: pod is being deleted", this.podSummary(pod));
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -188,7 +188,7 @@ export class FailedPodHandler {
|
||||
}
|
||||
|
||||
private async onPodSucceeded(pod: V1Pod) {
|
||||
this.logger.info("pod-succeeded", this.podSummary(pod));
|
||||
this.logger.debug("pod-succeeded", this.podSummary(pod));
|
||||
this.processedPodsTotal.inc({
|
||||
namespace: this.namespace,
|
||||
status: this.podStatus(pod),
|
||||
@@ -196,7 +196,7 @@ export class FailedPodHandler {
|
||||
}
|
||||
|
||||
private async onPodFailed(pod: V1Pod) {
|
||||
this.logger.info("pod-failed", this.podSummary(pod));
|
||||
this.logger.debug("pod-failed", this.podSummary(pod));
|
||||
|
||||
try {
|
||||
await this.processFailedPod(pod);
|
||||
@@ -208,7 +208,7 @@ export class FailedPodHandler {
|
||||
}
|
||||
|
||||
private async processFailedPod(pod: V1Pod) {
|
||||
this.logger.info("pod-failed: processing pod", this.podSummary(pod));
|
||||
this.logger.verbose("pod-failed: processing pod", this.podSummary(pod));
|
||||
|
||||
const mainContainer = pod.status?.containerStatuses?.find((c) => c.name === "run-controller");
|
||||
|
||||
@@ -231,7 +231,7 @@ export class FailedPodHandler {
|
||||
}
|
||||
|
||||
private async deletePod(pod: V1Pod) {
|
||||
this.logger.info("pod-failed: deleting pod", this.podSummary(pod));
|
||||
this.logger.verbose("pod-failed: deleting pod", this.podSummary(pod));
|
||||
try {
|
||||
await this.k8s.core.deleteNamespacedPod({
|
||||
name: pod.metadata!.name!,
|
||||
|
||||
@@ -90,7 +90,7 @@ export class PodCleaner {
|
||||
status: "succeeded",
|
||||
});
|
||||
|
||||
this.logger.info("Deleted batch of pods", { continuationToken });
|
||||
this.logger.debug("Deleted batch of pods", { continuationToken });
|
||||
} catch (err) {
|
||||
this.logger.error("Failed to delete batch of pods", {
|
||||
err: err instanceof Error ? err.message : String(err),
|
||||
|
||||
@@ -62,7 +62,7 @@ export class DockerWorkloadManager implements WorkloadManager {
|
||||
}
|
||||
|
||||
async create(opts: WorkloadManagerCreateOptions) {
|
||||
this.logger.log("create()", { opts });
|
||||
this.logger.verbose("create()", { opts });
|
||||
|
||||
const runnerId = getRunnerId(opts.runFriendlyId, opts.nextAttemptNumber);
|
||||
|
||||
|
||||
@@ -100,7 +100,7 @@ export class KubernetesWorkloadManager implements WorkloadManager {
|
||||
}
|
||||
|
||||
async create(opts: WorkloadManagerCreateOptions) {
|
||||
this.logger.log("[KubernetesWorkloadManager] Creating container", { opts });
|
||||
this.logger.verbose("[KubernetesWorkloadManager] Creating container", { opts });
|
||||
|
||||
const runnerId = getRunnerId(opts.runFriendlyId, opts.nextAttemptNumber);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user