fix(schedule-engine,webapp): log out-of-entitlements scheduled triggers as warnings (#4067)

## Summary

When a scheduled task fires for an organization that is out of
entitlements, the trigger can't proceed. That's an expected outcome, but
it was being logged at error level and surfaced as a failure.

## Fix

The trigger callback now classifies an out-of-entitlements result as its
own error type (`OUT_OF_ENTITLEMENTS`), and the schedule engine logs
both that and the existing queue-limit result as warnings rather than
errors. The run still doesn't fire and the `schedule_execution_failure`
metric still records the outcome (now tagged `out_of_entitlements`), so
nothing about observability or behavior changes beyond the log level.

.server-changes/scheduled-task-out-of-entitlements-warning.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: improvement
+---
+
+Treat a scheduled task trigger that fails because the organization is out of entitlements as an expected outcome: the schedule engine now logs it as a warning instead of an error, mirroring how environment queue-limit results are already handled.

apps/webapp/app/v3/scheduleEngine.server.ts

@@ -6,7 +6,7 @@ import { env } from "~/env.server";
 import { devPresence } from "~/presenters/v3/DevPresence.server";
 import { logger } from "~/services/logger.server";
 import { singleton } from "~/utils/singleton";
-import { TriggerTaskService } from "./services/triggerTask.server";
+import { OutOfEntitlementError, TriggerTaskService } from "./services/triggerTask.server";
 import { meter, tracer } from "./tracer.server";
 import { workerQueue } from "~/services/worker.server";
 import { ServiceValidationError } from "./services/common.server";
@@ -136,6 +136,11 @@ function createScheduleEngine() {
           errorMessage.includes("queue size limit for this environment has been reached")
         ) {
           errorType = "QUEUE_LIMIT";
+        } else if (error instanceof OutOfEntitlementError) {
+          // The org is out of entitlements. This is an expected outcome, not a
+          // system error, so the engine logs it as a warning rather than
+          // reporting it as an error.
+          errorType = "OUT_OF_ENTITLEMENTS";
         }
 
         return {

internal-packages/schedule-engine/src/engine/index.ts

@@ -522,13 +522,19 @@ export class ScheduleEngine {
 
               span.setAttribute("trigger_success", true);
             } else {
-              const isQueueLimit = result.errorType === "QUEUE_LIMIT";
+              // QUEUE_LIMIT and OUT_OF_ENTITLEMENTS are expected,
+              // non-actionable outcomes (the environment is at its queue limit,
+              // or the org is out of entitlements). Log them as warnings so they
+              // aren't reported as errors, while still recording the metric.
+              const isExpectedFailure =
+                result.errorType === "QUEUE_LIMIT" || result.errorType === "OUT_OF_ENTITLEMENTS";
 
-              if (isQueueLimit) {
-                this.logger.warn("Scheduled task trigger skipped due to queue limit", {
+              if (isExpectedFailure) {
+                this.logger.warn("Scheduled task trigger skipped", {
                   instanceId: params.instanceId,
                   taskIdentifier: instance.taskSchedule.taskIdentifier,
                   durationMs: triggerDuration,
+                  errorType: result.errorType,
                   error: result.error,
                 });
               } else {
@@ -540,10 +546,17 @@ export class ScheduleEngine {
                 });
               }
 
+              const failureErrorType =
+                result.errorType === "QUEUE_LIMIT"
+                  ? "queue_limit"
+                  : result.errorType === "OUT_OF_ENTITLEMENTS"
+                    ? "out_of_entitlements"
+                    : "task_failure";
+
               this.scheduleExecutionFailureCounter.add(1, {
                 environment_type: environmentType,
                 schedule_type: scheduleType,
-                error_type: isQueueLimit ? "queue_limit" : "task_failure",
+                error_type: failureErrorType,
               });
 
               span.setAttribute("trigger_success", false);

internal-packages/schedule-engine/src/engine/types.ts

@@ -24,7 +24,7 @@ export type TriggerScheduledTaskParams = {
   exactScheduleTime?: Date;
 };
 
-export type TriggerScheduledTaskErrorType = "QUEUE_LIMIT" | "SYSTEM_ERROR";
+export type TriggerScheduledTaskErrorType = "QUEUE_LIMIT" | "OUT_OF_ENTITLEMENTS" | "SYSTEM_ERROR";
 
 export interface TriggerScheduledTaskCallback {
   (params: TriggerScheduledTaskParams): Promise<{