diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index dc47b890b..4e229834c 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -40,8 +40,7 @@ const EnvironmentSchema = z.object({ EXECUTION_WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000), WORKER_ENABLED: z.string().default("true"), EXECUTION_WORKER_ENABLED: z.string().default("true"), - GRACEFUL_SHUTDOWN_TIMEOUT: z.coerce.number().int().default(60_000), - WORKER_MIGRATION_DELAY: z.coerce.number().int().default(60_000 * 2 + 15_000), + GRACEFUL_SHUTDOWN_TIMEOUT: z.coerce.number().int().default(60000), }); export type Environment = z.infer; diff --git a/apps/webapp/app/services/worker.server.ts b/apps/webapp/app/services/worker.server.ts index 2ac9efa7b..a1c2b4c58 100644 --- a/apps/webapp/app/services/worker.server.ts +++ b/apps/webapp/app/services/worker.server.ts @@ -157,10 +157,13 @@ async function addMigrationDelayAndNotify() { return; } - console.log(`⚠️ detected pending graphile migration`); - console.log(`⚠️ delaying worker startup by ${env.WORKER_MIGRATION_DELAY}ms`); + // add 15s to graceful shutdown timeout, just to be safe + const migrationDelayInMs = env.GRACEFUL_SHUTDOWN_TIMEOUT + 15000 - await new Promise((resolve) => setTimeout(resolve, env.WORKER_MIGRATION_DELAY)); + console.log(`⚠️ detected pending graphile migration`); + console.log(`⚠️ delaying worker startup by ${migrationDelayInMs}ms`); + + await new Promise((resolve) => setTimeout(resolve, migrationDelayInMs)); console.log(`⚠️ notifying running workers about incoming migration`); diff --git a/docs/documentation/guides/self-hosting/graphile-migration.mdx b/docs/documentation/guides/self-hosting/graphile-migration.mdx index 9e359c3e6..0f378be66 100644 --- a/docs/documentation/guides/self-hosting/graphile-migration.mdx +++ b/docs/documentation/guides/self-hosting/graphile-migration.mdx @@ -21,12 +21,9 @@ To prevent this, please shut your server down _gracefully_. How you do this will - Using `docker stop` or `docker-compose down` - Triggering a deployment on your managed platform of choice -## Environment variables +### Changing the default timeout -Two environment variables may be helpful here: - -1. `GRACEFUL_SHUTDOWN_TIMEOUT` - gives your workers more time to gracefully shut down -2. `WORKER_MIGRATION_DELAY` - adds a delay before starting workers and applying migrations (useful for rolling deplyoments) +Set the `GRACEFUL_SHUTDOWN_TIMEOUT` environment variable to change the default of 60 seconds - values are specified in _milliseconds_. This may be required if you have active jobs that need longer to complete. ## Problems