Eric Allam
b1b9321ad2
* WIP job run performance improvements - Added a `perf` tool to better measure job run performance under heavy load - Removed `runFinished` job (not really needed) - startQueuedRuns now uses a jobKey with replace - Fixed an issue with ZodWorker when using jobKey * Publish improvement docker images * fixed the improvement docker publishing * Downgrade back to prisma 4.16.0 because 5.1.x broke docker builds * Changes to how queued runs work - Split the worker into two different workers, one dedicated to performRunExecution - Schedule performRunExecution in a single place, with a queue and using a round robin manually controlled concurrency - Remove startQueuedRuns - All runs are queued before they are started - Setting the worker maxPoolSize to the same as the worker concurrency - Starting to be able to split the docker image * Remove queue name from startRun graphile job * Make the prisma connection pool stuff configurable through env vars * Hardcode (for now) the max concurrent runs limit * Rewrite performRunExecution to be more performant PerformRunExecutionV2: - Does not create and manage jobRunExecution records - Does not reimplement retrying, uses graphile worker retrying instead I’ve kept around PerformRunExecutionV1 so this works when deploying. Definitely needs LOTS of testing * Fix issues with cached tasks - Limit the size of the cached tasks sent when executing a run, using the knapsack problem dynamic programming approach - Actually USE the cached tasks in IO by using the idempotencyKey instead of the task ID - Remove output from all logs - Added a stress test job catalog * Forgot to commit the logger updates * Never log connectionString * Login to docker hub to get around rate limits * Add additional logging to the graphile workers * Fix the *_ENABLED env vars * Allow adding and removing jobs to be done from the webapp * Don’t set the job to failed if it’s being retried * Deprecated queue options in the job and removed startPosition. Now using the job/env combo as the job queue name * Dequeung jobs doesn’t check if the runner is initialized * Fixed issues with retrying a run getting stuck on a cancelled task, and errors from parsing the results of dequeing a job * Remove queued round robin thing that isn’t used anymore * Added slack to job catalog * Better forwards compat * Added long delay * Fixed lock file
83 lines
2.5 KiB
TypeScript
83 lines
2.5 KiB
TypeScript
import path from "path";
|
|
import express from "express";
|
|
import compression from "compression";
|
|
import morgan from "morgan";
|
|
import { createRequestHandler } from "@remix-run/express";
|
|
import { createTerminus } from "@godaddy/terminus";
|
|
|
|
const app = express();
|
|
|
|
app.use((req, res, next) => {
|
|
// helpful headers:
|
|
res.set("Strict-Transport-Security", `max-age=${60 * 60 * 24 * 365 * 100}`);
|
|
|
|
// /clean-urls/ -> /clean-urls
|
|
if (req.path.endsWith("/") && req.path.length > 1) {
|
|
const query = req.url.slice(req.path.length);
|
|
const safepath = req.path.slice(0, -1).replace(/\/+/g, "/");
|
|
res.redirect(301, safepath + query);
|
|
return;
|
|
}
|
|
next();
|
|
});
|
|
|
|
app.use(compression());
|
|
|
|
// http://expressjs.com/en/advanced/best-practice-security.html#at-a-minimum-disable-x-powered-by-header
|
|
app.disable("x-powered-by");
|
|
|
|
// Remix fingerprints its assets so we can cache forever.
|
|
app.use("/build", express.static("public/build", { immutable: true, maxAge: "1y" }));
|
|
|
|
// Everything else (like favicon.ico) is cached for an hour. You may want to be
|
|
// more aggressive with this caching.
|
|
app.use(express.static("public", { maxAge: "1h" }));
|
|
|
|
app.use(morgan("tiny"));
|
|
|
|
const MODE = process.env.NODE_ENV;
|
|
const BUILD_DIR = path.join(process.cwd(), "build");
|
|
|
|
app.all(
|
|
"*",
|
|
MODE === "production"
|
|
? createRequestHandler({ build: require(BUILD_DIR) })
|
|
: (...args) => {
|
|
purgeRequireCache();
|
|
const requestHandler = createRequestHandler({
|
|
build: require(BUILD_DIR),
|
|
mode: MODE,
|
|
});
|
|
return requestHandler(...args);
|
|
}
|
|
);
|
|
|
|
const port = process.env.REMIX_APP_PORT || 3000;
|
|
|
|
if (process.env.HTTP_SERVER_DISABLED !== "true") {
|
|
const server = app.listen(port, () => {
|
|
// require the built app so we're ready when the first request comes in
|
|
require(BUILD_DIR);
|
|
console.log(`✅ app ready: http://localhost:${port}`);
|
|
});
|
|
|
|
// Handle shutdowns gracefully
|
|
createTerminus(server, { signals: ["SIGINT", "SIGTERM"], timeout: 5000 });
|
|
} else {
|
|
console.log(`✅ app ready (skipping http server)`);
|
|
}
|
|
|
|
function purgeRequireCache() {
|
|
// purge require cache on requests for "server side HMR" this won't let
|
|
// you have in-memory objects between requests in development,
|
|
// alternatively you can set up nodemon/pm2-dev to restart the server on
|
|
// file changes, we prefer the DX of this though, so we've included it
|
|
// for you by default
|
|
for (const key in require.cache) {
|
|
if (key.startsWith(BUILD_DIR)) {
|
|
// eslint-disable-next-line @typescript-eslint/no-dynamic-delete
|
|
delete require.cache[key];
|
|
}
|
|
}
|
|
}
|