Compare commits

...

57 Commits

Author SHA1 Message Date
nicktrn 2ce630072c Merge branch 'main' into v3/infra-updates
🚢 Publish Infra Images / build (coordinator) (push) Has been cancelled
🚢 Publish Infra Images / build (kubernetes-provider) (push) Has been cancelled
2024-03-25 19:00:26 +00:00
nicktrn 671dff6d6f add secure flag to zod connection 2024-03-25 18:59:05 +00:00
nicktrn 3ee096d50c skip connection handler when waiting for post start hook 2024-03-25 14:30:50 +00:00
nicktrn fb9e63bab5 Merge branch 'main' into v3/infra-updates 2024-03-25 12:12:15 +00:00
nicktrn ad2d6d2f1e lifecycle hook fixes 2024-03-25 12:11:16 +00:00
nicktrn 26439105a9 fix a few more resume issues 2024-03-24 22:16:01 +00:00
nicktrn 9c586759a6 only checkpoint for retry if large delay 2024-03-23 23:29:39 +00:00
nicktrn 3ef4949270 make docker send postStart hook 2024-03-23 23:26:34 +00:00
nicktrn fe38d12661 fix hooks and improve reconnect 2024-03-23 11:17:38 +00:00
nicktrn b8d0663d0d Merge branch 'main' into v3/infra-updates 2024-03-22 22:09:29 +00:00
nicktrn 6a93aefa55 lifecycle hooks are back 2024-03-22 22:01:48 +00:00
nicktrn 8fd16fe3f3 reconnect wip 2024-03-22 18:56:45 +00:00
nicktrn 6ababeae55 restore dependencies from events 2024-03-22 18:54:07 +00:00
nicktrn 30c74f3c4b remove attempt id env var 2024-03-22 15:38:32 +00:00
nicktrn 0683ecaf39 restore from checkpoint events and fix statuses 2024-03-22 14:25:30 +00:00
nicktrn b017af4e6d checkpoint readiness and cancel signals 2024-03-22 08:41:31 +00:00
nicktrn 6eae7270c2 use system clock to end wait spans 2024-03-21 18:43:16 +00:00
nicktrn 75ce3c65cb always log disconnect reason 2024-03-21 18:37:03 +00:00
nicktrn cbc84e2011 measure basic checkpoint perf 2024-03-21 18:36:12 +00:00
nicktrn e2b10f42a0 improve wait accuracy 2024-03-21 16:10:22 +00:00
nicktrn 9fc62f7f10 switch to run id for container names 2024-03-21 14:56:23 +00:00
nicktrn 5180aacd4a atomic post-completion checkpoints 2024-03-21 14:14:46 +00:00
nicktrn 001bd97b63 remove init container from create 2024-03-21 11:21:37 +00:00
nicktrn 5017a8a808 don't restart restored containers 2024-03-21 11:20:34 +00:00
nicktrn d5b1bcac58 add missing restore label 2024-03-21 10:34:10 +00:00
nicktrn 0f6ea0b262 replace dapi with taskinfo 2024-03-21 10:07:12 +00:00
nicktrn e0507a9598 pass in coordinator host via volume 2024-03-20 21:56:44 +00:00
nicktrn 27ad6574c3 disable post start lifecycle hook 2024-03-20 19:18:46 +00:00
nicktrn a4209dac25 Merge branch 'main' into v3/infra-updates 2024-03-20 17:30:44 +00:00
nicktrn 6fd635cdea log handler payload 2024-03-20 17:16:15 +00:00
nicktrn b091347614 append shortcode to restore names 2024-03-20 15:15:51 +00:00
nicktrn f21c94c0ce update restore pull secret name 2024-03-20 15:13:03 +00:00
nicktrn 18bd7f4623 exit process after checkpointing 2024-03-20 14:38:21 +00:00
nicktrn 84250a6f9e fix structured log merge 2024-03-20 13:57:55 +00:00
nicktrn 961fb3219e Merge branch 'main' into v3/infra-updates 2024-03-20 13:20:51 +00:00
nicktrn 8fda3d7b36 structured logs for socket connections 2024-03-20 13:17:41 +00:00
nicktrn c734db3951 log options on error 2024-03-20 10:05:03 +00:00
nicktrn 8ca21c1dd4 delete checkpoint archive after successful push 2024-03-20 10:00:15 +00:00
nicktrn fdcb06a173 ensure valid registry login 2024-03-20 09:54:54 +00:00
nicktrn 6c0f8436ce update coordinator image 2024-03-19 22:24:32 +00:00
nicktrn a320f17bc4 Merge branch 'main' into v3/infra-updates 2024-03-19 19:22:27 +00:00
nicktrn 5ff738a18c checkpointing fixes 2024-03-19 19:21:12 +00:00
nicktrn 752f3f3d25 add push to deploy command for self-hosting 2024-03-19 16:24:32 +00:00
nicktrn d39aca8820 add missing run id on create 2024-03-19 15:45:42 +00:00
nicktrn 8f3bbc4c64 remove resource limits for now 2024-03-19 15:28:27 +00:00
nicktrn 561efe7bff rename type assertion 2024-03-19 15:14:41 +00:00
nicktrn b862724e6a fix error type assertion 2024-03-19 15:02:22 +00:00
nicktrn dc11de0714 shorten index container names 2024-03-19 13:54:44 +00:00
nicktrn ed7342bf49 rethrow k8s errors after logging 2024-03-19 13:49:42 +00:00
nicktrn 8cb09b3115 update infra publish workflow 2024-03-19 11:31:00 +00:00
nicktrn d1da2b5b2b Merge branch 'main' into v3/infra-updates 2024-03-19 09:20:54 +00:00
nicktrn 1cba308f56 upgrade prod base to node 20 2024-03-19 09:12:30 +00:00
nicktrn 9945980ab0 update kubernetes provider and fix builds again 2024-03-18 18:45:00 +00:00
nicktrn ce9c5d22fe fix provider build and dev 2024-03-18 16:01:18 +00:00
nicktrn 0871efe103 fix tsconfig paths 2024-03-18 15:34:25 +00:00
nicktrn 15d6ae9b85 fix retries.enabledInDev 2024-03-18 14:57:22 +00:00
nicktrn 593b66bfed add checkpoint restore events 2024-03-18 14:57:09 +00:00
7 changed files with 23 additions and 4 deletions
+2 -1
View File
@@ -1,3 +1,4 @@
HTTP_SERVER_PORT=8020
PLATFORM_ENABLED=true
PLATFORM_WS_PORT=3030
PLATFORM_WS_PORT=3030
SECURE_CONNECTION=false
+2
View File
@@ -26,6 +26,7 @@ const PLATFORM_ENABLED = ["1", "true"].includes(process.env.PLATFORM_ENABLED ??
const PLATFORM_HOST = process.env.PLATFORM_HOST || "127.0.0.1";
const PLATFORM_WS_PORT = process.env.PLATFORM_WS_PORT || 3030;
const PLATFORM_SECRET = process.env.PLATFORM_SECRET || "coordinator-secret";
const SECURE_CONNECTION = ["1", "true"].includes(process.env.SECURE_CONNECTION ?? "true");
const logger = new SimpleLogger(`[${NODE_NAME}]`);
@@ -365,6 +366,7 @@ class TaskCoordinator {
namespace: "coordinator",
host: PLATFORM_HOST,
port: Number(PLATFORM_WS_PORT),
secure: SECURE_CONNECTION,
clientMessages: CoordinatorToPlatformMessages,
serverMessages: PlatformToCoordinatorMessages,
authToken: PLATFORM_SECRET,
+1
View File
@@ -2,6 +2,7 @@ HTTP_SERVER_PORT=8050
PLATFORM_WS_PORT=3030
PLATFORM_SECRET=provider-secret
SECURE_CONNECTION=false
# Use this if you are on macOS
# COORDINATOR_HOST="host.docker.internal"
+1
View File
@@ -2,6 +2,7 @@ HTTP_SERVER_PORT=8060
PLATFORM_WS_PORT=3030
PLATFORM_SECRET=provider-secret
SECURE_CONNECTION=false
# Use this if you are on macOS
# COORDINATOR_HOST="host.docker.internal"
+1
View File
@@ -42,6 +42,7 @@ import {
import { logger } from "../utilities/logger.js";
import { createTaskFileImports, gatherTaskFiles } from "../utilities/taskFiles";
import { login } from "./login";
import { SetOptional } from "type-fest";
import type { SetOptional } from "type-fest";
import { bundleDependenciesPlugin, workerSetupImportConfigPlugin } from "../utilities/build";
+3
View File
@@ -18,6 +18,7 @@ const MACHINE_NAME = process.env.MACHINE_NAME || "local";
const PLATFORM_HOST = process.env.PLATFORM_HOST || "127.0.0.1";
const PLATFORM_WS_PORT = process.env.PLATFORM_WS_PORT || 3030;
const PLATFORM_SECRET = process.env.PLATFORM_SECRET || "provider-secret";
const SECURE_CONNECTION = ["1", "true"].includes(process.env.SECURE_CONNECTION ?? "true");
const logger = new SimpleLogger(`[${MACHINE_NAME}]`);
@@ -87,6 +88,7 @@ export class ProviderShell implements Provider {
namespace: "shared-queue",
host: PLATFORM_HOST,
port: Number(PLATFORM_WS_PORT),
secure: SECURE_CONNECTION,
clientMessages: ClientToSharedQueueMessages,
serverMessages: SharedQueueToClientMessages,
authToken: PLATFORM_SECRET,
@@ -138,6 +140,7 @@ export class ProviderShell implements Provider {
namespace: "provider",
host: PLATFORM_HOST,
port: Number(PLATFORM_WS_PORT),
secure: SECURE_CONNECTION,
clientMessages: ProviderToPlatformMessages,
serverMessages: PlatformToProviderMessages,
authToken: PLATFORM_SECRET,
+13 -3
View File
@@ -263,7 +263,8 @@ interface ZodSocketConnectionOptions<
TServerMessages extends ZodSocketMessageCatalogSchema,
> {
host: string;
port: number;
port?: number;
secure?: boolean;
namespace: string;
clientMessages: TClientMessages;
serverMessages: TServerMessages;
@@ -302,15 +303,24 @@ export class ZodSocketConnection<
#logger: StructuredLogger;
constructor(opts: ZodSocketConnectionOptions<TClientMessages, TServerMessages>) {
this.socket = io(`ws://${opts.host}:${opts.port}/${opts.namespace}`, {
const uri = `${opts.secure ? "wss" : "ws"}://${opts.host}:${
opts.port ?? (opts.secure ? "443" : "80")
}/${opts.namespace}`;
const logger = new SimpleStructuredLogger(opts.namespace, LogLevel.info);
logger.log("new zod socket", { uri });
this.socket = io(uri, {
transports: ["websocket"],
auth: {
token: opts.authToken,
},
extraHeaders: opts.extraHeaders,
reconnectionDelay: 500,
reconnectionDelayMax: 1000,
});
this.#logger = new SimpleStructuredLogger(opts.namespace, LogLevel.info, {
this.#logger = logger.child({
socketId: this.socket.id,
});