diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9d0efe5..391c77f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,6 +8,7 @@ on: - 'feature/**' - 'fix/**' pull_request: + workflow_dispatch: permissions: contents: read @@ -106,6 +107,7 @@ jobs: uses: helm/kind-action@v1.12.0 with: cluster_name: clawmanager-ci + ignore_failed_clean: true - name: Load image into kind run: kind load docker-image clawmanager:ci --name clawmanager-ci @@ -157,6 +159,12 @@ jobs: count=1, ) + text = re.sub( + r"(\n\s+- name: workspaces\s+)nfs:\s+server: workspace-store\.clawmanager-system\.svc\.cluster\.local\s+path: /exports/workspaces", + r"\1emptyDir: {}", + text, + ) + path.write_text(text, encoding="utf-8") PY diff --git a/.gitignore b/.gitignore index ad3f021..40af172 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,9 @@ Thumbs.db .codex-logs/ +.cache/ +.VSCodeCounter/ +.superpowers/ dev_docs/ devplan.md AGENTS.md diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go index d0892ed..8abb0de 100644 --- a/backend/cmd/server/main.go +++ b/backend/cmd/server/main.go @@ -6,6 +6,7 @@ import ( "net/http" "os" "os/signal" + "strings" "syscall" "time" @@ -14,6 +15,7 @@ import ( "clawreef/internal/db" "clawreef/internal/handlers" "clawreef/internal/middleware" + "clawreef/internal/models" "clawreef/internal/repository" "clawreef/internal/services" "clawreef/internal/services/k8s" @@ -65,9 +67,14 @@ func main() { instanceDesiredStateRepo := repository.NewInstanceDesiredStateRepository(database) instanceCommandRepo := repository.NewInstanceCommandRepository(database) instanceConfigRevisionRepo := repository.NewInstanceConfigRevisionRepository(database) + runtimePodRepo := repository.NewRuntimePodRepository(database) + bindingRepo := repository.NewInstanceRuntimeBindingRepository(database) + rolloutRepo := repository.NewRuntimeRolloutRepository(database) + workspaceFileAuditRepo := repository.NewWorkspaceFileAuditRepository(database) teamRepo := repository.NewTeamRepository(database) skillRepo := repository.NewSkillRepository(database) securityScanRepo := repository.NewSecurityScanRepository(database) + instanceExternalAccessRepo := repository.NewInstanceExternalAccessRepository(database) if repaired, repairErr := services.RepairSeededAdminPassword(userRepo); repairErr != nil { log.Printf("Warning: failed to repair seeded admin password: %v", repairErr) @@ -98,26 +105,53 @@ func main() { aiObservabilityService := services.NewAIObservabilityService(modelInvocationRepo, auditEventRepo, costRecordRepo, riskHitRepo, chatMessageRepo, llmModelRepo, instanceRepo, userRepo) clusterResourceService := services.NewClusterResourceService(instanceRepo) services.SetRuntimeImageSettingsProvider(systemImageSettingService) + services.SetOpenClawTransferRuntimeRepositories(instanceRepo, bindingRepo, runtimePodRepo) + runtimeAgentClient := services.NewRuntimeAgentClient(cfg.Runtime.AgentControlToken) instanceService := services.NewInstanceService( instanceRepo, quotaRepo, llmModelRepo, openClawConfigService, services.WithPrivilegedInstancePods(cfg.Kubernetes.Runtime.Pod.Privileged), + services.WithV2RuntimeLifecycle(runtimePodRepo, bindingRepo, runtimeAgentClient, cfg.Runtime.WorkspaceRoot), ) instanceAgentService := services.NewInstanceAgentService(instanceRepo, instanceAgentRepo, instanceDesiredStateRepo, instanceRuntimeStatusRepo, instanceCommandRepo) instanceRuntimeStatusService := services.NewInstanceRuntimeStatusService(instanceRuntimeStatusRepo, instanceAgentRepo, instanceDesiredStateRepo) instanceCommandService := services.NewInstanceCommandService(instanceCommandRepo, instanceRuntimeStatusRepo, instanceDesiredStateRepo) instanceConfigRevisionService := services.NewInstanceConfigRevisionService(instanceConfigRevisionRepo) - teamService := services.NewTeamService(teamRepo, instanceService) + teamService := services.NewTeamService(teamRepo, instanceService, services.WithTeamRuntimeWorkspaceRoot(cfg.Runtime.WorkspaceRoot)) + var platformRedis services.PlatformRedisClient + if redisURL := strings.TrimSpace(cfg.Runtime.RedisURL); redisURL != "" { + var redisErr error + platformRedis, redisErr = services.NewPlatformRedisClient(redisURL) + if redisErr != nil { + log.Printf("platform redis disabled: %v", redisErr) + } + } else { + log.Printf("platform redis disabled: redis url is empty") + } + runtimeEvents := services.NewRuntimeEventService(platformRedis) + workspaceFileService := services.NewWorkspaceFileService(workspaceFileAuditRepo) + runtimeWorkspaceFileService := services.NewRuntimeWorkspaceFileService(workspaceFileAuditRepo) skillService := services.NewSkillService(skillRepo, instanceRepo, instanceCommandService, objectStorageService, skillScannerClient) securityScanService := services.NewSecurityScanService(securityScanRepo, skillRepo, objectStorageService, skillScannerClient) + externalAccessService := services.NewInstanceExternalAccessService(instanceExternalAccessRepo) aiGatewayService := aigateway.NewService(llmModelRepo, modelInvocationService, auditEventService, costRecordService, riskDetectionService, riskHitService, chatSessionService, chatMessageService) // Initialize handlers authHandler := handlers.NewAuthHandler(authService) userHandler := handlers.NewUserHandler(userService, quotaService) - instanceHandler := handlers.NewInstanceHandler(instanceService, instanceAgentService, instanceRuntimeStatusService, instanceCommandService, instanceConfigRevisionService, openClawConfigService, skillService) + instanceHandler := handlers.NewInstanceHandler( + instanceService, + instanceAgentService, + instanceRuntimeStatusService, + instanceCommandService, + instanceConfigRevisionService, + openClawConfigService, + skillService, + externalAccessService, + services.WithInstanceProxyRuntimeRepositories(instanceRepo, runtimePodRepo, bindingRepo), + ) systemSettingsHandler := handlers.NewSystemSettingsHandler(systemImageSettingService) llmModelHandler := handlers.NewLLMModelHandler(llmModelService) aiGatewayHandler := handlers.NewAIGatewayHandler(aiGatewayService) @@ -130,15 +164,65 @@ func main() { securityHandler := handlers.NewSecurityHandler(securityScanService) agentHandler := handlers.NewAgentHandler(instanceAgentService, instanceCommandService, instanceRuntimeStatusService, instanceConfigRevisionService, skillService) teamHandler := handlers.NewTeamHandler(teamService) + workspaceFileHandler := handlers.NewWorkspaceFileHandler(instanceService, workspaceFileService, runtimeWorkspaceFileService) + runtimeAgentHandler := handlers.NewRuntimeAgentHandler(cfg.Runtime, runtimePodRepo, bindingRepo, runtimeEvents) // Initialize WebSocket hub and handler wsHub := services.GetHub() wsHandler := handlers.NewWebSocketHandler(wsHub) + var runtimeAdminEventBridgeCancel context.CancelFunc + if platformRedis != nil { + var bridgeCtx context.Context + bridgeCtx, runtimeAdminEventBridgeCancel = context.WithCancel(context.Background()) + services.StartRuntimeAdminEventBridge(bridgeCtx, runtimeEvents, wsHub) + } // Start sync service to keep instance status in sync with K8s syncService := services.NewSyncService(instanceRepo, instanceRuntimeStatusService) syncService.Start() teamService.Start() + var runtimeSchedulerCancel context.CancelFunc + var runtimeScheduler *services.RuntimeScheduler + if cfg.Runtime.SchedulerEnabled { + k8sClient := k8s.GetClient() + if k8sClient == nil || k8sClient.Clientset == nil { + log.Printf("runtime scheduler disabled: k8s client is unavailable") + } else { + leader := services.NewRuntimeLeaderService(k8sClient.Clientset, cfg.Runtime.Namespace, cfg.Runtime.BackendReplicaID) + runtimeDeployments := k8s.NewRuntimeDeploymentService(k8sClient.Clientset) + runtimeSchedulerOptions := []services.RuntimeSchedulerOption{ + services.WithRuntimeSchedulerWorkspaceRoot(cfg.Runtime.WorkspaceRoot), + services.WithRuntimeSchedulerNamespace(cfg.Runtime.Namespace), + services.WithRuntimeSchedulerGatewayPortRange(cfg.Runtime.GatewayPortStart, cfg.Runtime.GatewayPortEnd), + services.WithRuntimeSchedulerHeartbeatTimeout(cfg.Runtime.HeartbeatTimeout), + services.WithRuntimeSchedulerMaxGatewaysPerPod(cfg.Runtime.MaxGatewaysPerPod), + } + if gatewayEnvProvider, ok := instanceService.(interface { + BuildGatewayEnv(*models.Instance) (map[string]string, error) + }); ok { + runtimeSchedulerOptions = append(runtimeSchedulerOptions, services.WithRuntimeSchedulerGatewayEnvBuilder(gatewayEnvProvider.BuildGatewayEnv)) + } + runtimeScheduler = services.NewRuntimeScheduler( + instanceRepo, + runtimePodRepo, + bindingRepo, + rolloutRepo, + runtimeAgentClient, + runtimeEvents, + leader, + runtimeDeployments, + cfg.Runtime.SchedulerTick, + runtimeSchedulerOptions..., + ) + var schedulerCtx context.Context + schedulerCtx, runtimeSchedulerCancel = context.WithCancel(context.Background()) + runtimeScheduler.Start(schedulerCtx) + log.Printf("runtime scheduler started") + } + } else { + log.Printf("runtime scheduler disabled by configuration") + } + runtimePoolHandler := handlers.NewRuntimePoolHandler(runtimePodRepo, bindingRepo, rolloutRepo, runtimeScheduler, runtimeEvents) // Setup router r := gin.Default() @@ -150,8 +234,20 @@ func main() { r.NoMethod(egressProxyHandler.Handle) // Routes + r.Any("/s/:code", instanceHandler.OpenShortExternalAccess) + r.Any("/s/:code/*path", instanceHandler.OpenShortExternalAccess) + api := r.Group("/api/v1") { + runtimeAgent := api.Group("/runtime-agent") + { + runtimeAgent.POST("/register", runtimeAgentHandler.Register) + runtimeAgent.POST("/heartbeat", runtimeAgentHandler.Heartbeat) + runtimeAgent.POST("/metrics/report", runtimeAgentHandler.ReportMetrics) + runtimeAgent.POST("/gateways/report", runtimeAgentHandler.ReportGateways) + runtimeAgent.POST("/skills/report", runtimeAgentHandler.ReportSkills) + } + // Auth routes auth := api.Group("/auth") { @@ -212,6 +308,17 @@ func main() { instances.POST("/:id/openclaw/import", instanceHandler.ImportOpenClaw) instances.GET("/:id/hermes/export", instanceHandler.ExportHermes) instances.POST("/:id/hermes/import", instanceHandler.ImportHermes) + instances.GET("/:id/external-access", instanceHandler.GetExternalAccess) + instances.POST("/:id/external-access/share-link", instanceHandler.EnableShareLink) + instances.POST("/:id/external-access/password", instanceHandler.CreateExternalAccessPassword) + instances.DELETE("/:id/external-access", instanceHandler.DisableExternalAccess) + instances.GET("/:id/workspace/files", workspaceFileHandler.List) + instances.GET("/:id/workspace/preview", workspaceFileHandler.Preview) + instances.GET("/:id/workspace/download", workspaceFileHandler.Download) + instances.POST("/:id/workspace/upload", workspaceFileHandler.Upload) + instances.POST("/:id/workspace/folders", workspaceFileHandler.Mkdir) + instances.PATCH("/:id/workspace/entries", workspaceFileHandler.Rename) + instances.DELETE("/:id/workspace/entries", workspaceFileHandler.Delete) instances.GET("/:id/skills", skillHandler.ListInstanceSkills) instances.POST("/:id/skills", skillHandler.AttachSkillToInstance) instances.DELETE("/:id/skills/:skillId", skillHandler.RemoveSkillFromInstance) @@ -229,6 +336,17 @@ func main() { adminInstances.GET("", instanceHandler.ListAllInstances) } + adminRuntime := api.Group("/admin") + adminRuntime.Use(middleware.Auth()) + adminRuntime.Use(middleware.SetUserInfo(userRepo)) + adminRuntime.Use(middleware.NewAdminAuth(userRepo)) + { + adminRuntime.GET("/runtime-pods", runtimePoolHandler.ListPods) + adminRuntime.GET("/runtime-pods/:id/gateways", runtimePoolHandler.GetPodGateways) + adminRuntime.POST("/runtime-pods/:id/drain", runtimePoolHandler.DrainPod) + adminRuntime.POST("/runtime-rollouts", runtimePoolHandler.StartRollout) + } + teams := api.Group("/teams") teams.Use(middleware.Auth()) teams.Use(middleware.SetUserInfo(userRepo)) @@ -360,7 +478,7 @@ func main() { } gatewayLLM := api.Group("/gateway/llm") - gatewayLLM.Use(middleware.GatewayAuth(instanceRepo)) + gatewayLLM.Use(middleware.GatewayAuth(instanceRepo, bindingRepo)) { gatewayLLM.GET("/models", aiGatewayHandler.ListModels) gatewayLLM.POST("/chat/completions", aiGatewayHandler.ChatCompletions) @@ -423,6 +541,12 @@ func main() { } // Stop background services + if runtimeSchedulerCancel != nil { + runtimeSchedulerCancel() + } + if runtimeAdminEventBridgeCancel != nil { + runtimeAdminEventBridgeCancel() + } syncService.Stop() teamService.Stop() wsHub.Stop() diff --git a/backend/deployments/k8s/clawreef-incluster.yaml b/backend/deployments/k8s/clawreef-incluster.yaml index f306284..96c2c0e 100644 --- a/backend/deployments/k8s/clawreef-incluster.yaml +++ b/backend/deployments/k8s/clawreef-incluster.yaml @@ -413,8 +413,8 @@ data: SELECT 'openclaw', 'shell', - 'OpenClaw Shell', - 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-shell:latest', + 'OpenClaw Lite', + 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest', TRUE WHERE NOT EXISTS ( SELECT 1 @@ -449,6 +449,39 @@ data: updated_at = CURRENT_TIMESTAMP WHERE type IN ('openclaw', 'ubuntu', 'webtop', 'hermes') AND mount_path IN ('/data', '/home/user/data', '/config/.hermes'); + 031_add_hermes_lite_runtime_image.sql: | + USE clawreef; + UPDATE system_image_settings + SET + image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest', + display_name = 'Hermes Lite', + is_enabled = TRUE + WHERE instance_type = 'hermes' + AND runtime_type = 'shell' + AND image IN ( + 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-shell:latest', + 'registry.example.com/your-custom-shell-image:latest' + ); + + INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) + SELECT + 'hermes', + 'shell', + 'Hermes Lite', + 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest', + TRUE + WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'hermes' + AND runtime_type = 'shell' + ) + AND NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'hermes' + AND is_enabled = FALSE + ); --- apiVersion: v1 kind: PersistentVolumeClaim diff --git a/backend/internal/aigateway/service.go b/backend/internal/aigateway/service.go index bb515db..4fc84b1 100644 --- a/backend/internal/aigateway/service.go +++ b/backend/internal/aigateway/service.go @@ -72,6 +72,10 @@ type ChatCompletionRequest struct { User *string `json:"user,omitempty"` SessionID *string `json:"session_id,omitempty"` InstanceID *int `json:"instance_id,omitempty"` + InstanceMode *string `json:"instance_mode,omitempty"` + RuntimeType *string `json:"runtime_type,omitempty"` + GatewayID *string `json:"gateway_id,omitempty"` + RuntimePodID *int64 `json:"runtime_pod_id,omitempty"` TraceID *string `json:"trace_id,omitempty"` RequestID *string `json:"request_id,omitempty"` } @@ -357,6 +361,10 @@ func (s *service) ChatCompletions(ctx context.Context, userID int, req ChatCompl RequestID: prepared.requestIDPtr, UserID: prepared.userIDPtr, InstanceID: prepared.req.InstanceID, + InstanceMode: runtimeAttributionString(prepared.req.InstanceMode), + RuntimeType: runtimeAttributionString(prepared.req.RuntimeType), + GatewayID: runtimeAttributionString(prepared.req.GatewayID), + RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID), EventType: "gateway.request.blocked", TrafficClass: models.TrafficClassLLM, Severity: models.AuditSeverityWarn, @@ -388,6 +396,10 @@ func (s *service) StreamChatCompletions(ctx context.Context, userID int, req Cha RequestID: prepared.requestIDPtr, UserID: prepared.userIDPtr, InstanceID: prepared.req.InstanceID, + InstanceMode: runtimeAttributionString(prepared.req.InstanceMode), + RuntimeType: runtimeAttributionString(prepared.req.RuntimeType), + GatewayID: runtimeAttributionString(prepared.req.GatewayID), + RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID), EventType: "gateway.request.blocked", TrafficClass: models.TrafficClassLLM, Severity: models.AuditSeverityWarn, @@ -449,6 +461,10 @@ func (s *service) prepareChatRequest(userID int, req ChatCompletionRequest) (*pr RequestID: prepared.requestIDPtr, UserID: prepared.userIDPtr, InstanceID: prepared.req.InstanceID, + InstanceMode: runtimeAttributionString(prepared.req.InstanceMode), + RuntimeType: runtimeAttributionString(prepared.req.RuntimeType), + GatewayID: runtimeAttributionString(prepared.req.GatewayID), + RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID), EventType: "gateway.request.received", TrafficClass: models.TrafficClassLLM, Severity: models.AuditSeverityInfo, @@ -465,6 +481,10 @@ func (s *service) prepareChatRequest(userID int, req ChatCompletionRequest) (*pr RequestID: prepared.requestIDPtr, UserID: prepared.userIDPtr, InstanceID: prepared.req.InstanceID, + InstanceMode: runtimeAttributionString(prepared.req.InstanceMode), + RuntimeType: runtimeAttributionString(prepared.req.RuntimeType), + GatewayID: runtimeAttributionString(prepared.req.GatewayID), + RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID), EventType: "gateway.risk.detected", TrafficClass: models.TrafficClassLLM, Severity: models.AuditSeverityWarn, @@ -477,7 +497,7 @@ func (s *service) prepareChatRequest(userID int, req ChatCompletionRequest) (*pr resolvedModel, riskAction, resolveErr := s.resolveTargetModel(prepared.selectedModel, riskAnalysis) if resolveErr != nil { blockedInvocationID := s.recordBlockedInvocation(prepared.traceID, prepared.requestID, prepared.req, prepared.userID, prepared.selectedModel, resolveErr.Error()) - if err := s.riskHitService.RecordHits(prepared.traceID, prepared.req.SessionID, prepared.requestIDPtr, prepared.userIDPtr, prepared.req.InstanceID, blockedInvocationID, riskAction, riskAnalysis.Hits); err != nil { + if err := s.riskHitService.RecordHits(prepared.traceID, prepared.req.SessionID, prepared.requestIDPtr, prepared.userIDPtr, prepared.req.InstanceID, blockedInvocationID, riskHitAttribution(prepared.req), riskAction, riskAnalysis.Hits); err != nil { logPersistenceError("record blocked risk hits", prepared.traceID, err) } if err := s.auditEventService.RecordEvent(&models.AuditEvent{ @@ -486,6 +506,10 @@ func (s *service) prepareChatRequest(userID int, req ChatCompletionRequest) (*pr RequestID: prepared.requestIDPtr, UserID: prepared.userIDPtr, InstanceID: prepared.req.InstanceID, + InstanceMode: runtimeAttributionString(prepared.req.InstanceMode), + RuntimeType: runtimeAttributionString(prepared.req.RuntimeType), + GatewayID: runtimeAttributionString(prepared.req.GatewayID), + RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID), InvocationID: blockedInvocationID, EventType: "gateway.request.blocked", TrafficClass: models.TrafficClassLLM, @@ -498,7 +522,7 @@ func (s *service) prepareChatRequest(userID int, req ChatCompletionRequest) (*pr } if riskAnalysis.IsSensitive { - if err := s.riskHitService.RecordHits(prepared.traceID, prepared.req.SessionID, prepared.requestIDPtr, prepared.userIDPtr, prepared.req.InstanceID, nil, riskAction, riskAnalysis.Hits); err != nil { + if err := s.riskHitService.RecordHits(prepared.traceID, prepared.req.SessionID, prepared.requestIDPtr, prepared.userIDPtr, prepared.req.InstanceID, nil, riskHitAttribution(prepared.req), riskAction, riskAnalysis.Hits); err != nil { logPersistenceError("record risk hits", prepared.traceID, err) } if riskAction == models.RiskActionRouteSecureModel && resolvedModel != nil && resolvedModel.ID != prepared.selectedModel.ID { @@ -508,6 +532,10 @@ func (s *service) prepareChatRequest(userID int, req ChatCompletionRequest) (*pr RequestID: prepared.requestIDPtr, UserID: prepared.userIDPtr, InstanceID: prepared.req.InstanceID, + InstanceMode: runtimeAttributionString(prepared.req.InstanceMode), + RuntimeType: runtimeAttributionString(prepared.req.RuntimeType), + GatewayID: runtimeAttributionString(prepared.req.GatewayID), + RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID), EventType: "gateway.request.rerouted", TrafficClass: models.TrafficClassLLM, Severity: models.AuditSeverityWarn, @@ -655,6 +683,10 @@ func (s *service) recordFailure(traceID, requestID string, req ChatCompletionReq RequestID: requestID, UserID: intPtr(userID), InstanceID: req.InstanceID, + InstanceMode: runtimeAttributionString(req.InstanceMode), + RuntimeType: runtimeAttributionString(req.RuntimeType), + GatewayID: runtimeAttributionString(req.GatewayID), + RuntimePodID: runtimeAttributionInt64(req.RuntimePodID), ModelID: intPtr(model.ID), ProviderType: model.ProviderType, RequestedModel: req.Model, @@ -679,6 +711,10 @@ func (s *service) recordFailure(traceID, requestID string, req ChatCompletionReq RequestID: stringPtr(requestID), UserID: intPtr(userID), InstanceID: req.InstanceID, + InstanceMode: runtimeAttributionString(req.InstanceMode), + RuntimeType: runtimeAttributionString(req.RuntimeType), + GatewayID: runtimeAttributionString(req.GatewayID), + RuntimePodID: runtimeAttributionInt64(req.RuntimePodID), InvocationID: intPtr(invocation.ID), EventType: "gateway.request.failed", TrafficClass: models.TrafficClassLLM, @@ -703,6 +739,10 @@ func (s *service) recordSuccess(prepared *preparedChatRequest, providerRequestBo RequestID: prepared.requestID, UserID: prepared.userIDPtr, InstanceID: prepared.req.InstanceID, + InstanceMode: runtimeAttributionString(prepared.req.InstanceMode), + RuntimeType: runtimeAttributionString(prepared.req.RuntimeType), + GatewayID: runtimeAttributionString(prepared.req.GatewayID), + RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID), ModelID: intPtr(prepared.resolvedModel.ID), ProviderType: prepared.resolvedModel.ProviderType, RequestedModel: prepared.req.Model, @@ -750,6 +790,10 @@ func (s *service) recordSuccess(prepared *preparedChatRequest, providerRequestBo RequestID: prepared.requestIDPtr, UserID: prepared.userIDPtr, InstanceID: prepared.req.InstanceID, + InstanceMode: runtimeAttributionString(prepared.req.InstanceMode), + RuntimeType: runtimeAttributionString(prepared.req.RuntimeType), + GatewayID: runtimeAttributionString(prepared.req.GatewayID), + RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID), InvocationID: intPtr(invocation.ID), ModelID: intPtr(prepared.resolvedModel.ID), ProviderType: prepared.resolvedModel.ProviderType, @@ -772,6 +816,10 @@ func (s *service) recordSuccess(prepared *preparedChatRequest, providerRequestBo RequestID: prepared.requestIDPtr, UserID: prepared.userIDPtr, InstanceID: prepared.req.InstanceID, + InstanceMode: runtimeAttributionString(prepared.req.InstanceMode), + RuntimeType: runtimeAttributionString(prepared.req.RuntimeType), + GatewayID: runtimeAttributionString(prepared.req.GatewayID), + RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID), InvocationID: intPtr(invocation.ID), EventType: "gateway.request.completed", TrafficClass: models.TrafficClassLLM, @@ -787,6 +835,10 @@ func (s *service) recordSuccess(prepared *preparedChatRequest, providerRequestBo RequestID: prepared.requestIDPtr, UserID: prepared.userIDPtr, InstanceID: prepared.req.InstanceID, + InstanceMode: runtimeAttributionString(prepared.req.InstanceMode), + RuntimeType: runtimeAttributionString(prepared.req.RuntimeType), + GatewayID: runtimeAttributionString(prepared.req.GatewayID), + RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID), InvocationID: intPtr(invocation.ID), EventType: "gateway.usage.estimated", TrafficClass: models.TrafficClassLLM, @@ -1312,6 +1364,10 @@ func buildOpenAICompatibleRequestBody(req ChatCompletionRequest, model *models.L } delete(payload, "session_id") delete(payload, "instance_id") + delete(payload, "instance_mode") + delete(payload, "runtime_type") + delete(payload, "gateway_id") + delete(payload, "runtime_pod_id") delete(payload, "trace_id") delete(payload, "request_id") modelPayload, err := json.Marshal(model.ProviderModelName) @@ -1847,6 +1903,10 @@ func (s *service) recordBlockedInvocation(traceID, requestID string, req ChatCom RequestID: requestID, UserID: intPtr(userID), InstanceID: req.InstanceID, + InstanceMode: runtimeAttributionString(req.InstanceMode), + RuntimeType: runtimeAttributionString(req.RuntimeType), + GatewayID: runtimeAttributionString(req.GatewayID), + RuntimePodID: runtimeAttributionInt64(req.RuntimePodID), ModelID: intPtr(model.ID), ProviderType: model.ProviderType, RequestedModel: req.Model, @@ -2042,6 +2102,68 @@ func intPtr(value int) *int { return &value } +func runtimeAttributionString(value *string) *string { + if value == nil { + return nil + } + trimmed := strings.TrimSpace(*value) + if trimmed == "" { + return nil + } + return &trimmed +} + +func runtimeAttributionInt64(value *int64) *int64 { + if value == nil || *value <= 0 { + return nil + } + copied := *value + return &copied +} + +func applyInvocationRuntimeAttribution(invocation *models.ModelInvocation, req ChatCompletionRequest) { + if invocation == nil { + return + } + invocation.InstanceMode = runtimeAttributionString(req.InstanceMode) + invocation.RuntimeType = runtimeAttributionString(req.RuntimeType) + invocation.GatewayID = runtimeAttributionString(req.GatewayID) + invocation.RuntimePodID = runtimeAttributionInt64(req.RuntimePodID) +} + +func applyAuditRuntimeAttribution(event *models.AuditEvent, req ChatCompletionRequest) { + if event == nil { + return + } + event.InstanceMode = runtimeAttributionString(req.InstanceMode) + event.RuntimeType = runtimeAttributionString(req.RuntimeType) + event.GatewayID = runtimeAttributionString(req.GatewayID) + event.RuntimePodID = runtimeAttributionInt64(req.RuntimePodID) +} + +func applyCostRuntimeAttribution(record *models.CostRecord, req ChatCompletionRequest) { + if record == nil { + return + } + record.InstanceMode = runtimeAttributionString(req.InstanceMode) + record.RuntimeType = runtimeAttributionString(req.RuntimeType) + record.GatewayID = runtimeAttributionString(req.GatewayID) + record.RuntimePodID = runtimeAttributionInt64(req.RuntimePodID) +} + +func riskHitAttribution(req ChatCompletionRequest) *services.RiskHitAttribution { + attribution := &services.RiskHitAttribution{ + InstanceMode: runtimeAttributionString(req.InstanceMode), + RuntimeType: runtimeAttributionString(req.RuntimeType), + GatewayID: runtimeAttributionString(req.GatewayID), + RuntimePodID: runtimeAttributionInt64(req.RuntimePodID), + } + if attribution.InstanceMode == nil && attribution.RuntimeType == nil && attribution.GatewayID == nil && attribution.RuntimePodID == nil { + return nil + } + return attribution +} + func userIDOrZero(value *int) int { if value == nil { return 0 diff --git a/backend/internal/aigateway/service_test.go b/backend/internal/aigateway/service_test.go index 32f00b7..18772a7 100644 --- a/backend/internal/aigateway/service_test.go +++ b/backend/internal/aigateway/service_test.go @@ -47,7 +47,7 @@ func (s *stubChatSessionService) EnsureSession(sessionID string, userID, instanc func TestBuildProviderRequestPreservesToolConfiguration(t *testing.T) { req := ChatCompletionRequest{ Model: "gateway-model", - RawBody: []byte(`{"model":"gateway-model","messages":[{"role":"user","content":[{"type":"text","text":"weather in shanghai"}]}],"tools":[{"type":"function","function":{"name":"get_weather","parameters":{"type":"object"}}}],"tool_choice":{"type":"function","function":{"name":"get_weather"}},"stream":true,"stream_options":{"include_usage":false,"custom":"value"},"custom_field":{"nested":[1,2,3]},"session_id":"sess_123","request_id":"req_123","trace_id":"trc_123","instance_id":99}`), + RawBody: []byte(`{"model":"gateway-model","messages":[{"role":"user","content":[{"type":"text","text":"weather in shanghai"}]}],"tools":[{"type":"function","function":{"name":"get_weather","parameters":{"type":"object"}}}],"tool_choice":{"type":"function","function":{"name":"get_weather"}},"stream":true,"stream_options":{"include_usage":false,"custom":"value"},"custom_field":{"nested":[1,2,3]},"session_id":"sess_123","request_id":"req_123","trace_id":"trc_123","instance_id":99,"instance_mode":"lite","runtime_type":"gateway","gateway_id":"gw_123","runtime_pod_id":5}`), } model := &models.LLMModel{ @@ -95,6 +95,18 @@ func TestBuildProviderRequestPreservesToolConfiguration(t *testing.T) { if _, ok := payload["instance_id"]; ok { t.Fatalf("expected internal instance_id to be stripped") } + if _, ok := payload["instance_mode"]; ok { + t.Fatalf("expected internal instance_mode to be stripped") + } + if _, ok := payload["runtime_type"]; ok { + t.Fatalf("expected internal runtime_type to be stripped") + } + if _, ok := payload["gateway_id"]; ok { + t.Fatalf("expected internal gateway_id to be stripped") + } + if _, ok := payload["runtime_pod_id"]; ok { + t.Fatalf("expected internal runtime_pod_id to be stripped") + } if string(payload["stream_options"]) != `{"include_usage":false,"custom":"value"}` { t.Fatalf("expected stream_options to pass through unchanged, got %s", string(payload["stream_options"])) } @@ -135,6 +147,48 @@ func TestBuildProviderRequestUsesAnthropicProtocolForLocalModel(t *testing.T) { } } +func TestRuntimeAttributionHelpersPopulateRecords(t *testing.T) { + mode := "lite" + runtimeType := "gateway" + gatewayID := "gw_instance_1" + runtimePodID := int64(17) + req := ChatCompletionRequest{ + InstanceMode: &mode, + RuntimeType: &runtimeType, + GatewayID: &gatewayID, + RuntimePodID: &runtimePodID, + } + + invocation := &models.ModelInvocation{} + applyInvocationRuntimeAttribution(invocation, req) + if invocation.InstanceMode == nil || *invocation.InstanceMode != mode { + t.Fatalf("invocation instance mode not populated") + } + if invocation.GatewayID == nil || *invocation.GatewayID != gatewayID { + t.Fatalf("invocation gateway ID not populated") + } + + audit := &models.AuditEvent{} + applyAuditRuntimeAttribution(audit, req) + if audit.RuntimeType == nil || *audit.RuntimeType != runtimeType { + t.Fatalf("audit runtime type not populated") + } + if audit.RuntimePodID == nil || *audit.RuntimePodID != runtimePodID { + t.Fatalf("audit runtime pod ID not populated") + } + + cost := &models.CostRecord{} + applyCostRuntimeAttribution(cost, req) + if cost.GatewayID == nil || *cost.GatewayID != gatewayID { + t.Fatalf("cost gateway ID not populated") + } + + risk := riskHitAttribution(req) + if risk == nil || risk.GatewayID == nil || *risk.GatewayID != gatewayID { + t.Fatalf("risk attribution gateway ID not populated") + } +} + func TestRewriteStreamLineKeepsToolCalls(t *testing.T) { line := "data: {\"id\":\"chatcmpl-1\",\"model\":\"provider-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"id\":\"call_1\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Shanghai\\\"}\"}}]},\"finish_reason\":null}]}\n" diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index b9560ba..84bc033 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -3,7 +3,9 @@ package config import ( "fmt" "os" + "strconv" "strings" + "time" "gopkg.in/yaml.v3" ) @@ -14,6 +16,7 @@ type Config struct { Database DatabaseConfig `yaml:"database"` JWT JWTConfig `yaml:"jwt"` Kubernetes KubernetesConfig `yaml:"kubernetes"` + Runtime RuntimePoolConfig `yaml:"runtime"` ObjectStorage ObjectStorageConfig `yaml:"objectStorage"` SkillScanner SkillScannerConfig `yaml:"skillScanner"` } @@ -42,12 +45,12 @@ type JWTConfig struct { // KubernetesConfig holds Kubernetes-related configuration type KubernetesConfig struct { - Mode string `yaml:"mode"` // 连接模式: auto, incluster, outofcluster - OutOfCluster OutOfClusterConfig `yaml:"outOfCluster"` - InCluster InClusterConfig `yaml:"inCluster"` - Common CommonKubernetesConfig `yaml:"common"` - Runtime RuntimeConfig `yaml:"runtime"` - Logging LoggingConfig `yaml:"logging"` + Mode string `yaml:"mode"` // 连接模式: auto, incluster, outofcluster + OutOfCluster OutOfClusterConfig `yaml:"outOfCluster"` + InCluster InClusterConfig `yaml:"inCluster"` + Common CommonKubernetesConfig `yaml:"common"` + Runtime KubernetesRuntimeConfig `yaml:"runtime"` + Logging LoggingConfig `yaml:"logging"` } // OutOfClusterConfig holds out-of-cluster Kubernetes configuration @@ -82,8 +85,8 @@ type CommonKubernetesConfig struct { AutoCreateNamespace bool `yaml:"autoCreateNamespace"` } -// RuntimeConfig holds runtime configuration -type RuntimeConfig struct { +// KubernetesRuntimeConfig holds Kubernetes runtime configuration +type KubernetesRuntimeConfig struct { Pod RuntimePodConfig `yaml:"pod"` PVC RuntimePVCConfig `yaml:"pvc"` } @@ -116,6 +119,26 @@ type RuntimePVCConfig struct { HostPathPrefix string `yaml:"hostPathPrefix"` } +// RuntimePoolConfig holds shared V2 runtime pool configuration. +type RuntimePoolConfig struct { + Namespace string `yaml:"namespace"` + WorkspaceRoot string `yaml:"workspaceRoot"` + WorkspaceNFSServer string `yaml:"workspaceNfsServer"` + WorkspaceNFSPath string `yaml:"workspaceNfsPath"` + AgentControlToken string `yaml:"agentControlToken"` + AgentReportToken string `yaml:"agentReportToken"` + BackendReplicaID string `yaml:"backendReplicaId"` + RedisURL string `yaml:"redisUrl"` + SchedulerEnabled bool `yaml:"schedulerEnabled"` + HeartbeatTimeout time.Duration `yaml:"heartbeatTimeout"` + SchedulerTick time.Duration `yaml:"schedulerTick"` + OpenClawImage string `yaml:"openClawImage"` + HermesImage string `yaml:"hermesImage"` + MaxGatewaysPerPod int `yaml:"maxGatewaysPerPod"` + GatewayPortStart int `yaml:"gatewayPortStart"` + GatewayPortEnd int `yaml:"gatewayPortEnd"` +} + // LoggingConfig holds logging configuration type LoggingConfig struct { Level string `yaml:"level"` @@ -135,14 +158,15 @@ type ObjectStorageConfig struct { } type SkillScannerConfig struct { - BaseURL string `yaml:"baseUrl"` - APIKey string `yaml:"apiKey"` - TimeoutSeconds int `yaml:"timeoutSeconds"` - Enabled bool `yaml:"enabled"` + BaseURL string `yaml:"baseUrl"` + APIKey string `yaml:"apiKey"` + TimeoutSeconds int `yaml:"timeoutSeconds"` + Enabled bool `yaml:"enabled"` } // Load loads configuration from file and environment variables func Load() (*Config, error) { + runtimeNamespace := getEnv("RUNTIME_NAMESPACE", getEnv("K8S_NAMESPACE", "clawmanager-system")) config := &Config{ Server: ServerConfig{ Address: ":9001", @@ -177,7 +201,7 @@ func Load() (*Config, error) { RetryCount: 3, AutoCreateNamespace: true, }, - Runtime: RuntimeConfig{ + Runtime: KubernetesRuntimeConfig{ Pod: RuntimePodConfig{ ImageRegistry: "docker.io/clawreef", ContainerPort: 3001, @@ -199,6 +223,24 @@ func Load() (*Config, error) { LogAPICalls: false, }, }, + Runtime: RuntimePoolConfig{ + Namespace: runtimeNamespace, + WorkspaceRoot: getEnv("RUNTIME_WORKSPACE_ROOT", "/workspaces"), + WorkspaceNFSServer: getEnv("RUNTIME_WORKSPACE_NFS_SERVER", defaultWorkspaceNFSServer(runtimeNamespace)), + WorkspaceNFSPath: getEnv("RUNTIME_WORKSPACE_NFS_PATH", "/"), + AgentControlToken: getEnv("RUNTIME_AGENT_CONTROL_TOKEN", ""), + AgentReportToken: getEnv("RUNTIME_AGENT_REPORT_TOKEN", ""), + BackendReplicaID: getEnv("HOSTNAME", "clawmanager-backend-local"), + RedisURL: getEnv("PLATFORM_REDIS_URL", getEnv("TEAM_REDIS_URL", "")), + SchedulerEnabled: getEnvBool("RUNTIME_SCHEDULER_ENABLED", true), + HeartbeatTimeout: getEnvDuration("RUNTIME_HEARTBEAT_TIMEOUT", 10*time.Second), + SchedulerTick: getEnvDuration("RUNTIME_SCHEDULER_TICK", 2*time.Second), + OpenClawImage: getEnv("OPENCLAW_RUNTIME_IMAGE", "ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest"), + HermesImage: getEnv("HERMES_RUNTIME_IMAGE", "ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest"), + MaxGatewaysPerPod: getEnvInt("RUNTIME_MAX_GATEWAYS_PER_POD", 100), + GatewayPortStart: getEnvInt("RUNTIME_GATEWAY_PORT_START", 20000), + GatewayPortEnd: getEnvInt("RUNTIME_GATEWAY_PORT_END", 20099), + }, ObjectStorage: ObjectStorageConfig{ Endpoint: getEnv("OBJECT_STORAGE_ENDPOINT", ""), Region: getEnv("OBJECT_STORAGE_REGION", ""), @@ -211,10 +253,10 @@ func Load() (*Config, error) { LocalFallback: getEnv("OBJECT_STORAGE_LOCAL_FALLBACK", ".data/object-storage"), }, SkillScanner: SkillScannerConfig{ - BaseURL: getEnv("SKILL_SCANNER_BASE_URL", ""), - APIKey: getEnv("SKILL_SCANNER_API_KEY", ""), + BaseURL: getEnv("SKILL_SCANNER_BASE_URL", ""), + APIKey: getEnv("SKILL_SCANNER_API_KEY", ""), TimeoutSeconds: 30, - Enabled: strings.EqualFold(getEnv("SKILL_SCANNER_ENABLED", "false"), "true"), + Enabled: strings.EqualFold(getEnv("SKILL_SCANNER_ENABLED", "false"), "true"), }, } @@ -308,6 +350,29 @@ func applyEnvOverrides(config *Config) { config.Kubernetes.Runtime.PVC.HostPathPrefix = hostPathPrefix } + config.Runtime.Namespace = getEnv("RUNTIME_NAMESPACE", getEnv("K8S_NAMESPACE", config.Runtime.Namespace)) + config.Runtime.WorkspaceRoot = getEnv("RUNTIME_WORKSPACE_ROOT", config.Runtime.WorkspaceRoot) + config.Runtime.WorkspaceNFSServer = getEnv("RUNTIME_WORKSPACE_NFS_SERVER", config.Runtime.WorkspaceNFSServer) + config.Runtime.WorkspaceNFSPath = getEnv("RUNTIME_WORKSPACE_NFS_PATH", config.Runtime.WorkspaceNFSPath) + if strings.TrimSpace(config.Runtime.WorkspaceNFSServer) == "" { + config.Runtime.WorkspaceNFSServer = defaultWorkspaceNFSServer(config.Runtime.Namespace) + } + if strings.TrimSpace(config.Runtime.WorkspaceNFSPath) == "" { + config.Runtime.WorkspaceNFSPath = "/" + } + config.Runtime.AgentControlToken = getEnv("RUNTIME_AGENT_CONTROL_TOKEN", config.Runtime.AgentControlToken) + config.Runtime.AgentReportToken = getEnv("RUNTIME_AGENT_REPORT_TOKEN", config.Runtime.AgentReportToken) + config.Runtime.BackendReplicaID = getEnv("HOSTNAME", config.Runtime.BackendReplicaID) + config.Runtime.RedisURL = getEnv("PLATFORM_REDIS_URL", getEnv("TEAM_REDIS_URL", config.Runtime.RedisURL)) + config.Runtime.SchedulerEnabled = getEnvBool("RUNTIME_SCHEDULER_ENABLED", config.Runtime.SchedulerEnabled) + config.Runtime.HeartbeatTimeout = getEnvDuration("RUNTIME_HEARTBEAT_TIMEOUT", config.Runtime.HeartbeatTimeout) + config.Runtime.SchedulerTick = getEnvDuration("RUNTIME_SCHEDULER_TICK", config.Runtime.SchedulerTick) + config.Runtime.OpenClawImage = getEnv("OPENCLAW_RUNTIME_IMAGE", config.Runtime.OpenClawImage) + config.Runtime.HermesImage = getEnv("HERMES_RUNTIME_IMAGE", config.Runtime.HermesImage) + config.Runtime.MaxGatewaysPerPod = getEnvInt("RUNTIME_MAX_GATEWAYS_PER_POD", config.Runtime.MaxGatewaysPerPod) + config.Runtime.GatewayPortStart = getEnvInt("RUNTIME_GATEWAY_PORT_START", config.Runtime.GatewayPortStart) + config.Runtime.GatewayPortEnd = getEnvInt("RUNTIME_GATEWAY_PORT_END", config.Runtime.GatewayPortEnd) + if endpoint := os.Getenv("OBJECT_STORAGE_ENDPOINT"); endpoint != "" { config.ObjectStorage.Endpoint = endpoint } @@ -356,6 +421,50 @@ func getEnv(key, defaultValue string) string { return defaultValue } +func defaultWorkspaceNFSServer(namespace string) string { + namespace = strings.TrimSpace(namespace) + if namespace == "" { + namespace = "clawmanager-system" + } + return fmt.Sprintf("workspace-store.%s.svc.cluster.local", namespace) +} + +func getEnvInt(key string, defaultValue int) int { + value := os.Getenv(key) + if value == "" { + return defaultValue + } + parsed, err := strconv.Atoi(value) + if err != nil { + return defaultValue + } + return parsed +} + +func getEnvBool(key string, defaultValue bool) bool { + value := os.Getenv(key) + if value == "" { + return defaultValue + } + parsed, err := strconv.ParseBool(value) + if err != nil { + return defaultValue + } + return parsed +} + +func getEnvDuration(key string, defaultValue time.Duration) time.Duration { + value := os.Getenv(key) + if value == "" { + return defaultValue + } + parsed, err := time.ParseDuration(value) + if err != nil { + return defaultValue + } + return parsed +} + // GetKubeconfigPath returns the kubeconfig path for out-of-cluster mode func (c *Config) GetKubeconfigPath() string { return c.Kubernetes.OutOfCluster.Kubeconfig diff --git a/backend/internal/config/config_test.go b/backend/internal/config/config_test.go new file mode 100644 index 0000000..7df0dcc --- /dev/null +++ b/backend/internal/config/config_test.go @@ -0,0 +1,29 @@ +package config + +import "testing" + +func TestLoadRuntimeDefaults(t *testing.T) { + for _, key := range []string{ + "HERMES_RUNTIME_IMAGE", + "OPENCLAW_RUNTIME_IMAGE", + "RUNTIME_NAMESPACE", + "K8S_NAMESPACE", + "HOSTNAME", + "PLATFORM_REDIS_URL", + "TEAM_REDIS_URL", + } { + t.Setenv(key, "") + } + + cfg, err := Load() + if err != nil { + t.Fatalf("Load returned error: %v", err) + } + + if got, want := cfg.Runtime.HermesImage, "ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest"; got != want { + t.Fatalf("expected Hermes default image %q, got %q", want, got) + } + if got, want := cfg.Runtime.OpenClawImage, "ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest"; got != want { + t.Fatalf("expected OpenClaw default image %q, got %q", want, got) + } +} diff --git a/backend/internal/db/migrations/021_add_openclaw_shell_runtime_image.sql b/backend/internal/db/migrations/021_add_openclaw_shell_runtime_image.sql index 4e8f129..eddf004 100644 --- a/backend/internal/db/migrations/021_add_openclaw_shell_runtime_image.sql +++ b/backend/internal/db/migrations/021_add_openclaw_shell_runtime_image.sql @@ -61,8 +61,8 @@ INSERT INTO system_image_settings (instance_type, runtime_type, display_name, im SELECT 'openclaw', 'shell', - 'OpenClaw Shell', - 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-shell:latest', + 'OpenClaw Lite', + 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest', TRUE WHERE NOT EXISTS ( SELECT 1 diff --git a/backend/internal/db/migrations/023_add_runtime_pool_v2.sql b/backend/internal/db/migrations/023_add_runtime_pool_v2.sql new file mode 100644 index 0000000..76838d0 --- /dev/null +++ b/backend/internal/db/migrations/023_add_runtime_pool_v2.sql @@ -0,0 +1,121 @@ +SET @stmt = IF( + (SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'instances' AND COLUMN_NAME = 'workspace_path') = 0, + 'ALTER TABLE instances ADD COLUMN workspace_path VARCHAR(1024) NULL AFTER mount_path', + 'SELECT 1' +); +PREPARE stmt FROM @stmt; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @stmt = IF( + (SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'instances' AND COLUMN_NAME = 'workspace_usage_bytes') = 0, + 'ALTER TABLE instances ADD COLUMN workspace_usage_bytes BIGINT NOT NULL DEFAULT 0 AFTER workspace_path', + 'SELECT 1' +); +PREPARE stmt FROM @stmt; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @stmt = IF( + (SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'instances' AND COLUMN_NAME = 'runtime_generation') = 0, + 'ALTER TABLE instances ADD COLUMN runtime_generation INT NOT NULL DEFAULT 1 AFTER workspace_usage_bytes', + 'SELECT 1' +); +PREPARE stmt FROM @stmt; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @stmt = IF( + (SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'instances' AND COLUMN_NAME = 'runtime_error_message') = 0, + 'ALTER TABLE instances ADD COLUMN runtime_error_message TEXT NULL AFTER runtime_generation', + 'SELECT 1' +); +PREPARE stmt FROM @stmt; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +ALTER TABLE instances + MODIFY COLUMN runtime_type ENUM('desktop', 'shell', 'gateway') NOT NULL DEFAULT 'desktop'; + +CREATE TABLE IF NOT EXISTS runtime_pods ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + runtime_type VARCHAR(32) NOT NULL, + namespace VARCHAR(128) NOT NULL, + pod_name VARCHAR(255) NOT NULL, + pod_uid VARCHAR(128) NULL, + pod_ip VARCHAR(64) NULL, + node_name VARCHAR(255) NULL, + deployment_name VARCHAR(255) NOT NULL, + image_ref VARCHAR(512) NOT NULL, + agent_endpoint VARCHAR(255) NULL, + state VARCHAR(32) NOT NULL DEFAULT 'pending', + capacity INT NOT NULL DEFAULT 100, + used_slots INT NOT NULL DEFAULT 0, + draining TINYINT(1) NOT NULL DEFAULT 0, + cpu_millis_used BIGINT NOT NULL DEFAULT 0, + memory_bytes_used BIGINT NOT NULL DEFAULT 0, + disk_bytes_used BIGINT NOT NULL DEFAULT 0, + network_rx_bytes BIGINT NOT NULL DEFAULT 0, + network_tx_bytes BIGINT NOT NULL DEFAULT 0, + metrics_json JSON NULL, + last_seen_at DATETIME NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY uk_runtime_pod_namespace_name (namespace, pod_name), + KEY idx_runtime_pods_schedulable (runtime_type, state, draining, used_slots, capacity), + KEY idx_runtime_pods_last_seen (last_seen_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS instance_runtime_bindings ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + instance_id INT NOT NULL, + runtime_pod_id BIGINT NOT NULL, + runtime_type VARCHAR(32) NOT NULL, + gateway_id VARCHAR(128) NOT NULL, + gateway_port INT NOT NULL, + gateway_pid INT NULL, + workspace_path VARCHAR(1024) NOT NULL, + state VARCHAR(32) NOT NULL DEFAULT 'creating', + generation INT NOT NULL DEFAULT 1, + last_health_at DATETIME NULL, + error_message TEXT NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY uk_instance_runtime_binding_instance (instance_id), + UNIQUE KEY uk_instance_runtime_binding_gateway (runtime_pod_id, gateway_port), + KEY idx_instance_runtime_binding_pod_state (runtime_pod_id, state), + CONSTRAINT fk_instance_runtime_binding_instance + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + CONSTRAINT fk_instance_runtime_binding_pod + FOREIGN KEY (runtime_pod_id) REFERENCES runtime_pods(id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS runtime_rollouts ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + runtime_type VARCHAR(32) NOT NULL, + target_image_ref VARCHAR(512) NOT NULL, + status VARCHAR(32) NOT NULL DEFAULT 'pending', + batch_size INT NOT NULL DEFAULT 1, + max_unavailable INT NOT NULL DEFAULT 1, + started_by INT NULL, + started_at DATETIME NULL, + finished_at DATETIME NULL, + error_message TEXT NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + KEY idx_runtime_rollouts_type_status (runtime_type, status) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS workspace_file_audits ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + instance_id INT NOT NULL, + user_id INT NOT NULL, + action VARCHAR(32) NOT NULL, + relative_path VARCHAR(1024) NOT NULL, + bytes BIGINT NOT NULL DEFAULT 0, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + KEY idx_workspace_file_audits_instance_time (instance_id, created_at), + KEY idx_workspace_file_audits_user_time (user_id, created_at), + CONSTRAINT fk_workspace_file_audits_instance + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/backend/internal/db/migrations/024_add_gateway_runtime_type.sql b/backend/internal/db/migrations/024_add_gateway_runtime_type.sql new file mode 100644 index 0000000..82f5f5d --- /dev/null +++ b/backend/internal/db/migrations/024_add_gateway_runtime_type.sql @@ -0,0 +1,2 @@ +ALTER TABLE instances + MODIFY COLUMN runtime_type ENUM('desktop', 'shell', 'gateway') NOT NULL DEFAULT 'desktop'; diff --git a/backend/internal/db/migrations/025_add_instance_mode.sql b/backend/internal/db/migrations/025_add_instance_mode.sql new file mode 100644 index 0000000..8a24cf6 --- /dev/null +++ b/backend/internal/db/migrations/025_add_instance_mode.sql @@ -0,0 +1,24 @@ +SET @instance_mode_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instances' + AND COLUMN_NAME = 'instance_mode' +); +SET @instance_mode_column_sql = IF( + @instance_mode_column_exists = 0, + 'ALTER TABLE instances ADD COLUMN instance_mode ENUM(''lite'', ''pro'') NOT NULL DEFAULT ''lite'' AFTER runtime_type', + 'SELECT 1' +); +PREPARE instance_mode_column_stmt FROM @instance_mode_column_sql; +EXECUTE instance_mode_column_stmt; +DEALLOCATE PREPARE instance_mode_column_stmt; + +UPDATE instances +SET instance_mode = CASE + WHEN runtime_type = 'gateway' THEN 'lite' + ELSE 'pro' +END +WHERE instance_mode IS NULL + OR instance_mode = '' + OR (runtime_type <> 'gateway' AND instance_mode = 'lite'); diff --git a/backend/internal/db/migrations/027_add_instance_external_access.sql b/backend/internal/db/migrations/027_add_instance_external_access.sql new file mode 100644 index 0000000..18208c0 --- /dev/null +++ b/backend/internal/db/migrations/027_add_instance_external_access.sql @@ -0,0 +1,23 @@ +CREATE TABLE IF NOT EXISTS instance_external_access ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + instance_id INT NOT NULL, + enabled TINYINT(1) NOT NULL DEFAULT 0, + auth_mode VARCHAR(32) NOT NULL DEFAULT 'public_link', + public_slug VARCHAR(64) NULL, + public_token_hash VARCHAR(128) NULL, + short_code_hash VARCHAR(128) NULL, + api_key_hash VARCHAR(128) NULL, + password_value VARCHAR(128) NULL, + api_key_prefix VARCHAR(32) NULL, + expires_at DATETIME NULL, + created_by INT NULL, + last_used_at DATETIME NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY uk_instance_external_access_instance (instance_id), + UNIQUE KEY uk_instance_external_access_slug (public_slug), + UNIQUE KEY uk_instance_external_access_short_code (short_code_hash), + KEY idx_instance_external_access_enabled (enabled, auth_mode), + CONSTRAINT fk_instance_external_access_instance + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/backend/internal/db/migrations/028_add_ai_gateway_runtime_attribution.sql b/backend/internal/db/migrations/028_add_ai_gateway_runtime_attribution.sql new file mode 100644 index 0000000..1f5b6cc --- /dev/null +++ b/backend/internal/db/migrations/028_add_ai_gateway_runtime_attribution.sql @@ -0,0 +1,266 @@ +CREATE TABLE IF NOT EXISTS model_invocations ( + id INT AUTO_INCREMENT PRIMARY KEY, + trace_id VARCHAR(100) NOT NULL, + session_id VARCHAR(100) NULL, + request_id VARCHAR(100) NOT NULL, + user_id INT NULL, + instance_id INT NULL, + instance_mode VARCHAR(16) NULL, + runtime_type VARCHAR(32) NULL, + gateway_id VARCHAR(128) NULL, + runtime_pod_id BIGINT NULL, + model_id INT NULL, + provider_type VARCHAR(100) NOT NULL, + requested_model VARCHAR(255) NOT NULL, + actual_provider_model VARCHAR(255) NOT NULL, + traffic_class VARCHAR(50) NOT NULL, + request_payload LONGTEXT NULL, + response_payload LONGTEXT NULL, + prompt_tokens INT NOT NULL DEFAULT 0, + completion_tokens INT NOT NULL DEFAULT 0, + total_tokens INT NOT NULL DEFAULT 0, + cached_tokens INT NULL, + reasoning_tokens INT NULL, + latency_ms INT NULL, + is_streaming BOOLEAN NOT NULL DEFAULT FALSE, + status VARCHAR(50) NOT NULL, + error_message TEXT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + completed_at TIMESTAMP NULL, + INDEX idx_model_invocations_trace_id (trace_id), + INDEX idx_model_invocations_session_id (session_id), + INDEX idx_model_invocations_request_id (request_id), + INDEX idx_model_invocations_user_id (user_id), + INDEX idx_model_invocations_instance_id (instance_id), + INDEX idx_model_invocations_gateway_id (gateway_id), + INDEX idx_model_invocations_model_id (model_id), + INDEX idx_model_invocations_status (status), + INDEX idx_model_invocations_created_at (created_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS audit_events ( + id INT AUTO_INCREMENT PRIMARY KEY, + trace_id VARCHAR(100) NOT NULL, + session_id VARCHAR(100) NULL, + request_id VARCHAR(100) NULL, + user_id INT NULL, + instance_id INT NULL, + instance_mode VARCHAR(16) NULL, + runtime_type VARCHAR(32) NULL, + gateway_id VARCHAR(128) NULL, + runtime_pod_id BIGINT NULL, + invocation_id INT NULL, + event_type VARCHAR(100) NOT NULL, + traffic_class VARCHAR(50) NOT NULL, + severity VARCHAR(20) NOT NULL, + message VARCHAR(500) NOT NULL, + details LONGTEXT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + INDEX idx_audit_events_trace_id (trace_id), + INDEX idx_audit_events_request_id (request_id), + INDEX idx_audit_events_user_id (user_id), + INDEX idx_audit_events_gateway_id (gateway_id), + INDEX idx_audit_events_invocation_id (invocation_id), + INDEX idx_audit_events_event_type (event_type), + INDEX idx_audit_events_created_at (created_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS cost_records ( + id INT AUTO_INCREMENT PRIMARY KEY, + trace_id VARCHAR(100) NOT NULL, + session_id VARCHAR(100) NULL, + request_id VARCHAR(100) NULL, + user_id INT NULL, + instance_id INT NULL, + instance_mode VARCHAR(16) NULL, + runtime_type VARCHAR(32) NULL, + gateway_id VARCHAR(128) NULL, + runtime_pod_id BIGINT NULL, + invocation_id INT NULL, + model_id INT NULL, + provider_type VARCHAR(100) NOT NULL, + model_name VARCHAR(255) NOT NULL, + currency VARCHAR(16) NOT NULL DEFAULT 'USD', + prompt_tokens INT NOT NULL DEFAULT 0, + completion_tokens INT NOT NULL DEFAULT 0, + total_tokens INT NOT NULL DEFAULT 0, + input_unit_price DECIMAL(18,8) NOT NULL DEFAULT 0, + output_unit_price DECIMAL(18,8) NOT NULL DEFAULT 0, + estimated_cost DECIMAL(18,8) NOT NULL DEFAULT 0, + actual_cost DECIMAL(18,8) NULL, + internal_cost DECIMAL(18,8) NOT NULL DEFAULT 0, + recorded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + INDEX idx_cost_records_trace_id (trace_id), + INDEX idx_cost_records_user_id (user_id), + INDEX idx_cost_records_gateway_id (gateway_id), + INDEX idx_cost_records_model_id (model_id), + INDEX idx_cost_records_provider_type (provider_type), + INDEX idx_cost_records_recorded_at (recorded_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE IF NOT EXISTS risk_hits ( + id INT AUTO_INCREMENT PRIMARY KEY, + trace_id VARCHAR(100) NOT NULL, + session_id VARCHAR(100) NULL, + request_id VARCHAR(100) NULL, + user_id INT NULL, + instance_id INT NULL, + instance_mode VARCHAR(16) NULL, + runtime_type VARCHAR(32) NULL, + gateway_id VARCHAR(128) NULL, + runtime_pod_id BIGINT NULL, + invocation_id INT NULL, + rule_id VARCHAR(100) NOT NULL, + rule_name VARCHAR(255) NOT NULL, + severity VARCHAR(20) NOT NULL, + action VARCHAR(50) NOT NULL, + match_summary VARCHAR(500) NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + INDEX idx_risk_hits_trace_id (trace_id), + INDEX idx_risk_hits_request_id (request_id), + INDEX idx_risk_hits_user_id (user_id), + INDEX idx_risk_hits_gateway_id (gateway_id), + INDEX idx_risk_hits_invocation_id (invocation_id), + INDEX idx_risk_hits_severity (severity), + INDEX idx_risk_hits_created_at (created_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'model_invocations' AND COLUMN_NAME = 'instance_mode' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE model_invocations ADD COLUMN instance_mode VARCHAR(16) NULL AFTER instance_id', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'model_invocations' AND COLUMN_NAME = 'runtime_type' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE model_invocations ADD COLUMN runtime_type VARCHAR(32) NULL AFTER instance_mode', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'model_invocations' AND COLUMN_NAME = 'gateway_id' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE model_invocations ADD COLUMN gateway_id VARCHAR(128) NULL AFTER runtime_type', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'model_invocations' AND COLUMN_NAME = 'runtime_pod_id' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE model_invocations ADD COLUMN runtime_pod_id BIGINT NULL AFTER gateway_id', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'audit_events' AND COLUMN_NAME = 'instance_mode' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE audit_events ADD COLUMN instance_mode VARCHAR(16) NULL AFTER instance_id', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'audit_events' AND COLUMN_NAME = 'runtime_type' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE audit_events ADD COLUMN runtime_type VARCHAR(32) NULL AFTER instance_mode', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'audit_events' AND COLUMN_NAME = 'gateway_id' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE audit_events ADD COLUMN gateway_id VARCHAR(128) NULL AFTER runtime_type', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'audit_events' AND COLUMN_NAME = 'runtime_pod_id' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE audit_events ADD COLUMN runtime_pod_id BIGINT NULL AFTER gateway_id', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'cost_records' AND COLUMN_NAME = 'instance_mode' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE cost_records ADD COLUMN instance_mode VARCHAR(16) NULL AFTER instance_id', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'cost_records' AND COLUMN_NAME = 'runtime_type' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE cost_records ADD COLUMN runtime_type VARCHAR(32) NULL AFTER instance_mode', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'cost_records' AND COLUMN_NAME = 'gateway_id' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE cost_records ADD COLUMN gateway_id VARCHAR(128) NULL AFTER runtime_type', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'cost_records' AND COLUMN_NAME = 'runtime_pod_id' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE cost_records ADD COLUMN runtime_pod_id BIGINT NULL AFTER gateway_id', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'risk_hits' AND COLUMN_NAME = 'instance_mode' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE risk_hits ADD COLUMN instance_mode VARCHAR(16) NULL AFTER instance_id', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'risk_hits' AND COLUMN_NAME = 'runtime_type' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE risk_hits ADD COLUMN runtime_type VARCHAR(32) NULL AFTER instance_mode', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'risk_hits' AND COLUMN_NAME = 'gateway_id' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE risk_hits ADD COLUMN gateway_id VARCHAR(128) NULL AFTER runtime_type', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @column_exists = ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'risk_hits' AND COLUMN_NAME = 'runtime_pod_id' +); +SET @ddl = IF(@column_exists = 0, 'ALTER TABLE risk_hits ADD COLUMN runtime_pod_id BIGINT NULL AFTER gateway_id', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @index_exists = ( + SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'model_invocations' AND INDEX_NAME = 'idx_model_invocations_gateway_id' +); +SET @ddl = IF(@index_exists = 0, 'ALTER TABLE model_invocations ADD INDEX idx_model_invocations_gateway_id (gateway_id)', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @index_exists = ( + SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'audit_events' AND INDEX_NAME = 'idx_audit_events_gateway_id' +); +SET @ddl = IF(@index_exists = 0, 'ALTER TABLE audit_events ADD INDEX idx_audit_events_gateway_id (gateway_id)', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @index_exists = ( + SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'cost_records' AND INDEX_NAME = 'idx_cost_records_gateway_id' +); +SET @ddl = IF(@index_exists = 0, 'ALTER TABLE cost_records ADD INDEX idx_cost_records_gateway_id (gateway_id)', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @index_exists = ( + SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'risk_hits' AND INDEX_NAME = 'idx_risk_hits_gateway_id' +); +SET @ddl = IF(@index_exists = 0, 'ALTER TABLE risk_hits ADD INDEX idx_risk_hits_gateway_id (gateway_id)', 'SELECT 1'); +PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/backend/internal/db/migrations/029_add_short_external_access_links.sql b/backend/internal/db/migrations/029_add_short_external_access_links.sql new file mode 100644 index 0000000..291932e --- /dev/null +++ b/backend/internal/db/migrations/029_add_short_external_access_links.sql @@ -0,0 +1,47 @@ +SET @column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instance_external_access' + AND COLUMN_NAME = 'short_code_hash' +); +SET @ddl = IF( + @column_exists = 0, + 'ALTER TABLE instance_external_access ADD COLUMN short_code_hash VARCHAR(128) NULL AFTER public_token_hash', + 'SELECT 1' +); +PREPARE stmt FROM @ddl; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @public_slug_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instance_external_access' + AND COLUMN_NAME = 'public_slug' +); +SET @ddl = IF( + @public_slug_exists > 0, + 'ALTER TABLE instance_external_access MODIFY COLUMN public_slug VARCHAR(64) NULL', + 'SELECT 1' +); +PREPARE stmt FROM @ddl; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @index_exists = ( + SELECT COUNT(*) + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instance_external_access' + AND INDEX_NAME = 'uk_instance_external_access_short_code' +); +SET @ddl = IF( + @index_exists = 0, + 'ALTER TABLE instance_external_access ADD UNIQUE KEY uk_instance_external_access_short_code (short_code_hash)', + 'SELECT 1' +); +PREPARE stmt FROM @ddl; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; diff --git a/backend/internal/db/migrations/030_add_external_access_password_value.sql b/backend/internal/db/migrations/030_add_external_access_password_value.sql new file mode 100644 index 0000000..ee56933 --- /dev/null +++ b/backend/internal/db/migrations/030_add_external_access_password_value.sql @@ -0,0 +1,15 @@ +SET @column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'instance_external_access' + AND COLUMN_NAME = 'password_value' +); +SET @ddl = IF( + @column_exists = 0, + 'ALTER TABLE instance_external_access ADD COLUMN password_value VARCHAR(128) NULL AFTER api_key_hash', + 'SELECT 1' +); +PREPARE stmt FROM @ddl; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; diff --git a/backend/internal/db/migrations/031_add_hermes_lite_runtime_image.sql b/backend/internal/db/migrations/031_add_hermes_lite_runtime_image.sql new file mode 100644 index 0000000..fa7855f --- /dev/null +++ b/backend/internal/db/migrations/031_add_hermes_lite_runtime_image.sql @@ -0,0 +1,31 @@ +UPDATE system_image_settings +SET + image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest', + display_name = 'Hermes Lite', + is_enabled = TRUE +WHERE instance_type = 'hermes' + AND runtime_type = 'shell' + AND image IN ( + 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-shell:latest', + 'registry.example.com/your-custom-shell-image:latest' + ); + +INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) +SELECT + 'hermes', + 'shell', + 'Hermes Lite', + 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest', + TRUE +WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'hermes' + AND runtime_type = 'shell' +) +AND NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'hermes' + AND is_enabled = FALSE +); diff --git a/backend/internal/db/migrations/032_add_team_member_instance_mode.sql b/backend/internal/db/migrations/032_add_team_member_instance_mode.sql new file mode 100644 index 0000000..57d9dec --- /dev/null +++ b/backend/internal/db/migrations/032_add_team_member_instance_mode.sql @@ -0,0 +1,27 @@ +SET @team_member_instance_mode_column_exists = ( + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'team_members' + AND COLUMN_NAME = 'instance_mode' +); +SET @team_member_instance_mode_column_sql = IF( + @team_member_instance_mode_column_exists = 0, + 'ALTER TABLE team_members ADD COLUMN instance_mode ENUM(''lite'', ''pro'') NOT NULL DEFAULT ''lite'' AFTER runtime_type', + 'SELECT 1' +); +PREPARE team_member_instance_mode_column_stmt FROM @team_member_instance_mode_column_sql; +EXECUTE team_member_instance_mode_column_stmt; +DEALLOCATE PREPARE team_member_instance_mode_column_stmt; + +UPDATE team_members tm +LEFT JOIN instances i ON i.id = tm.instance_id +SET tm.instance_mode = CASE + WHEN i.instance_mode IN ('lite', 'pro') THEN i.instance_mode + WHEN i.runtime_type = 'gateway' THEN 'lite' + WHEN i.id IS NOT NULL THEN 'pro' + ELSE tm.instance_mode +END +WHERE tm.instance_mode IS NULL + OR tm.instance_mode = '' + OR (i.id IS NOT NULL AND tm.instance_mode <> COALESCE(i.instance_mode, CASE WHEN i.runtime_type = 'gateway' THEN 'lite' ELSE 'pro' END)); diff --git a/backend/internal/db/migrations/033_system_image_gateway_runtime_type.sql b/backend/internal/db/migrations/033_system_image_gateway_runtime_type.sql new file mode 100644 index 0000000..674c9f5 --- /dev/null +++ b/backend/internal/db/migrations/033_system_image_gateway_runtime_type.sql @@ -0,0 +1,26 @@ +ALTER TABLE system_image_settings + MODIFY COLUMN runtime_type ENUM('desktop', 'shell', 'gateway') NOT NULL DEFAULT 'desktop'; + +UPDATE system_image_settings +SET runtime_type = 'gateway', + display_name = 'OpenClaw Lite' +WHERE instance_type = 'openclaw' + AND runtime_type = 'shell'; + +UPDATE system_image_settings +SET runtime_type = 'gateway', + display_name = 'Hermes Lite' +WHERE instance_type = 'hermes' + AND runtime_type = 'shell'; + +UPDATE system_image_settings +SET display_name = 'OpenClaw Pro' +WHERE instance_type = 'openclaw' + AND runtime_type = 'desktop' + AND display_name IN ('OpenClaw Desktop', 'OpenClaw Runtime'); + +UPDATE system_image_settings +SET display_name = 'Hermes Pro' +WHERE instance_type = 'hermes' + AND runtime_type = 'desktop' + AND display_name IN ('Hermes Runtime', 'Hermes Desktop'); diff --git a/backend/internal/db/migrations/034_update_lite_default_images.sql b/backend/internal/db/migrations/034_update_lite_default_images.sql new file mode 100644 index 0000000..e819b76 --- /dev/null +++ b/backend/internal/db/migrations/034_update_lite_default_images.sql @@ -0,0 +1,54 @@ +UPDATE system_image_settings +SET + image = 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest', + display_name = 'OpenClaw Lite', + is_enabled = TRUE +WHERE instance_type = 'openclaw' + AND runtime_type IN ('shell', 'gateway') + AND image IN ( + 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-shell:latest', + '172.16.1.12:5010/openclaw-shell:local5', + '172.16.1.12:5010/openclaw:v2dev' + ); + +UPDATE system_image_settings +SET + image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest', + display_name = 'Hermes Lite', + is_enabled = TRUE +WHERE instance_type = 'hermes' + AND runtime_type IN ('shell', 'gateway') + AND image IN ( + 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-shell:latest', + '172.16.1.12:5010/hermes:codex-shared-agent-tui-bundle-20260609153019', + '172.16.1.12:5010/hermes:team-lite-tui-dist-20260612174205', + 'registry.example.com/your-custom-shell-image:latest' + ); + +INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) +SELECT + 'openclaw', + 'gateway', + 'OpenClaw Lite', + 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest', + TRUE +WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'openclaw' + AND runtime_type IN ('shell', 'gateway') +); + +INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) +SELECT + 'hermes', + 'gateway', + 'Hermes Lite', + 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest', + TRUE +WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'hermes' + AND runtime_type IN ('shell', 'gateway') +); diff --git a/backend/internal/db/migrations_test.go b/backend/internal/db/migrations_test.go index 9eec03e..16fe86f 100644 --- a/backend/internal/db/migrations_test.go +++ b/backend/internal/db/migrations_test.go @@ -2,6 +2,7 @@ package db import ( "reflect" + "strings" "testing" ) @@ -29,3 +30,69 @@ UPDATE demo SET note = "a;quoted" WHERE id = 1; t.Fatalf("unexpected statements:\nwant: %#v\ngot: %#v", want, got) } } + +func TestMigration023IsEmbedded(t *testing.T) { + files, err := embeddedMigrations.ReadDir("migrations") + if err != nil { + t.Fatalf("read embedded migrations: %v", err) + } + + found := false + for _, file := range files { + if file.Name() == "023_add_runtime_pool_v2.sql" { + found = true + break + } + } + if !found { + t.Fatalf("migration 023_add_runtime_pool_v2.sql is not embedded") + } +} + +func TestMigration034UpdatesLiteDefaultImages(t *testing.T) { + raw, err := embeddedMigrations.ReadFile("migrations/034_update_lite_default_images.sql") + if err != nil { + t.Fatalf("read migration 034: %v", err) + } + sql := string(raw) + for _, image := range []string{ + "ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest", + "ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest", + } { + if !strings.Contains(sql, image) { + t.Fatalf("migration 034 must update lite image %s", image) + } + } +} + +func TestMigration023IsRetrySafe(t *testing.T) { + raw, err := embeddedMigrations.ReadFile("migrations/023_add_runtime_pool_v2.sql") + if err != nil { + t.Fatalf("read migration 023: %v", err) + } + + sql := string(raw) + if !strings.Contains(sql, "information_schema.COLUMNS") { + t.Fatalf("migration 023 must guard instance column additions with information_schema.COLUMNS") + } + for _, column := range []string{ + "workspace_path", + "workspace_usage_bytes", + "runtime_generation", + "runtime_error_message", + } { + if !strings.Contains(sql, "COLUMN_NAME = '"+column+"'") { + t.Fatalf("migration 023 must guard %s column addition", column) + } + } + for _, table := range []string{ + "runtime_pods", + "instance_runtime_bindings", + "runtime_rollouts", + "workspace_file_audits", + } { + if !strings.Contains(sql, "CREATE TABLE IF NOT EXISTS "+table) { + t.Fatalf("migration 023 must create %s idempotently", table) + } + } +} diff --git a/backend/internal/db/runtime_manifest_test.go b/backend/internal/db/runtime_manifest_test.go new file mode 100644 index 0000000..36e6233 --- /dev/null +++ b/backend/internal/db/runtime_manifest_test.go @@ -0,0 +1,56 @@ +package db + +import ( + "os" + "path/filepath" + "regexp" + "strings" + "testing" +) + +func TestRuntimeManifestsStartHermesRuntime(t *testing.T) { + repoRoot := filepath.Clean(filepath.Join("..", "..", "..")) + for _, manifest := range []string{ + filepath.Join(repoRoot, "deployments", "k8s", "clawmanager.yaml"), + filepath.Join(repoRoot, "deployments", "k3s", "clawmanager.yaml"), + } { + t.Run(manifest, func(t *testing.T) { + raw, err := os.ReadFile(manifest) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + pattern := regexp.MustCompile(`(?s)name:\s+hermes-runtime.*?spec:\s+replicas:\s+([0-9]+)`) + matches := pattern.FindSubmatch(raw) + if len(matches) != 2 { + t.Fatalf("could not find hermes-runtime replicas in %s", manifest) + } + if string(matches[1]) != "1" { + t.Fatalf("expected hermes-runtime replicas 1 in %s, got %s", manifest, matches[1]) + } + }) + } +} + +func TestRuntimeManifestsSeedLiteDefaultImages(t *testing.T) { + repoRoot := filepath.Clean(filepath.Join("..", "..", "..")) + for _, manifest := range []string{ + filepath.Join(repoRoot, "deployments", "k8s", "clawmanager.yaml"), + filepath.Join(repoRoot, "deployments", "k3s", "clawmanager.yaml"), + filepath.Join(repoRoot, "backend", "deployments", "k8s", "clawreef-incluster.yaml"), + } { + t.Run(manifest, func(t *testing.T) { + raw, err := os.ReadFile(manifest) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + for _, image := range []string{ + "ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest", + "ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest", + } { + if !strings.Contains(string(raw), image) { + t.Fatalf("manifest %s must seed lite image %s", manifest, image) + } + } + }) + } +} diff --git a/backend/internal/handlers/ai_gateway_handler.go b/backend/internal/handlers/ai_gateway_handler.go index 2dee250..28e77d0 100644 --- a/backend/internal/handlers/ai_gateway_handler.go +++ b/backend/internal/handlers/ai_gateway_handler.go @@ -77,6 +77,13 @@ func (h *AIGatewayHandler) ChatCompletions(c *gin.Context) { } } } + if !setStringMetadata(c, &req.InstanceMode, "instanceMode") || + !setStringMetadata(c, &req.RuntimeType, "runtimeType") || + !setStringMetadata(c, &req.GatewayID, "gatewayID") || + !setInt64Metadata(c, &req.RuntimePodID, "runtimePodID") { + utils.Error(c, http.StatusForbidden, "Gateway token metadata does not match request") + return + } if req.Stream { traceID, err := h.service.StreamChatCompletions(c.Request.Context(), userID.(int), req, c.Writer) @@ -107,3 +114,47 @@ func (h *AIGatewayHandler) ChatCompletions(c *gin.Context) { c.Status(response.StatusCode) _, _ = c.Writer.Write(response.Body) } + +func setStringMetadata(c *gin.Context, field **string, key string) bool { + raw, exists := c.Get(key) + if !exists { + return true + } + value, ok := raw.(string) + if !ok { + return true + } + value = strings.TrimSpace(value) + if value == "" { + return true + } + if *field == nil { + *field = &value + return true + } + return strings.TrimSpace(**field) == value +} + +func setInt64Metadata(c *gin.Context, field **int64, key string) bool { + raw, exists := c.Get(key) + if !exists { + return true + } + var value int64 + switch typed := raw.(type) { + case int64: + value = typed + case int: + value = int64(typed) + default: + return true + } + if value <= 0 { + return true + } + if *field == nil { + *field = &value + return true + } + return **field == value +} diff --git a/backend/internal/handlers/instance_handler.go b/backend/internal/handlers/instance_handler.go index fc4733f..98dc1bf 100644 --- a/backend/internal/handlers/instance_handler.go +++ b/backend/internal/handlers/instance_handler.go @@ -3,8 +3,10 @@ package handlers import ( "errors" "fmt" + "html" "io" "net/http" + "net/url" "os" "strconv" "strings" @@ -57,10 +59,11 @@ type InstanceHandler struct { openClawTransferService services.OpenClawTransferService openClawConfigService services.OpenClawConfigService skillService services.SkillService + externalAccessService services.InstanceExternalAccessService } // NewInstanceHandler creates a new instance handler -func NewInstanceHandler(instanceService services.InstanceService, instanceAgentService services.InstanceAgentService, runtimeStatusService services.InstanceRuntimeStatusService, instanceCommandService services.InstanceCommandService, instanceConfigRevisionService services.InstanceConfigRevisionService, openClawConfigService services.OpenClawConfigService, skillService services.SkillService) *InstanceHandler { +func NewInstanceHandler(instanceService services.InstanceService, instanceAgentService services.InstanceAgentService, runtimeStatusService services.InstanceRuntimeStatusService, instanceCommandService services.InstanceCommandService, instanceConfigRevisionService services.InstanceConfigRevisionService, openClawConfigService services.OpenClawConfigService, skillService services.SkillService, externalAccessService services.InstanceExternalAccessService, proxyOptions ...services.InstanceProxyServiceOption) *InstanceHandler { accessService := services.NewInstanceAccessService() return &InstanceHandler{ instanceService: instanceService, @@ -69,11 +72,12 @@ func NewInstanceHandler(instanceService services.InstanceService, instanceAgentS instanceCommandService: instanceCommandService, instanceConfigRevisionService: instanceConfigRevisionService, accessService: accessService, - proxyService: services.NewInstanceProxyService(accessService), + proxyService: services.NewInstanceProxyService(accessService, proxyOptions...), shellService: services.NewInstanceShellService(), openClawTransferService: services.NewOpenClawTransferService(), openClawConfigService: openClawConfigService, skillService: skillService, + externalAccessService: externalAccessService, } } @@ -98,12 +102,20 @@ type PublishConfigRevisionRequest struct { SnapshotID int `json:"snapshot_id" binding:"required,min=1"` } +type ExternalAccessRequest struct { + ExpiresMode string `json:"expires_mode,omitempty"` + ExpiresPreset string `json:"expires_preset,omitempty"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` +} + // CreateInstanceRequest represents a create instance request type CreateInstanceRequest struct { Name string `json:"name" binding:"required,min=3,max=50"` Description *string `json:"description,omitempty"` - Type string `json:"type" binding:"required,oneof=openclaw ubuntu debian centos custom webtop hermes"` - RuntimeType string `json:"runtime_type" binding:"omitempty,oneof=desktop shell"` + Type string `json:"type" binding:"required,oneof=openclaw hermes"` + Mode string `json:"mode" binding:"omitempty,oneof=lite pro"` + InstanceMode string `json:"instance_mode" binding:"omitempty,oneof=lite pro"` + RuntimeType string `json:"runtime_type" binding:"omitempty,oneof=gateway desktop shell"` CPUCores float64 `json:"cpu_cores" binding:"required,min=0.1,max=32"` MemoryGB int `json:"memory_gb" binding:"required,min=1,max=128"` DiskGB int `json:"disk_gb" binding:"required,min=10,max=1000"` @@ -210,6 +222,8 @@ func (h *InstanceHandler) CreateInstance(c *gin.Context) { Name: req.Name, Description: req.Description, Type: req.Type, + Mode: req.Mode, + InstanceMode: req.InstanceMode, RuntimeType: req.RuntimeType, CPUCores: req.CPUCores, MemoryGB: req.MemoryGB, @@ -550,6 +564,13 @@ func (h *InstanceHandler) GetInstanceStatus(c *gin.Context) { return } + if userRole != "admin" { + utils.Success(c, http.StatusOK, "Instance status retrieved successfully", gin.H{ + "instance_status": buildUserSafeInstanceStatus(instance, status), + }) + return + } + runtime, _ := h.runtimeStatusService.GetByInstanceID(id) agent, _ := h.instanceAgentService.GetPayloadByInstanceID(id) @@ -560,6 +581,43 @@ func (h *InstanceHandler) GetInstanceStatus(c *gin.Context) { }) } +func buildUserSafeInstanceStatus(instance *models.Instance, status *services.InstanceStatus) map[string]interface{} { + if status == nil { + return map[string]interface{}{} + } + payload := map[string]interface{}{ + "instance_id": status.InstanceID, + "status": status.Status, + "availability": status.Availability, + "created_at": status.CreatedAt, + } + if payload["availability"] == "" { + payload["availability"] = availabilityForInstanceStatus(status.Status) + } + if status.StartedAt != nil { + payload["started_at"] = status.StartedAt + } + if instance == nil { + return payload + } + if agentType, ok := services.NormalizeV2RuntimeType(instance.Type); ok && (strings.EqualFold(strings.TrimSpace(instance.RuntimeType), "gateway") || strings.EqualFold(strings.TrimSpace(instance.InstanceMode), "lite")) { + payload["agent_type"] = agentType + payload["workspace_usage_bytes"] = instance.WorkspaceUsageBytes + } + return payload +} + +func availabilityForInstanceStatus(status string) string { + switch strings.ToLower(strings.TrimSpace(status)) { + case "running": + return "available" + case "creating": + return "starting" + default: + return "unavailable" + } +} + func (h *InstanceHandler) GetRuntimeDetails(c *gin.Context) { id, _, ok := h.resolveOwnedInstance(c) if !ok { @@ -780,7 +838,7 @@ func (h *InstanceHandler) GenerateAccessToken(c *gin.Context) { } // Generate proxy entry URL. The actual Service remains internal-only. - accessURL := h.proxyService.GetProxyURL(instance.ID, "") + accessURL := h.proxyService.GetProxyURLForInstance(instance, "") if accessURL == "" { utils.Error(c, http.StatusServiceUnavailable, "Unable to generate access URL") @@ -818,7 +876,7 @@ func (h *InstanceHandler) GenerateAccessToken(c *gin.Context) { response := map[string]interface{}{ "token": token.Token, "access_url": accessURL, - "proxy_url": h.proxyService.GetProxyURL(instance.ID, token.Token), + "proxy_url": h.proxyService.GetProxyURLForInstance(instance, token.Token), "expires_at": token.ExpiresAt, } @@ -930,41 +988,62 @@ func (h *InstanceHandler) ProxyInstance(c *gin.Context) { return } - // Get token from query parameter - token := c.Query("token") - if token == "" { - cookieToken, err := c.Cookie(fmt.Sprintf("instance_access_%d", id)) - if err != nil || cookieToken == "" { - utils.Error(c, http.StatusBadRequest, "Access token required") - return - } - token = cookieToken - } else { - // Promote the one-time query token into a cookie so iframe subresources and - // websocket requests can reuse it without appending the token everywhere. - c.SetCookie( - fmt.Sprintf("instance_access_%d", id), - token, - int(time.Hour.Seconds()), - fmt.Sprintf("/api/v1/instances/%d/proxy", id), - "", - false, - true, - ) + token, ok := h.proxyAccessToken(c, id) + if !ok { + return } + h.proxyInstanceWithToken(c, id, token) +} + +func (h *InstanceHandler) proxyAccessToken(c *gin.Context, id int) (string, bool) { + cookieName := fmt.Sprintf("instance_access_%d", id) + if cookieToken, err := c.Cookie(cookieName); err == nil && strings.TrimSpace(cookieToken) != "" { + if accessToken, validateErr := h.accessService.ValidateToken(cookieToken); validateErr == nil && accessToken.InstanceID == id { + return cookieToken, true + } + } + + queryToken := strings.TrimSpace(c.Query("token")) + if queryToken == "" { + utils.Error(c, http.StatusBadRequest, "Access token required") + return "", false + } + accessToken, err := h.accessService.ValidateToken(queryToken) + if err != nil || accessToken.InstanceID != id { + utils.Error(c, http.StatusUnauthorized, "Access token expired or invalid") + return "", false + } + + // Promote only a validated ClawManager access token. Runtime applications may + // also use a token query parameter for their own websocket/session protocol. + c.SetCookie( + cookieName, + queryToken, + int(time.Hour.Seconds()), + fmt.Sprintf("/api/v1/instances/%d/proxy", id), + "", + false, + true, + ) + return queryToken, true +} + +func (h *InstanceHandler) proxyInstanceWithToken(c *gin.Context, id int, token string) { // Check if it's a WebSocket upgrade request if strings.EqualFold(c.GetHeader("Upgrade"), "websocket") { - err = h.proxyService.ProxyWebSocket(c.Request.Context(), id, token, c.Writer, c.Request) - if err != nil { - http.Error(c.Writer, err.Error(), http.StatusBadGateway) + if err := h.proxyService.ProxyWebSocket(c.Request.Context(), id, token, c.Writer, c.Request); err != nil { + if errors.Is(err, services.ErrInstanceGatewayUnavailable) { + http.Error(c.Writer, "Instance gateway is not available", http.StatusServiceUnavailable) + } else { + http.Error(c.Writer, err.Error(), http.StatusBadGateway) + } } return } // Proxy regular HTTP request - err = h.proxyService.ProxyRequest(c.Request.Context(), id, token, c.Writer, c.Request) - if err != nil { + if err := h.proxyService.ProxyRequest(c.Request.Context(), id, token, c.Writer, c.Request); err != nil { // Log the error fmt.Printf("Proxy error for instance %d: %v\n", id, err) @@ -974,6 +1053,8 @@ func (h *InstanceHandler) ProxyInstance(c *gin.Context) { http.Error(c.Writer, "Access token expired or invalid", http.StatusUnauthorized) } else if err.Error() == "token does not match instance" { http.Error(c.Writer, "Token does not match instance", http.StatusForbidden) + } else if errors.Is(err, services.ErrInstanceGatewayUnavailable) { + http.Error(c.Writer, "Instance gateway is not available", http.StatusServiceUnavailable) } else { http.Error(c.Writer, fmt.Sprintf("Failed to proxy request: %v", err), http.StatusBadGateway) } @@ -1204,6 +1285,428 @@ func (h *InstanceHandler) requireOwnedInstance(c *gin.Context) (*models.Instance return instance, true } +func (h *InstanceHandler) GetExternalAccess(c *gin.Context) { + instance, ok := h.requireOwnedInstance(c) + if !ok { + return + } + if h.externalAccessService == nil { + utils.Error(c, http.StatusServiceUnavailable, "External access is not configured") + return + } + access, err := h.externalAccessService.Get(c.Request.Context(), instance.ID) + if err != nil { + utils.HandleError(c, err) + return + } + payload := gin.H{"external_access": access} + if shareURL := services.ExternalAccessShareURL(access); shareURL != "" { + payload["share_url"] = shareURL + } + if password := services.ExternalAccessPassword(access); password != "" { + payload["password"] = password + } + utils.Success(c, http.StatusOK, "External access retrieved successfully", payload) +} + +func (h *InstanceHandler) EnableShareLink(c *gin.Context) { + instance, ok := h.requireOwnedInstance(c) + if !ok { + return + } + if h.externalAccessService == nil { + utils.Error(c, http.StatusServiceUnavailable, "External access is not configured") + return + } + var req ExternalAccessRequest + if err := c.ShouldBindJSON(&req); err != nil && err != io.EOF { + utils.ValidationError(c, err) + return + } + userID, _ := c.Get("userID") + result, err := h.externalAccessService.EnableShareLink(c.Request.Context(), instance.ID, userID.(int), externalAccessExpirationRequest(req)) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Share link enabled successfully", result) +} + +func (h *InstanceHandler) CreateExternalAccessPassword(c *gin.Context) { + instance, ok := h.requireOwnedInstance(c) + if !ok { + return + } + if h.externalAccessService == nil { + utils.Error(c, http.StatusServiceUnavailable, "External access is not configured") + return + } + var req ExternalAccessRequest + if err := c.ShouldBindJSON(&req); err != nil && err != io.EOF { + utils.ValidationError(c, err) + return + } + userID, _ := c.Get("userID") + result, err := h.externalAccessService.CreatePassword(c.Request.Context(), instance.ID, userID.(int), externalAccessExpirationRequest(req)) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Share link password created successfully", result) +} + +func (h *InstanceHandler) DisableExternalAccess(c *gin.Context) { + instance, ok := h.requireOwnedInstance(c) + if !ok { + return + } + if h.externalAccessService == nil { + utils.Error(c, http.StatusServiceUnavailable, "External access is not configured") + return + } + if err := h.externalAccessService.Disable(c.Request.Context(), instance.ID); err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "External access disabled successfully", nil) +} + +func (h *InstanceHandler) OpenShortExternalAccess(c *gin.Context) { + if h.externalAccessService == nil { + utils.Error(c, http.StatusServiceUnavailable, "External access is not configured") + return + } + code := strings.TrimSpace(c.Param("code")) + access, err := h.externalAccessService.ResolveShortLink(c.Request.Context(), code) + if err != nil { + utils.Error(c, http.StatusUnauthorized, err.Error()) + return + } + + var instance *models.Instance + var token string + canonicalAccessURL := "" + switch access.AuthMode { + case services.ExternalAccessModeShareLink: + if _, err := h.externalAccessService.ValidateShortLink(c.Request.Context(), code, ""); err != nil { + utils.Error(c, http.StatusUnauthorized, err.Error()) + return + } + case services.ExternalAccessModePassword: + token = h.validShortLinkAccessToken(c, code, access.InstanceID) + if token == "" { + password := externalPassword(c) + isPasswordFormPost := c.Request.Method == http.MethodPost && password == "" + if isPasswordFormPost { + password = c.PostForm("password") + } + if strings.TrimSpace(password) == "" { + if c.Request.Method == http.MethodGet || c.Request.Method == http.MethodHead { + renderShortLinkPasswordForm(c, code, "") + return + } + utils.Error(c, http.StatusUnauthorized, "share link password is required") + return + } + if _, err := h.externalAccessService.ValidateShortLink(c.Request.Context(), code, password); err != nil { + if c.Request.Method == http.MethodPost || shortExternalAccessWantsHTML(c) { + renderShortLinkPasswordForm(c, code, "Invalid password") + return + } + utils.Error(c, http.StatusUnauthorized, err.Error()) + return + } + var ok bool + instance, ok = h.requireExternalAccessInstance(c, access) + if !ok { + return + } + instanceToken, ok := h.issueShortExternalAccessToken(c, instance, code) + if !ok { + return + } + token = instanceToken.Token + canonicalAccessURL = instanceToken.AccessURL + if c.Request.Method == http.MethodPost { + c.Redirect(http.StatusSeeOther, shortExternalAccessEntryPath(code)) + return + } + } + default: + utils.Error(c, http.StatusBadRequest, "Unsupported share link mode") + return + } + + if instance == nil { + var ok bool + instance, ok = h.requireExternalAccessInstance(c, access) + if !ok { + return + } + } + if token == "" { + instanceToken, ok := h.issueShortExternalAccessToken(c, instance, code) + if !ok { + return + } + token = instanceToken.Token + canonicalAccessURL = instanceToken.AccessURL + } else { + setShortExternalAccessCookies(c, instance.ID, code, token, int(time.Hour.Seconds())) + canonicalAccessURL = h.proxyService.GetProxyURLForInstance(instance, "") + } + + originalPath := c.Request.URL.Path + if redirectTarget := shortExternalAccessEntryRedirectTarget(c.Request.Method, originalPath, code, canonicalAccessURL); redirectTarget != "" { + c.Redirect(http.StatusSeeOther, redirectTarget) + return + } + + originalRawPath := c.Request.URL.RawPath + originalAuthorization := c.Request.Header.Get("Authorization") + originalPassword := c.Request.Header.Get("X-Password") + c.Request.URL.Path = shortExternalAccessProxyPath(originalPath, code, instance.ID) + c.Request.URL.RawPath = "" + c.Request.Header.Del("Authorization") + c.Request.Header.Del("X-Password") + defer func() { + c.Request.URL.Path = originalPath + c.Request.URL.RawPath = originalRawPath + if originalAuthorization != "" { + c.Request.Header.Set("Authorization", originalAuthorization) + } + if originalPassword != "" { + c.Request.Header.Set("X-Password", originalPassword) + } + }() + + h.proxyInstanceWithToken(c, instance.ID, token) +} + +func bearerToken(header string) string { + value := strings.TrimSpace(header) + if value == "" { + return "" + } + if strings.HasPrefix(strings.ToLower(value), "bearer ") { + return strings.TrimSpace(value[len("bearer "):]) + } + return value +} + +func externalPassword(c *gin.Context) string { + if password := strings.TrimSpace(c.GetHeader("X-Password")); password != "" { + return password + } + return bearerToken(c.GetHeader("Authorization")) +} + +func externalAccessExpirationRequest(req ExternalAccessRequest) services.ExternalAccessExpirationRequest { + return services.ExternalAccessExpirationRequest{ + Mode: strings.TrimSpace(req.ExpiresMode), + Preset: strings.TrimSpace(req.ExpiresPreset), + ExpiresAt: req.ExpiresAt, + } +} + +func (h *InstanceHandler) requireExternalAccessInstance(c *gin.Context, access *models.InstanceExternalAccess) (*models.Instance, bool) { + if access == nil { + utils.Error(c, http.StatusUnauthorized, "External access is not enabled") + return nil, false + } + instance, err := h.instanceService.GetByID(access.InstanceID) + if err != nil { + utils.HandleError(c, err) + return nil, false + } + if instance == nil { + utils.Error(c, http.StatusNotFound, "Instance not found") + return nil, false + } + if instance.Status != "running" { + utils.Error(c, http.StatusServiceUnavailable, "Instance is not running") + return nil, false + } + if strings.EqualFold(strings.TrimSpace(instance.RuntimeType), "shell") { + utils.Error(c, http.StatusBadRequest, "External desktop access is not available for shell runtime instances") + return nil, false + } + return instance, true +} + +func (h *InstanceHandler) issueShortExternalAccessToken(c *gin.Context, instance *models.Instance, code string) (*services.AccessToken, bool) { + accessURL := h.proxyService.GetProxyURLForInstance(instance, "") + if accessURL == "" { + utils.Error(c, http.StatusServiceUnavailable, "Unable to generate access URL") + return nil, false + } + instanceToken, err := h.accessService.GenerateToken( + instance.UserID, + instance.ID, + instance.Type, + accessURL, + h.proxyService.GetTargetPortForInstance(instance), + 1*time.Hour, + ) + if err != nil { + utils.HandleError(c, err) + return nil, false + } + setShortExternalAccessCookies(c, instance.ID, code, instanceToken.Token, int(time.Hour.Seconds())) + return instanceToken, true +} + +func (h *InstanceHandler) validShortLinkAccessToken(c *gin.Context, code string, instanceID int) string { + if h == nil || h.accessService == nil { + return "" + } + token, err := c.Cookie(shortExternalAccessCookieName(code)) + if err != nil || strings.TrimSpace(token) == "" { + return "" + } + accessToken, err := h.accessService.ValidateToken(token) + if err != nil || accessToken.InstanceID != instanceID { + return "" + } + return token +} + +func setShortExternalAccessCookies(c *gin.Context, instanceID int, code, token string, maxAge int) { + c.SetCookie( + fmt.Sprintf("instance_access_%d", instanceID), + token, + maxAge, + fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID), + "", + false, + true, + ) + c.SetCookie( + shortExternalAccessCookieName(code), + token, + maxAge, + shortExternalAccessCookiePath(code), + "", + false, + true, + ) +} + +func shortExternalAccessCookieName(code string) string { + code = strings.Trim(strings.TrimSpace(code), "/") + var builder strings.Builder + for _, r := range code { + switch { + case r >= 'a' && r <= 'z': + builder.WriteRune(r) + case r >= 'A' && r <= 'Z': + builder.WriteRune(r) + case r >= '0' && r <= '9': + builder.WriteRune(r) + case r == '_' || r == '-': + builder.WriteRune(r) + default: + builder.WriteByte('_') + } + } + if builder.Len() == 0 { + return "share_link_access" + } + return "share_link_access_" + builder.String() +} + +func shortExternalAccessCookiePath(code string) string { + code = strings.Trim(strings.TrimSpace(code), "/") + if code == "" { + return "/s" + } + return "/s/" + code +} + +func shortExternalAccessEntryPath(code string) string { + return shortExternalAccessCookiePath(code) + "/" +} + +func shortExternalAccessEntryRedirectTarget(method, requestPath, code, canonicalPath string) string { + if method != http.MethodGet && method != http.MethodHead { + return "" + } + path := strings.TrimSpace(requestPath) + entryPath := shortExternalAccessEntryPath(code) + if path != entryPath && path != strings.TrimSuffix(entryPath, "/") { + return "" + } + target := strings.TrimSpace(canonicalPath) + if target == "" { + return "" + } + parsed, err := url.Parse(target) + if err != nil || parsed.Path == "" { + return "" + } + if !strings.HasPrefix(parsed.Path, "/api/v1/instances/") { + return "" + } + return parsed.Path +} + +func shortExternalAccessWantsHTML(c *gin.Context) bool { + accept := strings.ToLower(c.GetHeader("Accept")) + return strings.Contains(accept, "text/html") +} + +func renderShortLinkPasswordForm(c *gin.Context, code, errorMessage string) { + action := html.EscapeString(shortExternalAccessEntryPath(code)) + errorHTML := "" + if strings.TrimSpace(errorMessage) != "" { + errorHTML = fmt.Sprintf(`
%s
`, html.EscapeString(errorMessage)) + } + body := fmt.Sprintf(` + + + + + Share link password + + + +
+

Share link password

+ %s +
+ + + +
+
+ +`, errorHTML, action) + c.Data(http.StatusOK, "text/html; charset=utf-8", []byte(body)) +} + +func shortExternalAccessProxyPath(requestPath, code string, instanceID int) string { + internalPrefix := fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID) + shortPrefix := fmt.Sprintf("/s/%s", strings.Trim(strings.TrimSpace(code), "/")) + path := strings.TrimSpace(requestPath) + if path == "" || path == shortPrefix || path == shortPrefix+"/" { + return internalPrefix + "/" + } + if strings.HasPrefix(path, shortPrefix+"/") { + return internalPrefix + strings.TrimPrefix(path, shortPrefix) + } + return internalPrefix + "/" +} + func sanitizeDownloadName(name string, fallback ...string) string { name = strings.TrimSpace(name) if name == "" { diff --git a/backend/internal/handlers/instance_handler_test.go b/backend/internal/handlers/instance_handler_test.go index 684871b..b1a5342 100644 --- a/backend/internal/handlers/instance_handler_test.go +++ b/backend/internal/handlers/instance_handler_test.go @@ -1,6 +1,17 @@ package handlers -import "testing" +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "clawreef/internal/models" + "clawreef/internal/services" + + "github.com/gin-gonic/gin" +) func TestWorkspaceArchiveMaxMiB(t *testing.T) { t.Setenv(workspaceArchiveMaxMiBEnv, "") @@ -23,3 +34,198 @@ func TestWorkspaceArchiveMaxMiB(t *testing.T) { t.Fatalf("expected unparsable archive limit to fall back to %d MiB, got %d", defaultWorkspaceArchiveMaxMiB, got) } } + +func TestBuildUserSafeInstanceStatusHidesRuntimeSchedulingDetails(t *testing.T) { + podName := "runtime-openclaw-abc" + podNamespace := "clawmanager-system" + podIP := "10.42.0.12" + startedAt := time.Now().UTC() + workspacePath := "/workspaces/openclaw/user-45/instance-123" + instance := &models.Instance{ + ID: 123, + Type: "openclaw", + RuntimeType: "gateway", + Status: "running", + WorkspacePath: &workspacePath, + WorkspaceUsageBytes: 123456, + } + status := &services.InstanceStatus{ + InstanceID: 123, + Status: "running", + PodName: &podName, + PodNamespace: &podNamespace, + PodIP: &podIP, + PodStatus: "Running", + StartedAt: &startedAt, + } + + payload := buildUserSafeInstanceStatus(instance, status) + + for _, forbiddenKey := range []string{"pod_name", "pod_namespace", "pod_ip", "pod_status", "gateway_port", "capacity", "node_name"} { + if _, exists := payload[forbiddenKey]; exists { + t.Fatalf("user-safe status exposed %q: %#v", forbiddenKey, payload) + } + } + if payload["availability"] != "available" { + t.Fatalf("availability = %v, want available", payload["availability"]) + } + if payload["agent_type"] != "openclaw" { + t.Fatalf("agent_type = %v, want openclaw", payload["agent_type"]) + } + if payload["workspace_usage_bytes"] != int64(123456) { + t.Fatalf("workspace_usage_bytes = %v, want 123456", payload["workspace_usage_bytes"]) + } +} + +func TestShortExternalAccessProxyPath(t *testing.T) { + tests := []struct { + name string + requestPath string + want string + }{ + {name: "entry without slash", requestPath: "/s/abc123", want: "/api/v1/instances/71/proxy/"}, + {name: "entry with slash", requestPath: "/s/abc123/", want: "/api/v1/instances/71/proxy/"}, + {name: "asset", requestPath: "/s/abc123/assets/index.js", want: "/api/v1/instances/71/proxy/assets/index.js"}, + {name: "nested", requestPath: "/s/abc123/apps/openclaw/settings", want: "/api/v1/instances/71/proxy/apps/openclaw/settings"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := shortExternalAccessProxyPath(tt.requestPath, "abc123", 71); got != tt.want { + t.Fatalf("shortExternalAccessProxyPath(%q) = %q, want %q", tt.requestPath, got, tt.want) + } + }) + } +} + +func TestShortExternalAccessEntryRedirectTarget(t *testing.T) { + tests := []struct { + name string + method string + requestPath string + code string + canonicalPath string + want string + }{ + { + name: "html entry redirects to canonical proxy path", + method: http.MethodGet, + requestPath: "/s/sl_abc123/", + code: "sl_abc123", + canonicalPath: "/api/v1/instances/71/proxy/chat/", + want: "/api/v1/instances/71/proxy/chat/", + }, + { + name: "entry without trailing slash redirects", + method: http.MethodGet, + requestPath: "/s/sl_abc123", + code: "sl_abc123", + canonicalPath: "/api/v1/instances/71/proxy/chat/", + want: "/api/v1/instances/71/proxy/chat/", + }, + { + name: "asset path keeps short proxy handling", + method: http.MethodGet, + requestPath: "/s/sl_abc123/assets/index.js", + code: "sl_abc123", + canonicalPath: "/api/v1/instances/71/proxy/chat/", + want: "", + }, + { + name: "post keeps password form handling", + method: http.MethodPost, + requestPath: "/s/sl_abc123/", + code: "sl_abc123", + canonicalPath: "/api/v1/instances/71/proxy/chat/", + want: "", + }, + { + name: "target strips accidental token query", + method: http.MethodGet, + requestPath: "/s/sl_abc123/", + code: "sl_abc123", + canonicalPath: "/api/v1/instances/71/proxy/chat/?token=secret", + want: "/api/v1/instances/71/proxy/chat/", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := shortExternalAccessEntryRedirectTarget(tt.method, tt.requestPath, tt.code, tt.canonicalPath); got != tt.want { + t.Fatalf("shortExternalAccessEntryRedirectTarget() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestRenderShortLinkPasswordForm(t *testing.T) { + gin.SetMode(gin.TestMode) + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/s/sl_abc123/", nil) + + renderShortLinkPasswordForm(c, "sl_abc123", "") + + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, want %d", recorder.Code, http.StatusOK) + } + body := recorder.Body.String() + for _, want := range []string{ + "Share link password", + `name="password"`, + `type="password"`, + `action="/s/sl_abc123/"`, + } { + if !strings.Contains(body, want) { + t.Fatalf("password form missing %q in body:\n%s", want, body) + } + } + if contentType := recorder.Header().Get("Content-Type"); !strings.Contains(contentType, "text/html") { + t.Fatalf("content type = %q, want text/html", contentType) + } +} + +func TestProxyAccessTokenPrefersCookieOverRuntimeQueryToken(t *testing.T) { + gin.SetMode(gin.TestMode) + accessService := services.NewInstanceAccessService() + defer accessService.Stop() + token, err := accessService.GenerateToken(1, 76, "hermes", "/api/v1/instances/76/proxy/chat/", 3000, time.Hour) + if err != nil { + t.Fatalf("GenerateToken returned error: %v", err) + } + + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/instances/76/proxy/api/ws?token=hermes-session-token", nil) + c.Request.AddCookie(&http.Cookie{Name: "instance_access_76", Value: token.Token}) + + handler := &InstanceHandler{accessService: accessService} + got, ok := handler.proxyAccessToken(c, 76) + if !ok { + t.Fatal("proxyAccessToken rejected valid cookie") + } + if got != token.Token { + t.Fatalf("proxyAccessToken = %q, want cookie token", got) + } + if recorder.Code != http.StatusOK { + t.Fatalf("unexpected response status %d", recorder.Code) + } +} + +func TestProxyAccessTokenRejectsRuntimeQueryTokenWithoutCookie(t *testing.T) { + gin.SetMode(gin.TestMode) + accessService := services.NewInstanceAccessService() + defer accessService.Stop() + + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/instances/76/proxy/api/ws?token=hermes-session-token", nil) + + handler := &InstanceHandler{accessService: accessService} + if got, ok := handler.proxyAccessToken(c, 76); ok || got != "" { + t.Fatalf("proxyAccessToken = %q/%v, want rejected", got, ok) + } + if recorder.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want %d", recorder.Code, http.StatusUnauthorized) + } +} diff --git a/backend/internal/handlers/runtime_agent_handler.go b/backend/internal/handlers/runtime_agent_handler.go new file mode 100644 index 0000000..b29c66c --- /dev/null +++ b/backend/internal/handlers/runtime_agent_handler.go @@ -0,0 +1,373 @@ +package handlers + +import ( + "context" + "encoding/json" + "net/http" + "strings" + "time" + + "clawreef/internal/config" + "clawreef/internal/models" + "clawreef/internal/repository" + "clawreef/internal/services" + "clawreef/internal/utils" + + "github.com/gin-gonic/gin" +) + +const runtimeAgentTokenHeader = "X-ClawManager-Agent-Token" + +type runtimeEventPublisher interface { + Publish(ctx context.Context, eventType string, payload any) error +} + +type RuntimeAgentHandler struct { + cfg config.RuntimePoolConfig + podRepo repository.RuntimePodRepository + bindingRepo repository.InstanceRuntimeBindingRepository + events runtimeEventPublisher +} + +type runtimeAgentPodIdentity struct { + PodID int64 `json:"pod_id"` + Namespace string `json:"namespace"` + PodName string `json:"pod_name"` +} + +type runtimeAgentRegisterRequest struct { + RuntimeType string `json:"runtime_type" binding:"required"` + Namespace string `json:"namespace" binding:"required"` + PodName string `json:"pod_name" binding:"required"` + PodUID *string `json:"pod_uid,omitempty"` + PodIP *string `json:"pod_ip,omitempty"` + NodeName *string `json:"node_name,omitempty"` + DeploymentName string `json:"deployment_name" binding:"required"` + ImageRef string `json:"image_ref" binding:"required"` + AgentEndpoint *string `json:"agent_endpoint,omitempty"` + State string `json:"state"` + Capacity int `json:"capacity"` + UsedSlots int `json:"used_slots"` + Draining bool `json:"draining"` + Metrics json.RawMessage `json:"metrics,omitempty"` + ReportedAt *time.Time `json:"reported_at,omitempty"` +} + +type runtimeAgentHeartbeatRequest struct { + runtimeAgentPodIdentity + State string `json:"state" binding:"required"` + UsedSlots int `json:"used_slots"` + Draining bool `json:"draining"` + ReportedAt *time.Time `json:"reported_at,omitempty"` +} + +type runtimeAgentMetricsRequest struct { + runtimeAgentPodIdentity + CPUMillisUsed int64 `json:"cpu_millis_used"` + MemoryBytesUsed int64 `json:"memory_bytes_used"` + DiskBytesUsed int64 `json:"disk_bytes_used"` + NetworkRXBytes int64 `json:"network_rx_bytes"` + NetworkTXBytes int64 `json:"network_tx_bytes"` + Metrics json.RawMessage `json:"metrics,omitempty"` + ReportedAt *time.Time `json:"reported_at,omitempty"` +} + +type runtimeAgentGatewaysRequest struct { + runtimeAgentPodIdentity + Gateways []runtimeAgentGatewayReport `json:"gateways" binding:"required"` +} + +type runtimeAgentGatewayReport struct { + InstanceID int `json:"instance_id" binding:"required"` + GatewayID string `json:"gateway_id"` + GatewayPort int `json:"gateway_port"` + GatewayPID *int `json:"gateway_pid,omitempty"` + State string `json:"state" binding:"required"` + Generation int `json:"generation" binding:"required"` + ErrorMessage *string `json:"error_message,omitempty"` + HealthAt *time.Time `json:"health_at,omitempty"` +} + +func NewRuntimeAgentHandler(cfg config.RuntimePoolConfig, podRepo repository.RuntimePodRepository, bindingRepo repository.InstanceRuntimeBindingRepository, events runtimeEventPublisher) *RuntimeAgentHandler { + return &RuntimeAgentHandler{ + cfg: cfg, + podRepo: podRepo, + bindingRepo: bindingRepo, + events: events, + } +} + +func (h *RuntimeAgentHandler) Register(c *gin.Context) { + if !h.requireAgentToken(c) { + return + } + var req runtimeAgentRegisterRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + runtimeType, ok := services.NormalizeV2RuntimeType(req.RuntimeType) + if !ok { + utils.Error(c, http.StatusBadRequest, "unsupported runtime type") + return + } + state := strings.TrimSpace(req.State) + if state == "" { + state = "ready" + } + capacity := runtimePodCapacityFromReport(req.Capacity, h.cfg.MaxGatewaysPerPod) + lastSeen := time.Now().UTC() + if req.ReportedAt != nil && !req.ReportedAt.IsZero() { + lastSeen = req.ReportedAt.UTC() + } + var metricsJSON *string + if len(req.Metrics) > 0 { + if !json.Valid(req.Metrics) { + utils.Error(c, http.StatusBadRequest, "metrics must be valid JSON") + return + } + raw := string(req.Metrics) + metricsJSON = &raw + } + pod := &models.RuntimePod{ + RuntimeType: runtimeType, + Namespace: strings.TrimSpace(req.Namespace), + PodName: strings.TrimSpace(req.PodName), + PodUID: trimStringPtr(req.PodUID), + PodIP: trimStringPtr(req.PodIP), + NodeName: trimStringPtr(req.NodeName), + DeploymentName: strings.TrimSpace(req.DeploymentName), + ImageRef: strings.TrimSpace(req.ImageRef), + AgentEndpoint: trimStringPtr(req.AgentEndpoint), + State: state, + Capacity: capacity, + UsedSlots: req.UsedSlots, + Draining: req.Draining, + MetricsJSON: metricsJSON, + LastSeenAt: &lastSeen, + CPUMillisUsed: 0, + MemoryBytesUsed: 0, + DiskBytesUsed: 0, + NetworkRXBytes: 0, + NetworkTXBytes: 0, + } + if err := h.podRepo.UpsertFromAgent(c.Request.Context(), pod); err != nil { + utils.HandleError(c, err) + return + } + h.publish(c.Request.Context(), "runtime_pod_state", map[string]any{ + "pod_id": pod.ID, + "runtime_type": runtimeType, + "namespace": pod.Namespace, + "pod_name": pod.PodName, + "state": state, + "used_slots": req.UsedSlots, + "capacity": capacity, + "draining": req.Draining, + "last_seen_at": lastSeen, + }) + utils.Success(c, http.StatusOK, "Runtime pod registered successfully", gin.H{"pod": pod}) +} + +func (h *RuntimeAgentHandler) Heartbeat(c *gin.Context) { + if !h.requireAgentToken(c) { + return + } + var req runtimeAgentHeartbeatRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + podID, ok := h.resolvePodID(c, req.runtimeAgentPodIdentity) + if !ok { + return + } + lastSeen := time.Now().UTC() + if req.ReportedAt != nil && !req.ReportedAt.IsZero() { + lastSeen = req.ReportedAt.UTC() + } + capacity := runtimePodCapacityFromReport(0, h.cfg.MaxGatewaysPerPod) + if err := h.podRepo.UpdateHeartbeat(c.Request.Context(), podID, strings.TrimSpace(req.State), req.UsedSlots, capacity, req.Draining, lastSeen); err != nil { + utils.HandleError(c, err) + return + } + h.publish(c.Request.Context(), "runtime_pod_state", map[string]any{ + "pod_id": podID, + "state": strings.TrimSpace(req.State), + "used_slots": req.UsedSlots, + "capacity": capacity, + "draining": req.Draining, + "last_seen_at": lastSeen, + }) + utils.Success(c, http.StatusOK, "Runtime pod heartbeat accepted", nil) +} + +func (h *RuntimeAgentHandler) ReportMetrics(c *gin.Context) { + if !h.requireAgentToken(c) { + return + } + var req runtimeAgentMetricsRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + podID, ok := h.resolvePodID(c, req.runtimeAgentPodIdentity) + if !ok { + return + } + var metricsJSON *string + if len(req.Metrics) > 0 { + if !json.Valid(req.Metrics) { + utils.Error(c, http.StatusBadRequest, "metrics must be valid JSON") + return + } + raw := string(req.Metrics) + metricsJSON = &raw + } + lastSeen := time.Now().UTC() + if req.ReportedAt != nil && !req.ReportedAt.IsZero() { + lastSeen = req.ReportedAt.UTC() + } + update := repository.RuntimePodMetricsUpdate{ + CPUMillisUsed: req.CPUMillisUsed, + MemoryBytesUsed: req.MemoryBytesUsed, + DiskBytesUsed: req.DiskBytesUsed, + NetworkRXBytes: req.NetworkRXBytes, + NetworkTXBytes: req.NetworkTXBytes, + MetricsJSON: metricsJSON, + LastSeenAt: &lastSeen, + } + if err := h.podRepo.UpdateMetrics(c.Request.Context(), podID, update); err != nil { + utils.HandleError(c, err) + return + } + payload := map[string]any{ + "pod_id": podID, + "cpu_millis_used": req.CPUMillisUsed, + "memory_bytes_used": req.MemoryBytesUsed, + "disk_bytes_used": req.DiskBytesUsed, + "network_rx_bytes": req.NetworkRXBytes, + "network_tx_bytes": req.NetworkTXBytes, + "last_seen_at": lastSeen, + } + if metricsJSON != nil { + payload["metrics"] = json.RawMessage(*metricsJSON) + } + h.publish(c.Request.Context(), "runtime_pod_metrics", payload) + utils.Success(c, http.StatusOK, "Runtime pod metrics accepted", nil) +} + +func (h *RuntimeAgentHandler) ReportGateways(c *gin.Context) { + if !h.requireAgentToken(c) { + return + } + var req runtimeAgentGatewaysRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + podID, ok := h.resolvePodID(c, req.runtimeAgentPodIdentity) + if !ok { + return + } + for _, gateway := range req.Gateways { + binding, err := h.bindingRepo.GetByInstanceID(c.Request.Context(), gateway.InstanceID) + if err != nil { + utils.HandleError(c, err) + return + } + if binding == nil || binding.RuntimePodID != podID || binding.Generation != gateway.Generation { + continue + } + state := strings.TrimSpace(gateway.State) + switch state { + case "running", "healthy": + if err := h.bindingRepo.UpdateRunning(c.Request.Context(), gateway.InstanceID, gateway.Generation, strings.TrimSpace(gateway.GatewayID), gateway.GatewayPort, gateway.GatewayPID); err != nil { + utils.HandleError(c, err) + return + } + default: + if err := h.bindingRepo.UpdateState(c.Request.Context(), gateway.InstanceID, gateway.Generation, state, gateway.ErrorMessage); err != nil { + utils.HandleError(c, err) + return + } + } + } + h.publish(c.Request.Context(), "runtime_pod_gateways_reported", map[string]any{ + "pod_id": podID, + "gateway_count": len(req.Gateways), + }) + utils.Success(c, http.StatusOK, "Runtime gateway report accepted", nil) +} + +func (h *RuntimeAgentHandler) ReportSkills(c *gin.Context) { + if !h.requireAgentToken(c) { + return + } + var payload map[string]any + if err := c.ShouldBindJSON(&payload); err != nil { + utils.ValidationError(c, err) + return + } + h.publish(c.Request.Context(), "runtime_agent_skills_reported", payload) + utils.Success(c, http.StatusOK, "Runtime agent skills report accepted", nil) +} + +func (h *RuntimeAgentHandler) requireAgentToken(c *gin.Context) bool { + if strings.TrimSpace(h.cfg.AgentReportToken) == "" || c.GetHeader(runtimeAgentTokenHeader) != h.cfg.AgentReportToken { + utils.Error(c, http.StatusUnauthorized, "invalid runtime agent token") + return false + } + return true +} + +func (h *RuntimeAgentHandler) resolvePodID(c *gin.Context, identity runtimeAgentPodIdentity) (int64, bool) { + if identity.PodID > 0 { + return identity.PodID, true + } + namespace := strings.TrimSpace(identity.Namespace) + podName := strings.TrimSpace(identity.PodName) + if namespace == "" || podName == "" { + utils.Error(c, http.StatusBadRequest, "pod_id or namespace and pod_name are required") + return 0, false + } + pod, err := h.podRepo.GetByNamespaceName(c.Request.Context(), namespace, podName) + if err != nil { + utils.HandleError(c, err) + return 0, false + } + if pod == nil { + utils.Error(c, http.StatusNotFound, "runtime pod not found") + return 0, false + } + return pod.ID, true +} + +func (h *RuntimeAgentHandler) publish(ctx context.Context, eventType string, payload any) { + if h.events == nil { + return + } + _ = h.events.Publish(ctx, eventType, payload) +} + +func runtimePodCapacityFromReport(reported, configured int) int { + if configured > 0 { + return configured + } + capacity := reported + if capacity <= 0 { + capacity = services.RuntimePodCapacity + } + return capacity +} + +func trimStringPtr(value *string) *string { + if value == nil { + return nil + } + trimmed := strings.TrimSpace(*value) + if trimmed == "" { + return nil + } + return &trimmed +} diff --git a/backend/internal/handlers/runtime_agent_handler_test.go b/backend/internal/handlers/runtime_agent_handler_test.go new file mode 100644 index 0000000..ee825f9 --- /dev/null +++ b/backend/internal/handlers/runtime_agent_handler_test.go @@ -0,0 +1,342 @@ +package handlers + +import ( + "bytes" + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "clawreef/internal/config" + "clawreef/internal/models" + "clawreef/internal/repository" + + "github.com/gin-gonic/gin" +) + +func TestRuntimeAgentHandlerRejectsInvalidToken(t *testing.T) { + gin.SetMode(gin.TestMode) + podRepo := &runtimeAgentHandlerPodRepo{} + handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, podRepo, &runtimeAgentHandlerBindingRepo{}, &runtimeAgentHandlerEvents{}) + + router := gin.New() + router.POST("/api/v1/runtime-agent/metrics/report", handler.ReportMetrics) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/v1/runtime-agent/metrics/report", bytes.NewBufferString(`{"pod_id":9}`)) + req.Header.Set("Content-Type", "application/json") + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, body = %s, want 401", rec.Code, rec.Body.String()) + } + if podRepo.updateMetricsCalls != 0 { + t.Fatalf("UpdateMetrics called %d times, want 0", podRepo.updateMetricsCalls) + } +} + +func TestRuntimeAgentHandlerRegisterUsesConfiguredCapacity(t *testing.T) { + gin.SetMode(gin.TestMode) + podRepo := &runtimeAgentHandlerPodRepo{} + events := &runtimeAgentHandlerEvents{} + handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{ + AgentReportToken: "secret", + MaxGatewaysPerPod: 33, + }, podRepo, &runtimeAgentHandlerBindingRepo{}, events) + + router := gin.New() + router.POST("/api/v1/runtime-agent/register", handler.Register) + + body := `{ + "runtime_type": "openclaw", + "namespace": "clawmanager-system", + "pod_name": "openclaw-runtime-test", + "deployment_name": "openclaw-runtime", + "image_ref": "registry/openclaw:v1", + "agent_endpoint": "http://10.0.0.1:19090", + "state": "ready", + "capacity": 10, + "used_slots": 7 + }` + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/v1/runtime-agent/register", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-ClawManager-Agent-Token", "secret") + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String()) + } + pod := podRepo.podsByID[1] + if pod == nil { + t.Fatal("runtime pod was not persisted") + } + if pod.Capacity != 33 { + t.Fatalf("stored capacity = %d, want configured capacity 33", pod.Capacity) + } + if events.lastType != "runtime_pod_state" { + t.Fatalf("event type = %q, want runtime_pod_state", events.lastType) + } + payload, ok := events.lastPayload.(map[string]any) + if !ok { + t.Fatalf("event payload = %#v, want map", events.lastPayload) + } + if payload["capacity"] != 33 { + t.Fatalf("event capacity = %#v, want configured capacity 33", payload["capacity"]) + } +} + +func TestRuntimeAgentHandlerHeartbeatUsesConfiguredCapacity(t *testing.T) { + gin.SetMode(gin.TestMode) + podRepo := &runtimeAgentHandlerPodRepo{} + events := &runtimeAgentHandlerEvents{} + handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{ + AgentReportToken: "secret", + MaxGatewaysPerPod: 44, + }, podRepo, &runtimeAgentHandlerBindingRepo{}, events) + + router := gin.New() + router.POST("/api/v1/runtime-agent/heartbeat", handler.Heartbeat) + + body := `{ + "pod_id": 9, + "state": "ready", + "used_slots": 8, + "draining": false + }` + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/v1/runtime-agent/heartbeat", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-ClawManager-Agent-Token", "secret") + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String()) + } + if podRepo.updatedPodID != 9 { + t.Fatalf("updated pod id = %d, want 9", podRepo.updatedPodID) + } + if podRepo.lastHeartbeatCapacity != 44 { + t.Fatalf("heartbeat capacity = %d, want configured capacity 44", podRepo.lastHeartbeatCapacity) + } + payload, ok := events.lastPayload.(map[string]any) + if !ok { + t.Fatalf("event payload = %#v, want map", events.lastPayload) + } + if payload["capacity"] != 44 { + t.Fatalf("event capacity = %#v, want configured capacity 44", payload["capacity"]) + } +} + +func TestRuntimeAgentHandlerMetricsReportUpdatesPodAndPublishesEvent(t *testing.T) { + gin.SetMode(gin.TestMode) + podRepo := &runtimeAgentHandlerPodRepo{} + events := &runtimeAgentHandlerEvents{} + handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, podRepo, &runtimeAgentHandlerBindingRepo{}, events) + + router := gin.New() + router.POST("/api/v1/runtime-agent/metrics/report", handler.ReportMetrics) + + body := `{ + "pod_id": 9, + "cpu_millis_used": 2400, + "memory_bytes_used": 4096, + "disk_bytes_used": 8192, + "network_rx_bytes": 12, + "network_tx_bytes": 34, + "metrics": {"load": 0.42} + }` + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/v1/runtime-agent/metrics/report", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-ClawManager-Agent-Token", "secret") + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String()) + } + if podRepo.updatedPodID != 9 { + t.Fatalf("updated pod id = %d, want 9", podRepo.updatedPodID) + } + if podRepo.lastMetrics.CPUMillisUsed != 2400 || podRepo.lastMetrics.MemoryBytesUsed != 4096 || podRepo.lastMetrics.DiskBytesUsed != 8192 { + t.Fatalf("metrics = %#v, want cpu/memory/disk values from request", podRepo.lastMetrics) + } + if podRepo.lastMetrics.MetricsJSON == nil || *podRepo.lastMetrics.MetricsJSON == "" { + t.Fatalf("metrics json was not persisted") + } + if events.lastType != "runtime_pod_metrics" { + t.Fatalf("event type = %q, want runtime_pod_metrics", events.lastType) + } + payload, ok := events.lastPayload.(map[string]any) + if !ok { + t.Fatalf("event payload = %#v, want map", events.lastPayload) + } + if payload["pod_id"] != int64(9) { + t.Fatalf("event pod_id = %#v, want 9", payload["pod_id"]) + } +} + +func TestRuntimeAgentHandlerGatewayReportOnlyUpdatesCurrentPodBinding(t *testing.T) { + gin.SetMode(gin.TestMode) + bindingRepo := &runtimeAgentHandlerBindingRepo{ + bindings: map[int]*models.InstanceRuntimeBinding{ + 10: {InstanceID: 10, RuntimePodID: 9, Generation: 2}, + 11: {InstanceID: 11, RuntimePodID: 99, Generation: 2}, + 12: {InstanceID: 12, RuntimePodID: 9, Generation: 3}, + }, + } + handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, &runtimeAgentHandlerPodRepo{}, bindingRepo, &runtimeAgentHandlerEvents{}) + + router := gin.New() + router.POST("/api/v1/runtime-agent/gateways/report", handler.ReportGateways) + + body := `{ + "pod_id": 9, + "gateways": [ + {"instance_id":10,"gateway_id":"gw-10","gateway_port":20010,"state":"running","generation":2}, + {"instance_id":11,"gateway_id":"gw-11","gateway_port":20011,"state":"running","generation":2}, + {"instance_id":12,"gateway_id":"gw-12","gateway_port":20012,"state":"running","generation":2} + ] + }` + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/v1/runtime-agent/gateways/report", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-ClawManager-Agent-Token", "secret") + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String()) + } + if bindingRepo.updateRunningCalls != 1 { + t.Fatalf("UpdateRunning calls = %d, want 1", bindingRepo.updateRunningCalls) + } +} + +type runtimeAgentHandlerPodRepo struct { + updatedPodID int64 + lastMetrics repository.RuntimePodMetricsUpdate + lastHeartbeatCapacity int + updateMetricsCalls int + podsByID map[int64]*models.RuntimePod +} + +func (r *runtimeAgentHandlerPodRepo) UpsertFromAgent(ctx context.Context, pod *models.RuntimePod) error { + if r.podsByID == nil { + r.podsByID = map[int64]*models.RuntimePod{} + } + if pod.ID == 0 { + pod.ID = int64(len(r.podsByID) + 1) + } + cp := *pod + r.podsByID[pod.ID] = &cp + return nil +} + +func (r *runtimeAgentHandlerPodRepo) GetByID(ctx context.Context, id int64) (*models.RuntimePod, error) { + if r.podsByID == nil { + return nil, nil + } + return r.podsByID[id], nil +} + +func (r *runtimeAgentHandlerPodRepo) GetByNamespaceName(ctx context.Context, namespace, podName string) (*models.RuntimePod, error) { + for _, pod := range r.podsByID { + if pod.Namespace == namespace && pod.PodName == podName { + return pod, nil + } + } + return nil, nil +} + +func (r *runtimeAgentHandlerPodRepo) List(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) { + return nil, nil +} + +func (r *runtimeAgentHandlerPodRepo) ListSchedulable(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) { + return nil, nil +} + +func (r *runtimeAgentHandlerPodRepo) TryClaimSlot(ctx context.Context, podID int64) (bool, error) { + return false, nil +} + +func (r *runtimeAgentHandlerPodRepo) ReleaseSlot(ctx context.Context, podID int64) error { + return nil +} + +func (r *runtimeAgentHandlerPodRepo) MarkState(ctx context.Context, podID int64, state string, draining bool) error { + return nil +} + +func (r *runtimeAgentHandlerPodRepo) UpdateHeartbeat(ctx context.Context, podID int64, state string, usedSlots int, capacity int, draining bool, lastSeenAt time.Time) error { + r.updatedPodID = podID + r.lastHeartbeatCapacity = capacity + return nil +} + +func (r *runtimeAgentHandlerPodRepo) MarkUnseenUnhealthy(ctx context.Context, cutoff time.Time) error { + return nil +} + +func (r *runtimeAgentHandlerPodRepo) UpdateMetrics(ctx context.Context, podID int64, metrics repository.RuntimePodMetricsUpdate) error { + r.updatedPodID = podID + r.lastMetrics = metrics + r.updateMetricsCalls++ + return nil +} + +type runtimeAgentHandlerBindingRepo struct { + updateRunningCalls int + updateStateCalls int + bindings map[int]*models.InstanceRuntimeBinding +} + +func (r *runtimeAgentHandlerBindingRepo) Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error { + return nil +} + +func (r *runtimeAgentHandlerBindingRepo) GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) { + return r.bindings[instanceID], nil +} + +func (r *runtimeAgentHandlerBindingRepo) GetRunningByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) { + return nil, nil +} + +func (r *runtimeAgentHandlerBindingRepo) ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error) { + return nil, nil +} + +func (r *runtimeAgentHandlerBindingRepo) ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error) { + return nil, nil +} + +func (r *runtimeAgentHandlerBindingRepo) UpdateRunning(ctx context.Context, instanceID int, generation int, gatewayID string, port int, pid *int) error { + r.updateRunningCalls++ + return nil +} + +func (r *runtimeAgentHandlerBindingRepo) UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error { + r.updateStateCalls++ + return nil +} + +func (r *runtimeAgentHandlerBindingRepo) DeleteByInstanceID(ctx context.Context, instanceID int) error { + return nil +} + +func (r *runtimeAgentHandlerBindingRepo) DeleteByInstanceIDAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64) error { + return nil +} + +type runtimeAgentHandlerEvents struct { + lastType string + lastPayload any +} + +func (e *runtimeAgentHandlerEvents) Publish(ctx context.Context, eventType string, payload any) error { + e.lastType = eventType + e.lastPayload = payload + return nil +} diff --git a/backend/internal/handlers/runtime_pool_handler.go b/backend/internal/handlers/runtime_pool_handler.go new file mode 100644 index 0000000..af703ad --- /dev/null +++ b/backend/internal/handlers/runtime_pool_handler.go @@ -0,0 +1,270 @@ +package handlers + +import ( + "context" + "log" + "net/http" + "strconv" + "strings" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" + "clawreef/internal/services" + "clawreef/internal/utils" + + "github.com/gin-gonic/gin" +) + +type RuntimePoolHandler struct { + podRepo repository.RuntimePodRepository + bindingRepo repository.InstanceRuntimeBindingRepository + rolloutRepo repository.RuntimeRolloutRepository + scheduler *services.RuntimeScheduler + events runtimeEventPublisher +} + +const ( + runtimePodListFallbackHeartbeatTimeout = 10 * time.Second + runtimePodListStaleWindowMultiplier = 3 +) + +type startRuntimeRolloutRequest struct { + RuntimeType string `json:"runtime_type" binding:"required"` + TargetImageRef string `json:"target_image_ref" binding:"required"` + BatchSize int `json:"batch_size"` + MaxUnavailable int `json:"max_unavailable"` +} + +type runtimePoolPodListItem struct { + models.RuntimePod + AgentReported bool `json:"agent_reported"` +} + +func NewRuntimePoolHandler( + podRepo repository.RuntimePodRepository, + bindingRepo repository.InstanceRuntimeBindingRepository, + rolloutRepo repository.RuntimeRolloutRepository, + scheduler *services.RuntimeScheduler, + events runtimeEventPublisher, +) *RuntimePoolHandler { + return &RuntimePoolHandler{ + podRepo: podRepo, + bindingRepo: bindingRepo, + rolloutRepo: rolloutRepo, + scheduler: scheduler, + events: events, + } +} + +func (h *RuntimePoolHandler) ListPods(c *gin.Context) { + runtimeType := strings.TrimSpace(c.Query("runtime_type")) + if runtimeType != "" { + normalized, ok := services.NormalizeV2RuntimeType(runtimeType) + if !ok { + utils.Error(c, http.StatusBadRequest, "unsupported runtime type") + return + } + runtimeType = normalized + } + pods, err := h.podRepo.List(c.Request.Context(), runtimeType) + if err != nil { + utils.HandleError(c, err) + return + } + currentPods := filterCurrentRuntimePods(pods, time.Now().UTC(), h.runtimePodListHeartbeatTimeout()) + items := runtimePoolPodListItems(currentPods, true) + if h.scheduler != nil { + deploymentPods, err := h.scheduler.RuntimeDeploymentPods(c.Request.Context(), runtimeType) + if err != nil { + log.Printf("runtime pool list deployment pods failed: %v", err) + } else { + items = mergeRuntimePoolDeploymentPods(items, deploymentPods) + } + } + utils.Success(c, http.StatusOK, "Runtime pods retrieved successfully", gin.H{"pods": items}) +} + +func (h *RuntimePoolHandler) GetPodGateways(c *gin.Context) { + podID, ok := parseRuntimePodID(c) + if !ok { + return + } + bindings, err := h.bindingRepo.ListByRuntimePodID(c.Request.Context(), podID) + if err != nil { + utils.HandleError(c, err) + return + } + utils.Success(c, http.StatusOK, "Runtime pod gateways retrieved successfully", gin.H{"gateways": bindings}) +} + +func (h *RuntimePoolHandler) DrainPod(c *gin.Context) { + podID, ok := parseRuntimePodID(c) + if !ok { + return + } + if h.scheduler != nil { + if err := h.scheduler.DrainPod(c.Request.Context(), podID); err != nil { + utils.HandleError(c, err) + return + } + } else if err := h.podRepo.MarkState(c.Request.Context(), podID, "draining", true); err != nil { + utils.HandleError(c, err) + return + } + h.publish(c.Request.Context(), "runtime_pod_state", map[string]any{ + "pod_id": podID, + "state": "draining", + "draining": true, + }) + utils.Success(c, http.StatusOK, "Runtime pod drain started", nil) +} + +func (h *RuntimePoolHandler) StartRollout(c *gin.Context) { + var req startRuntimeRolloutRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + runtimeType, ok := services.NormalizeV2RuntimeType(req.RuntimeType) + if !ok { + utils.Error(c, http.StatusBadRequest, "unsupported runtime type") + return + } + targetImage := strings.TrimSpace(req.TargetImageRef) + if targetImage == "" { + utils.Error(c, http.StatusBadRequest, "target image ref is required") + return + } + batchSize := req.BatchSize + if batchSize <= 0 { + batchSize = 1 + } + maxUnavailable := req.MaxUnavailable + if maxUnavailable <= 0 { + maxUnavailable = 1 + } + startedBy := currentUserIDPtr(c) + rollout := &models.RuntimeRollout{ + RuntimeType: runtimeType, + TargetImageRef: targetImage, + Status: "pending", + BatchSize: batchSize, + MaxUnavailable: maxUnavailable, + StartedBy: startedBy, + } + if err := h.rolloutRepo.Create(c.Request.Context(), rollout); err != nil { + utils.HandleError(c, err) + return + } + if h.scheduler != nil { + if err := h.scheduler.StartRollout(c.Request.Context(), rollout.ID); err != nil { + utils.HandleError(c, err) + return + } + } + h.publish(c.Request.Context(), "runtime_rollout", map[string]any{ + "rollout_id": rollout.ID, + "runtime_type": rollout.RuntimeType, + "target_image_ref": rollout.TargetImageRef, + "status": rollout.Status, + "batch_size": rollout.BatchSize, + "max_unavailable": rollout.MaxUnavailable, + "started_by": rollout.StartedBy, + }) + utils.Success(c, http.StatusCreated, "Runtime rollout created successfully", gin.H{"rollout": rollout}) +} + +func (h *RuntimePoolHandler) publish(ctx context.Context, eventType string, payload any) { + if h.events == nil { + return + } + _ = h.events.Publish(ctx, eventType, payload) +} + +func (h *RuntimePoolHandler) runtimePodListHeartbeatTimeout() time.Duration { + if h != nil && h.scheduler != nil { + if timeout := h.scheduler.HeartbeatTimeout(); timeout > 0 { + return timeout + } + } + return runtimePodListFallbackHeartbeatTimeout +} + +func filterCurrentRuntimePods(pods []models.RuntimePod, now time.Time, heartbeatTimeout time.Duration) []models.RuntimePod { + if heartbeatTimeout <= 0 { + return pods + } + cutoff := now.UTC().Add(-runtimePodListStaleWindowMultiplier * heartbeatTimeout) + current := pods[:0] + for _, pod := range pods { + if pod.LastSeenAt != nil && pod.LastSeenAt.UTC().Before(cutoff) { + continue + } + current = append(current, pod) + } + return current +} + +func runtimePoolPodListItems(pods []models.RuntimePod, agentReported bool) []runtimePoolPodListItem { + items := make([]runtimePoolPodListItem, 0, len(pods)) + for _, pod := range pods { + items = append(items, runtimePoolPodListItem{ + RuntimePod: pod, + AgentReported: agentReported, + }) + } + return items +} + +func mergeRuntimePoolDeploymentPods(items []runtimePoolPodListItem, deploymentPods []models.RuntimePod) []runtimePoolPodListItem { + seen := map[string]struct{}{} + for _, item := range items { + seen[runtimePoolPodKey(item.Namespace, item.PodName)] = struct{}{} + } + for _, pod := range deploymentPods { + key := runtimePoolPodKey(pod.Namespace, pod.PodName) + if key == "" { + continue + } + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + items = append(items, runtimePoolPodListItem{ + RuntimePod: pod, + AgentReported: false, + }) + } + return items +} + +func runtimePoolPodKey(namespace, podName string) string { + namespace = strings.TrimSpace(namespace) + podName = strings.TrimSpace(podName) + if namespace == "" || podName == "" { + return "" + } + return namespace + "/" + podName +} + +func parseRuntimePodID(c *gin.Context) (int64, bool) { + podID, err := strconv.ParseInt(c.Param("id"), 10, 64) + if err != nil || podID <= 0 { + utils.Error(c, http.StatusBadRequest, "invalid runtime pod ID") + return 0, false + } + return podID, true +} + +func currentUserIDPtr(c *gin.Context) *int { + raw, ok := c.Get("userID") + if !ok { + return nil + } + userID, ok := raw.(int) + if !ok { + return nil + } + return &userID +} diff --git a/backend/internal/handlers/runtime_pool_handler_test.go b/backend/internal/handlers/runtime_pool_handler_test.go new file mode 100644 index 0000000..51405c9 --- /dev/null +++ b/backend/internal/handlers/runtime_pool_handler_test.go @@ -0,0 +1,532 @@ +package handlers + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "clawreef/internal/middleware" + "clawreef/internal/models" + "clawreef/internal/repository" + "clawreef/internal/services" + "clawreef/internal/services/k8s" + + "github.com/gin-gonic/gin" +) + +func TestRuntimePoolHandlerRejectsNonAdmin(t *testing.T) { + gin.SetMode(gin.TestMode) + handler := NewRuntimePoolHandler(&runtimePoolHandlerPodRepo{}, &runtimePoolHandlerBindingRepo{}, &runtimePoolHandlerRolloutRepo{}, nil, &runtimePoolHandlerEvents{}) + + router := runtimePoolHandlerRouter(20, "user", handler) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/runtime-pods", nil) + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusForbidden { + t.Fatalf("status = %d, body = %s, want 403", rec.Code, rec.Body.String()) + } +} + +func TestRuntimePoolHandlerListPodsReturnsMetricsForAdmin(t *testing.T) { + gin.SetMode(gin.TestMode) + now := time.Now().UTC() + podIP := "10.42.0.31" + nodeName := "node-a" + handler := NewRuntimePoolHandler(&runtimePoolHandlerPodRepo{pods: []models.RuntimePod{{ + ID: 9, + RuntimeType: "openclaw", + PodName: "openclaw-runtime-abcde", + PodIP: &podIP, + NodeName: &nodeName, + State: "ready", + UsedSlots: 37, + Capacity: 100, + Draining: false, + CPUMillisUsed: 13600, + MemoryBytesUsed: 42949672960, + DiskBytesUsed: 214748364800, + NetworkRXBytes: 9223372, + NetworkTXBytes: 19223372, + LastSeenAt: &now, + }}}, &runtimePoolHandlerBindingRepo{}, &runtimePoolHandlerRolloutRepo{}, nil, &runtimePoolHandlerEvents{}) + + router := runtimePoolHandlerRouter(1, "admin", handler) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/runtime-pods", nil) + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String()) + } + + var resp struct { + Data struct { + Pods []struct { + ID int64 `json:"id"` + RuntimeType string `json:"runtime_type"` + PodName string `json:"pod_name"` + CPUMillisUsed int64 `json:"cpu_millis_used"` + MemoryBytesUsed int64 `json:"memory_bytes_used"` + DiskBytesUsed int64 `json:"disk_bytes_used"` + NetworkRXBytes int64 `json:"network_rx_bytes"` + NetworkTXBytes int64 `json:"network_tx_bytes"` + LastSeenAt string `json:"last_seen_at"` + } `json:"pods"` + } `json:"data"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(resp.Data.Pods) != 1 { + t.Fatalf("pods length = %d, want 1", len(resp.Data.Pods)) + } + pod := resp.Data.Pods[0] + if pod.ID != 9 || pod.RuntimeType != "openclaw" || pod.PodName != "openclaw-runtime-abcde" { + t.Fatalf("pod identity = %#v, want runtime pod data", pod) + } + if pod.CPUMillisUsed != 13600 || pod.MemoryBytesUsed != 42949672960 || pod.DiskBytesUsed != 214748364800 || pod.NetworkRXBytes != 9223372 || pod.NetworkTXBytes != 19223372 { + t.Fatalf("pod metrics = %#v, want persisted metrics", pod) + } +} + +func TestRuntimePoolHandlerListPodsOmitsStaleRuntimePods(t *testing.T) { + gin.SetMode(gin.TestMode) + recentSeen := time.Now().UTC() + staleSeen := recentSeen.Add(-5 * time.Minute) + handler := NewRuntimePoolHandler(&runtimePoolHandlerPodRepo{pods: []models.RuntimePod{ + {ID: 1, RuntimeType: "openclaw", PodName: "openclaw-runtime-live", State: "ready", LastSeenAt: &recentSeen}, + {ID: 2, RuntimeType: "openclaw", PodName: "openclaw-runtime-deleted", State: "unhealthy", LastSeenAt: &staleSeen}, + {ID: 3, RuntimeType: "hermes", PodName: "hermes-runtime-recent-unhealthy", State: "unhealthy", LastSeenAt: &recentSeen}, + }}, &runtimePoolHandlerBindingRepo{}, &runtimePoolHandlerRolloutRepo{}, nil, &runtimePoolHandlerEvents{}) + + router := runtimePoolHandlerRouter(1, "admin", handler) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/runtime-pods", nil) + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String()) + } + + var resp struct { + Data struct { + Pods []struct { + ID int64 `json:"id"` + PodName string `json:"pod_name"` + } `json:"pods"` + } `json:"data"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(resp.Data.Pods) != 2 { + t.Fatalf("pods length = %d, want 2: %#v", len(resp.Data.Pods), resp.Data.Pods) + } + for _, pod := range resp.Data.Pods { + if pod.ID == 2 || pod.PodName == "openclaw-runtime-deleted" { + t.Fatalf("stale deleted pod was returned: %#v", pod) + } + } +} + +func TestRuntimePoolHandlerListPodsIncludesUnreportedDeploymentPods(t *testing.T) { + gin.SetMode(gin.TestMode) + deployments := &runtimePoolHandlerDeploymentService{ + pods: []k8s.RuntimeDeploymentPod{{ + RuntimeType: "openclaw", + Namespace: "clawmanager-system", + DeploymentName: "openclaw-runtime", + PodName: "openclaw-runtime-current", + ImageRef: "registry/openclaw-lite:final2", + State: "ready", + }}, + } + scheduler := services.NewRuntimeScheduler( + nil, + &runtimePoolHandlerPodRepo{}, + nil, + &runtimePoolHandlerRolloutRepo{}, + &runtimePoolHandlerAgentClient{}, + services.NewRuntimeEventService(nil), + nil, + deployments, + time.Second, + services.WithRuntimeSchedulerNamespace("clawmanager-system"), + services.WithRuntimeSchedulerMaxGatewaysPerPod(33), + ) + handler := NewRuntimePoolHandler(&runtimePoolHandlerPodRepo{}, &runtimePoolHandlerBindingRepo{}, &runtimePoolHandlerRolloutRepo{}, scheduler, &runtimePoolHandlerEvents{}) + + router := runtimePoolHandlerRouter(1, "admin", handler) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/runtime-pods?runtime_type=openclaw", nil) + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String()) + } + + var resp struct { + Data struct { + Pods []struct { + ID int64 `json:"id"` + RuntimeType string `json:"runtime_type"` + PodName string `json:"pod_name"` + ImageRef string `json:"image_ref"` + State string `json:"state"` + Capacity int `json:"capacity"` + AgentReported bool `json:"agent_reported"` + DeploymentName string `json:"deployment_name"` + } `json:"pods"` + } `json:"data"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(resp.Data.Pods) != 1 { + t.Fatalf("pods length = %d, want 1: %#v", len(resp.Data.Pods), resp.Data.Pods) + } + pod := resp.Data.Pods[0] + if pod.ID >= 0 || pod.AgentReported { + t.Fatalf("fallback pod identity = id:%d agent_reported:%t, want synthetic unreported pod", pod.ID, pod.AgentReported) + } + if pod.RuntimeType != "openclaw" || pod.PodName != "openclaw-runtime-current" || pod.DeploymentName != "openclaw-runtime" { + t.Fatalf("fallback pod metadata = %#v, want OpenClaw deployment pod", pod) + } + if pod.ImageRef != "registry/openclaw-lite:final2" { + t.Fatalf("fallback pod image = %q, want registry/openclaw-lite:final2", pod.ImageRef) + } + if pod.State != "unhealthy" { + t.Fatalf("fallback pod state = %q, want unhealthy while agent is not reporting", pod.State) + } + if pod.Capacity != 33 { + t.Fatalf("fallback pod capacity = %d, want configured capacity 33", pod.Capacity) + } +} + +func TestRuntimePoolHandlerStartRolloutStoresRequesterAndPublishesEvent(t *testing.T) { + gin.SetMode(gin.TestMode) + rolloutRepo := &runtimePoolHandlerRolloutRepo{} + events := &runtimePoolHandlerEvents{} + handler := NewRuntimePoolHandler(&runtimePoolHandlerPodRepo{}, &runtimePoolHandlerBindingRepo{}, rolloutRepo, nil, events) + + router := runtimePoolHandlerRouter(7, "admin", handler) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/v1/admin/runtime-rollouts", bytes.NewBufferString(`{ + "runtime_type": "hermes", + "target_image_ref": "ghcr.io/example/hermes:v2", + "batch_size": 2, + "max_unavailable": 1 + }`)) + req.Header.Set("Content-Type", "application/json") + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusCreated { + t.Fatalf("status = %d, body = %s, want 201", rec.Code, rec.Body.String()) + } + if rolloutRepo.created == nil { + t.Fatalf("rollout was not created") + } + if rolloutRepo.created.StartedBy == nil || *rolloutRepo.created.StartedBy != 7 { + t.Fatalf("started_by = %#v, want 7", rolloutRepo.created.StartedBy) + } + if events.lastType != "runtime_rollout" { + t.Fatalf("event type = %q, want runtime_rollout", events.lastType) + } +} + +func TestRuntimePoolHandlerStartRolloutRunsSchedulerImmediately(t *testing.T) { + gin.SetMode(gin.TestMode) + rolloutRepo := &runtimePoolHandlerRolloutRepo{} + podRepo := &runtimePoolHandlerPodRepo{pods: []models.RuntimePod{{ + ID: 21, + RuntimeType: "hermes", + Namespace: "clawmanager-system", + PodName: "hermes-runtime-old", + DeploymentName: "hermes-runtime", + ImageRef: "registry/hermes:v1", + State: "ready", + Capacity: 100, + UsedSlots: 7, + }}} + deployments := &runtimePoolHandlerDeploymentService{} + scheduler := services.NewRuntimeScheduler( + nil, + podRepo, + nil, + rolloutRepo, + &runtimePoolHandlerAgentClient{}, + services.NewRuntimeEventService(nil), + nil, + deployments, + time.Second, + ) + handler := NewRuntimePoolHandler(podRepo, &runtimePoolHandlerBindingRepo{}, rolloutRepo, scheduler, &runtimePoolHandlerEvents{}) + + router := runtimePoolHandlerRouter(7, "admin", handler) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/api/v1/admin/runtime-rollouts", bytes.NewBufferString(`{ + "runtime_type": "hermes", + "target_image_ref": "registry/hermes:v2", + "batch_size": 1, + "max_unavailable": 1 + }`)) + req.Header.Set("Content-Type", "application/json") + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusCreated { + t.Fatalf("status = %d, body = %s, want 201", rec.Code, rec.Body.String()) + } + if got := len(deployments.rollouts); got != 1 { + t.Fatalf("deployment rollouts = %d, want 1", got) + } + rollout := deployments.rollouts[0] + if rollout.namespace != "clawmanager-system" || rollout.name != "hermes-runtime" || rollout.image != "registry/hermes:v2" { + t.Fatalf("deployment rollout = %+v, want hermes-runtime registry/hermes:v2", rollout) + } + if podRepo.markedPodID != 21 || podRepo.markedState != "draining" || !podRepo.markedDraining { + t.Fatalf("pod mark = id:%d state:%q draining:%t, want drain pod 21", podRepo.markedPodID, podRepo.markedState, podRepo.markedDraining) + } +} + +func runtimePoolHandlerRouter(userID int, role string, handler *RuntimePoolHandler) *gin.Engine { + userRepo := &runtimePoolHandlerUserRepo{users: map[int]*models.User{ + userID: {ID: userID, Username: "tester", Role: role}, + }} + router := gin.New() + admin := router.Group("/api/v1/admin") + admin.Use(func(c *gin.Context) { + c.Set("userID", userID) + c.Next() + }) + admin.Use(middleware.SetUserInfo(userRepo)) + admin.Use(middleware.NewAdminAuth(userRepo)) + { + admin.GET("/runtime-pods", handler.ListPods) + admin.POST("/runtime-rollouts", handler.StartRollout) + } + return router +} + +type runtimePoolHandlerUserRepo struct { + users map[int]*models.User +} + +func (r *runtimePoolHandlerUserRepo) Create(user *models.User) error { return nil } +func (r *runtimePoolHandlerUserRepo) GetByID(id int) (*models.User, error) { + return r.users[id], nil +} +func (r *runtimePoolHandlerUserRepo) GetByUsername(username string) (*models.User, error) { + return nil, nil +} +func (r *runtimePoolHandlerUserRepo) GetByEmail(email string) (*models.User, error) { + return nil, nil +} +func (r *runtimePoolHandlerUserRepo) Update(user *models.User) error { return nil } +func (r *runtimePoolHandlerUserRepo) Delete(id int) error { return nil } +func (r *runtimePoolHandlerUserRepo) List(offset, limit int) ([]models.User, error) { + return nil, nil +} +func (r *runtimePoolHandlerUserRepo) Count() (int, error) { return 0, nil } + +type runtimePoolHandlerPodRepo struct { + pods []models.RuntimePod + markedPodID int64 + markedState string + markedDraining bool +} + +func (r *runtimePoolHandlerPodRepo) UpsertFromAgent(ctx context.Context, pod *models.RuntimePod) error { + return nil +} +func (r *runtimePoolHandlerPodRepo) GetByID(ctx context.Context, id int64) (*models.RuntimePod, error) { + for _, pod := range r.pods { + if pod.ID == id { + cp := pod + return &cp, nil + } + } + return nil, nil +} +func (r *runtimePoolHandlerPodRepo) GetByNamespaceName(ctx context.Context, namespace, podName string) (*models.RuntimePod, error) { + return nil, nil +} +func (r *runtimePoolHandlerPodRepo) List(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) { + if runtimeType == "" { + return append([]models.RuntimePod(nil), r.pods...), nil + } + var pods []models.RuntimePod + for _, pod := range r.pods { + if pod.RuntimeType == runtimeType { + pods = append(pods, pod) + } + } + return pods, nil +} +func (r *runtimePoolHandlerPodRepo) ListSchedulable(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) { + return nil, nil +} +func (r *runtimePoolHandlerPodRepo) TryClaimSlot(ctx context.Context, podID int64) (bool, error) { + return false, nil +} +func (r *runtimePoolHandlerPodRepo) ReleaseSlot(ctx context.Context, podID int64) error { + return nil +} +func (r *runtimePoolHandlerPodRepo) MarkState(ctx context.Context, podID int64, state string, draining bool) error { + r.markedPodID = podID + r.markedState = state + r.markedDraining = draining + return nil +} +func (r *runtimePoolHandlerPodRepo) UpdateHeartbeat(ctx context.Context, podID int64, state string, usedSlots int, capacity int, draining bool, lastSeenAt time.Time) error { + return nil +} +func (r *runtimePoolHandlerPodRepo) MarkUnseenUnhealthy(ctx context.Context, cutoff time.Time) error { + return nil +} +func (r *runtimePoolHandlerPodRepo) UpdateMetrics(ctx context.Context, podID int64, metrics repository.RuntimePodMetricsUpdate) error { + return nil +} + +type runtimePoolHandlerBindingRepo struct { + bindings []models.InstanceRuntimeBinding +} + +func (r *runtimePoolHandlerBindingRepo) Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error { + return nil +} +func (r *runtimePoolHandlerBindingRepo) GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) { + return nil, nil +} +func (r *runtimePoolHandlerBindingRepo) GetRunningByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) { + return nil, nil +} +func (r *runtimePoolHandlerBindingRepo) ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error) { + var bindings []models.InstanceRuntimeBinding + for _, binding := range r.bindings { + if binding.RuntimePodID == runtimePodID { + bindings = append(bindings, binding) + } + } + return bindings, nil +} +func (r *runtimePoolHandlerBindingRepo) ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error) { + return nil, nil +} +func (r *runtimePoolHandlerBindingRepo) UpdateRunning(ctx context.Context, instanceID int, generation int, gatewayID string, port int, pid *int) error { + return nil +} +func (r *runtimePoolHandlerBindingRepo) UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error { + return nil +} +func (r *runtimePoolHandlerBindingRepo) DeleteByInstanceID(ctx context.Context, instanceID int) error { + return nil +} +func (r *runtimePoolHandlerBindingRepo) DeleteByInstanceIDAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64) error { + return nil +} + +type runtimePoolHandlerRolloutRepo struct { + created *models.RuntimeRollout + byID map[int64]*models.RuntimeRollout +} + +func (r *runtimePoolHandlerRolloutRepo) Create(ctx context.Context, rollout *models.RuntimeRollout) error { + rollout.ID = 12 + cp := *rollout + r.created = &cp + if r.byID == nil { + r.byID = map[int64]*models.RuntimeRollout{} + } + r.byID[rollout.ID] = &cp + return nil +} +func (r *runtimePoolHandlerRolloutRepo) GetByID(ctx context.Context, id int64) (*models.RuntimeRollout, error) { + if r.byID == nil { + return nil, nil + } + return r.byID[id], nil +} +func (r *runtimePoolHandlerRolloutRepo) ListActive(ctx context.Context, runtimeType string) ([]models.RuntimeRollout, error) { + return nil, nil +} +func (r *runtimePoolHandlerRolloutRepo) UpdateStatus(ctx context.Context, id int64, status string, startedAt *time.Time, finishedAt *time.Time, message *string) error { + return nil +} + +type runtimePoolHandlerEvents struct { + lastType string + lastPayload any +} + +func (e *runtimePoolHandlerEvents) Publish(ctx context.Context, eventType string, payload any) error { + e.lastType = eventType + e.lastPayload = payload + return nil +} + +type runtimePoolHandlerDeploymentService struct { + rollouts []runtimePoolHandlerDeploymentRollout + pods []k8s.RuntimeDeploymentPod +} + +type runtimePoolHandlerDeploymentRollout struct { + namespace string + name string + image string + maxUnavailable int + maxSurge int +} + +func (s *runtimePoolHandlerDeploymentService) Ensure(ctx context.Context, spec k8s.RuntimeDeploymentSpec) error { + return nil +} +func (s *runtimePoolHandlerDeploymentService) Scale(ctx context.Context, namespace, name string, replicas int32) error { + return nil +} +func (s *runtimePoolHandlerDeploymentService) RolloutImage(ctx context.Context, namespace, name, image string, maxUnavailable, maxSurge int) error { + s.rollouts = append(s.rollouts, runtimePoolHandlerDeploymentRollout{ + namespace: namespace, + name: name, + image: image, + maxUnavailable: maxUnavailable, + maxSurge: maxSurge, + }) + return nil +} +func (s *runtimePoolHandlerDeploymentService) ListPods(ctx context.Context, namespace, runtimeType string) ([]k8s.RuntimeDeploymentPod, error) { + var pods []k8s.RuntimeDeploymentPod + for _, pod := range s.pods { + if namespace != "" && pod.Namespace != namespace { + continue + } + if runtimeType != "" && pod.RuntimeType != runtimeType { + continue + } + pods = append(pods, pod) + } + return pods, nil +} + +type runtimePoolHandlerAgentClient struct{} + +func (c *runtimePoolHandlerAgentClient) Health(ctx context.Context, endpoint string) error { + return nil +} +func (c *runtimePoolHandlerAgentClient) CreateGateway(ctx context.Context, endpoint string, req services.RuntimeAgentCreateGatewayRequest) (*services.RuntimeAgentCreateGatewayResponse, error) { + return nil, nil +} +func (c *runtimePoolHandlerAgentClient) DeleteGateway(ctx context.Context, endpoint, gatewayID string) error { + return nil +} +func (c *runtimePoolHandlerAgentClient) Drain(ctx context.Context, endpoint string) error { return nil } diff --git a/backend/internal/handlers/system_settings_handler.go b/backend/internal/handlers/system_settings_handler.go index 90b3c35..4008aa6 100644 --- a/backend/internal/handlers/system_settings_handler.go +++ b/backend/internal/handlers/system_settings_handler.go @@ -19,7 +19,7 @@ type SystemSettingsHandler struct { type UpsertSystemImageSettingRequest struct { ID int `json:"id,omitempty"` InstanceType string `json:"instance_type" binding:"required"` - RuntimeType string `json:"runtime_type" binding:"omitempty,oneof=desktop shell"` + RuntimeType string `json:"runtime_type" binding:"omitempty,oneof=desktop gateway shell"` DisplayName string `json:"display_name"` Image string `json:"image" binding:"required"` } diff --git a/backend/internal/handlers/websocket_handler.go b/backend/internal/handlers/websocket_handler.go index 0d1b31b..98c29b4 100644 --- a/backend/internal/handlers/websocket_handler.go +++ b/backend/internal/handlers/websocket_handler.go @@ -27,9 +27,19 @@ func (h *WebSocketHandler) HandleWebSocket(c *gin.Context) { c.JSON(http.StatusUnauthorized, gin.H{"error": "Unauthorized"}) return } + roleRaw, _ := c.Get("userRole") + role, _ := roleRaw.(string) + topic := services.WebSocketTopicUser + if c.Query("topic") == string(services.WebSocketTopicRuntimeAdmin) { + if role != "admin" { + c.JSON(http.StatusForbidden, gin.H{"error": "Admin access required"}) + return + } + topic = services.WebSocketTopicRuntimeAdmin + } // Upgrade HTTP connection to WebSocket - services.ServeWS(h.hub, c.Writer, c.Request, userID.(int)) + services.ServeWSWithOptions(h.hub, c.Writer, c.Request, userID.(int), role, topic) } // GetConnectionCount returns the number of active WebSocket connections diff --git a/backend/internal/handlers/workspace_file_handler.go b/backend/internal/handlers/workspace_file_handler.go new file mode 100644 index 0000000..50ce2d7 --- /dev/null +++ b/backend/internal/handlers/workspace_file_handler.go @@ -0,0 +1,317 @@ +package handlers + +import ( + "errors" + "fmt" + "io" + "mime" + "net/http" + "net/url" + "path/filepath" + "strconv" + "strings" + "unicode" + + "clawreef/internal/models" + "clawreef/internal/services" + "clawreef/internal/utils" + + "github.com/gin-gonic/gin" +) + +type WorkspaceFileHandler struct { + instanceService services.InstanceService + fileService services.WorkspaceFileService + runtimeWorkspaceFileService services.WorkspaceFileService +} + +type createWorkspaceFolderRequest struct { + Path string `json:"path" binding:"required"` +} + +type renameWorkspaceEntryRequest struct { + OldPath string `json:"old_path" binding:"required"` + NewPath string `json:"new_path" binding:"required"` +} + +func NewWorkspaceFileHandler(instanceService services.InstanceService, fileService services.WorkspaceFileService, runtimeWorkspaceFileService ...services.WorkspaceFileService) *WorkspaceFileHandler { + runtimeFileService := fileService + if len(runtimeWorkspaceFileService) > 0 && runtimeWorkspaceFileService[0] != nil { + runtimeFileService = runtimeWorkspaceFileService[0] + } + return &WorkspaceFileHandler{ + instanceService: instanceService, + fileService: fileService, + runtimeWorkspaceFileService: runtimeFileService, + } +} + +func (h *WorkspaceFileHandler) List(c *gin.Context) { + _, service, scope, ok := h.workspaceScope(c) + if !ok { + return + } + entries, err := service.List(c.Request.Context(), scope, c.Query("path")) + if err != nil { + handleWorkspaceFileError(c, err) + return + } + utils.Success(c, http.StatusOK, "Workspace files retrieved successfully", gin.H{"entries": entries}) +} + +func (h *WorkspaceFileHandler) Preview(c *gin.Context) { + _, service, scope, ok := h.workspaceScope(c) + if !ok { + return + } + + if strings.EqualFold(strings.TrimSpace(c.Query("raw")), "1") { + file, contentType, size, err := service.OpenPreview(c.Request.Context(), scope, c.Query("path")) + if err != nil { + handleWorkspaceFileError(c, err) + return + } + defer file.Close() + filename := safeWorkspaceDownloadName(filepath.Base(c.Query("path"))) + streamWorkspaceFile(c, file, filename, contentType, "inline", size) + return + } + + preview, err := service.Preview(c.Request.Context(), scope, c.Query("path")) + if err != nil { + handleWorkspaceFileError(c, err) + return + } + utils.Success(c, http.StatusOK, "Workspace preview retrieved successfully", gin.H{"preview": preview}) +} + +func (h *WorkspaceFileHandler) Download(c *gin.Context) { + _, service, scope, ok := h.workspaceScope(c) + if !ok { + return + } + file, filename, size, err := service.OpenDownload(c.Request.Context(), scope, c.Query("path")) + if err != nil { + handleWorkspaceFileError(c, err) + return + } + defer file.Close() + + contentType := mime.TypeByExtension(strings.ToLower(filepath.Ext(filename))) + if contentType == "" { + contentType = "application/octet-stream" + } + streamWorkspaceFile(c, file, filename, contentType, "attachment", size) +} + +func (h *WorkspaceFileHandler) Upload(c *gin.Context) { + _, service, scope, ok := h.workspaceScope(c) + if !ok { + return + } + + maxBytes := services.WorkspaceUploadMaxBytes() + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxBytes+(1<<20)) + fileHeader, err := c.FormFile("file") + if err != nil { + var maxBytesErr *http.MaxBytesError + if errors.As(err, &maxBytesErr) { + utils.Error(c, http.StatusRequestEntityTooLarge, "workspace upload exceeds maximum size") + return + } + utils.Error(c, http.StatusBadRequest, "file is required") + return + } + if fileHeader.Size > maxBytes { + utils.Error(c, http.StatusRequestEntityTooLarge, "workspace upload exceeds maximum size") + return + } + file, err := fileHeader.Open() + if err != nil { + handleWorkspaceFileError(c, err) + return + } + defer file.Close() + + entry, err := service.Upload(c.Request.Context(), scope, c.Query("path"), fileHeader.Filename, file, fileHeader.Size) + if err != nil { + handleWorkspaceFileError(c, err) + return + } + utils.Success(c, http.StatusCreated, "Workspace file uploaded successfully", entry) +} + +func (h *WorkspaceFileHandler) Mkdir(c *gin.Context) { + _, service, scope, ok := h.workspaceScope(c) + if !ok { + return + } + var req createWorkspaceFolderRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + entry, err := service.Mkdir(c.Request.Context(), scope, req.Path) + if err != nil { + handleWorkspaceFileError(c, err) + return + } + utils.Success(c, http.StatusCreated, "Workspace folder created successfully", entry) +} + +func (h *WorkspaceFileHandler) Rename(c *gin.Context) { + _, service, scope, ok := h.workspaceScope(c) + if !ok { + return + } + var req renameWorkspaceEntryRequest + if err := c.ShouldBindJSON(&req); err != nil { + utils.ValidationError(c, err) + return + } + entry, err := service.Rename(c.Request.Context(), scope, req.OldPath, req.NewPath) + if err != nil { + handleWorkspaceFileError(c, err) + return + } + utils.Success(c, http.StatusOK, "Workspace entry renamed successfully", entry) +} + +func (h *WorkspaceFileHandler) Delete(c *gin.Context) { + _, service, scope, ok := h.workspaceScope(c) + if !ok { + return + } + if err := service.Delete(c.Request.Context(), scope, c.Query("path")); err != nil { + handleWorkspaceFileError(c, err) + return + } + utils.Success(c, http.StatusOK, "Workspace entry deleted successfully", nil) +} + +func (h *WorkspaceFileHandler) workspaceScope(c *gin.Context) (*models.Instance, services.WorkspaceFileService, services.WorkspaceFileScope, bool) { + id, err := strconv.Atoi(c.Param("id")) + if err != nil { + utils.Error(c, http.StatusBadRequest, "Invalid instance ID") + return nil, nil, services.WorkspaceFileScope{}, false + } + + instance, err := h.instanceService.GetByID(id) + if err != nil { + utils.HandleError(c, err) + return nil, nil, services.WorkspaceFileScope{}, false + } + if instance == nil { + utils.Error(c, http.StatusNotFound, "Instance not found") + return nil, nil, services.WorkspaceFileScope{}, false + } + + userIDRaw, ok := c.Get("userID") + if !ok { + utils.Error(c, http.StatusUnauthorized, "Unauthorized") + return nil, nil, services.WorkspaceFileScope{}, false + } + userID, ok := userIDRaw.(int) + if !ok { + utils.Error(c, http.StatusUnauthorized, "Unauthorized") + return nil, nil, services.WorkspaceFileScope{}, false + } + roleRaw, _ := c.Get("userRole") + userRole, _ := roleRaw.(string) + if userRole != "admin" && instance.UserID != userID { + utils.Error(c, http.StatusForbidden, "Access denied") + return nil, nil, services.WorkspaceFileScope{}, false + } + if isDesktopWorkspaceInstance(instance) { + return instance, h.runtimeWorkspaceFileService, services.WorkspaceFileScope{ + InstanceID: instance.ID, + UserID: instance.UserID, + WorkspacePath: "/config", + }, true + } + if instance.WorkspacePath == nil || strings.TrimSpace(*instance.WorkspacePath) == "" { + utils.Error(c, http.StatusNotFound, "Workspace not found") + return nil, nil, services.WorkspaceFileScope{}, false + } + + return instance, h.fileService, services.WorkspaceFileScope{ + InstanceID: instance.ID, + UserID: instance.UserID, + WorkspacePath: *instance.WorkspacePath, + }, true +} + +func isDesktopWorkspaceInstance(instance *models.Instance) bool { + if instance == nil { + return false + } + return strings.EqualFold(strings.TrimSpace(instance.InstanceMode), services.InstanceModePro) || + strings.EqualFold(strings.TrimSpace(instance.RuntimeType), services.RuntimeBackendDesktop) +} + +func streamWorkspaceFile(c *gin.Context, file io.ReadSeeker, filename, contentType, disposition string, size int64) { + safeName := safeWorkspaceDownloadName(filename) + c.Header("Content-Type", contentType) + c.Header("X-Content-Type-Options", "nosniff") + c.Header("Content-Disposition", workspaceContentDisposition(disposition, safeName)) + if size >= 0 { + c.Header("Content-Length", strconv.FormatInt(size, 10)) + } + if _, err := io.Copy(c.Writer, file); err != nil && !c.Writer.Written() { + utils.HandleError(c, err) + } +} + +func workspaceContentDisposition(disposition, filename string) string { + disposition = strings.TrimSpace(strings.ToLower(disposition)) + if disposition != "inline" { + disposition = "attachment" + } + escaped := strings.ReplaceAll(filename, `\`, `\\`) + escaped = strings.ReplaceAll(escaped, `"`, `\"`) + return fmt.Sprintf("%s; filename=\"%s\"; filename*=UTF-8''%s", disposition, escaped, url.PathEscape(filename)) +} + +func safeWorkspaceDownloadName(name string) string { + name = strings.TrimSpace(strings.ReplaceAll(name, "\\", "/")) + name = filepath.Base(name) + if name == "." || name == "/" || name == "" { + name = "download" + } + name = strings.Map(func(r rune) rune { + if unicode.IsControl(r) { + return -1 + } + switch r { + case '/', '\\', ':', '*', '?', '"', '<', '>', '|': + return '-' + default: + return r + } + }, name) + name = strings.TrimSpace(name) + if name == "" || name == "." || name == ".." { + return "download" + } + return name +} + +func handleWorkspaceFileError(c *gin.Context, err error) { + switch { + case errors.Is(err, services.ErrWorkspacePathNotFound): + utils.Error(c, http.StatusNotFound, err.Error()) + case errors.Is(err, services.ErrWorkspacePreviewTooLarge), errors.Is(err, services.ErrWorkspaceUploadTooLarge): + utils.Error(c, http.StatusRequestEntityTooLarge, err.Error()) + case errors.Is(err, services.ErrWorkspacePathEscape): + utils.Error(c, http.StatusForbidden, "Access denied") + case errors.Is(err, services.ErrWorkspacePathInvalid), + errors.Is(err, services.ErrWorkspaceDirectoryExpected), + errors.Is(err, services.ErrWorkspaceFileExpected), + errors.Is(err, services.ErrWorkspaceRootOperation), + errors.Is(err, services.ErrWorkspaceFileNameInvalid), + errors.Is(err, services.ErrWorkspaceEntryExists): + utils.Error(c, http.StatusBadRequest, err.Error()) + default: + utils.HandleError(c, err) + } +} diff --git a/backend/internal/handlers/workspace_file_handler_test.go b/backend/internal/handlers/workspace_file_handler_test.go new file mode 100644 index 0000000..fa95a8d --- /dev/null +++ b/backend/internal/handlers/workspace_file_handler_test.go @@ -0,0 +1,242 @@ +package handlers + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + "clawreef/internal/models" + "clawreef/internal/services" + + "github.com/gin-gonic/gin" +) + +type fakeWorkspaceHandlerInstanceService struct { + instances map[int]*models.Instance +} + +func (s *fakeWorkspaceHandlerInstanceService) Create(userID int, req services.CreateInstanceRequest) (*models.Instance, error) { + return nil, nil +} +func (s *fakeWorkspaceHandlerInstanceService) ValidateCreateRequests(userID int, requests []services.CreateInstanceRequest) error { + return nil +} +func (s *fakeWorkspaceHandlerInstanceService) GetByID(id int) (*models.Instance, error) { + return s.instances[id], nil +} +func (s *fakeWorkspaceHandlerInstanceService) GetByUserID(userID int, offset, limit int) ([]models.Instance, int, error) { + return nil, 0, nil +} +func (s *fakeWorkspaceHandlerInstanceService) GetAllInstances(offset, limit int) ([]models.Instance, int, error) { + return nil, 0, nil +} +func (s *fakeWorkspaceHandlerInstanceService) Start(instanceID int) error { return nil } +func (s *fakeWorkspaceHandlerInstanceService) Stop(instanceID int) error { return nil } +func (s *fakeWorkspaceHandlerInstanceService) Restart(instanceID int) error { return nil } +func (s *fakeWorkspaceHandlerInstanceService) Delete(instanceID int) error { return nil } +func (s *fakeWorkspaceHandlerInstanceService) Update(instanceID int, req services.UpdateInstanceRequest) error { + return nil +} +func (s *fakeWorkspaceHandlerInstanceService) GetInstanceStatus(instanceID int) (*services.InstanceStatus, error) { + return nil, nil +} +func (s *fakeWorkspaceHandlerInstanceService) ForceSyncInstance(instanceID int) error { return nil } + +type fakeWorkspaceFileService struct { + lastScope services.WorkspaceFileScope + listCalls int + file *os.File + filename string + size int64 +} + +func (s *fakeWorkspaceFileService) List(ctx context.Context, scope services.WorkspaceFileScope, relativePath string) ([]services.WorkspaceEntry, error) { + s.lastScope = scope + s.listCalls++ + return []services.WorkspaceEntry{{Name: "readme.md", Path: "readme.md", ModifiedAt: time.Unix(1, 0), Previewable: true, Downloadable: true}}, nil +} +func (s *fakeWorkspaceFileService) Preview(ctx context.Context, scope services.WorkspaceFileScope, relativePath string) (*services.WorkspacePreview, error) { + s.lastScope = scope + return &services.WorkspacePreview{Kind: "text", Text: "ok"}, nil +} +func (s *fakeWorkspaceFileService) OpenPreview(ctx context.Context, scope services.WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) { + s.lastScope = scope + return s.file, "text/plain; charset=utf-8", s.size, nil +} +func (s *fakeWorkspaceFileService) OpenDownload(ctx context.Context, scope services.WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) { + s.lastScope = scope + return s.file, s.filename, s.size, nil +} +func (s *fakeWorkspaceFileService) Upload(ctx context.Context, scope services.WorkspaceFileScope, relativeDir string, filename string, reader io.Reader, size int64) (*services.WorkspaceEntry, error) { + s.lastScope = scope + return &services.WorkspaceEntry{Name: filename, Path: filename, Downloadable: true}, nil +} +func (s *fakeWorkspaceFileService) Mkdir(ctx context.Context, scope services.WorkspaceFileScope, relativePath string) (*services.WorkspaceEntry, error) { + s.lastScope = scope + return &services.WorkspaceEntry{Name: "docs", Path: relativePath, IsDir: true}, nil +} +func (s *fakeWorkspaceFileService) Rename(ctx context.Context, scope services.WorkspaceFileScope, oldPath, newPath string) (*services.WorkspaceEntry, error) { + s.lastScope = scope + return &services.WorkspaceEntry{Name: "renamed", Path: newPath}, nil +} +func (s *fakeWorkspaceFileService) Delete(ctx context.Context, scope services.WorkspaceFileScope, relativePath string) error { + s.lastScope = scope + return nil +} + +func TestWorkspaceFileHandlerRejectsNonOwnerUser(t *testing.T) { + gin.SetMode(gin.TestMode) + workspacePath := t.TempDir() + instanceService := &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{ + 77: {ID: 77, UserID: 20, WorkspacePath: &workspacePath}, + }} + fileService := &fakeWorkspaceFileService{} + handler := NewWorkspaceFileHandler(instanceService, fileService) + + router := workspaceFileTestRouter(10, "user") + router.GET("/api/v1/instances/:id/workspace/files", handler.List) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/77/workspace/files", nil) + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusForbidden { + t.Fatalf("status = %d, body = %s, want 403", rec.Code, rec.Body.String()) + } + if fileService.listCalls != 0 { + t.Fatalf("workspace file service called %d times, want 0", fileService.listCalls) + } +} + +func TestWorkspaceFileHandlerAdminUsesInstanceOwnerScope(t *testing.T) { + gin.SetMode(gin.TestMode) + workspacePath := t.TempDir() + instanceService := &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{ + 77: {ID: 77, UserID: 20, WorkspacePath: &workspacePath}, + }} + fileService := &fakeWorkspaceFileService{} + handler := NewWorkspaceFileHandler(instanceService, fileService) + + router := workspaceFileTestRouter(10, "admin") + router.GET("/api/v1/instances/:id/workspace/files", handler.List) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/77/workspace/files?path=", nil) + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String()) + } + if fileService.lastScope.InstanceID != 77 || fileService.lastScope.UserID != 20 || fileService.lastScope.WorkspacePath != workspacePath { + t.Fatalf("scope = %#v, want instance owner scope", fileService.lastScope) + } +} + +func TestWorkspaceFileHandlerRequiresWorkspacePath(t *testing.T) { + gin.SetMode(gin.TestMode) + instanceService := &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{ + 77: {ID: 77, UserID: 10}, + }} + fileService := &fakeWorkspaceFileService{} + handler := NewWorkspaceFileHandler(instanceService, fileService) + + router := workspaceFileTestRouter(10, "user") + router.GET("/api/v1/instances/:id/workspace/files", handler.List) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/77/workspace/files", nil) + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusNotFound { + t.Fatalf("status = %d, body = %s, want 404", rec.Code, rec.Body.String()) + } + if fileService.listCalls != 0 { + t.Fatalf("workspace file service called %d times, want 0", fileService.listCalls) + } +} + +func TestWorkspaceFileHandlerProDesktopUsesRuntimeConfigWorkspace(t *testing.T) { + gin.SetMode(gin.TestMode) + instanceService := &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{ + 77: { + ID: 77, + UserID: 10, + RuntimeType: services.RuntimeBackendDesktop, + InstanceMode: services.InstanceModePro, + }, + }} + localFileService := &fakeWorkspaceFileService{} + runtimeFileService := &fakeWorkspaceFileService{} + handler := NewWorkspaceFileHandler(instanceService, localFileService, runtimeFileService) + + router := workspaceFileTestRouter(10, "user") + router.GET("/api/v1/instances/:id/workspace/files", handler.List) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/77/workspace/files?path=", nil) + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String()) + } + if localFileService.listCalls != 0 { + t.Fatalf("local workspace service called %d times, want 0", localFileService.listCalls) + } + if runtimeFileService.listCalls != 1 { + t.Fatalf("runtime workspace service called %d times, want 1", runtimeFileService.listCalls) + } + if runtimeFileService.lastScope.WorkspacePath != "/config" { + t.Fatalf("runtime workspace path = %q, want /config", runtimeFileService.lastScope.WorkspacePath) + } +} + +func TestWorkspaceFileHandlerDownloadUsesSafeContentDisposition(t *testing.T) { + gin.SetMode(gin.TestMode) + workspacePath := t.TempDir() + downloadFile, err := os.CreateTemp(t.TempDir(), "download-*") + if err != nil { + t.Fatal(err) + } + if _, err := downloadFile.WriteString("body"); err != nil { + t.Fatal(err) + } + if _, err := downloadFile.Seek(0, io.SeekStart); err != nil { + t.Fatal(err) + } + + instanceService := &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{ + 77: {ID: 77, UserID: 10, WorkspacePath: &workspacePath}, + }} + fileService := &fakeWorkspaceFileService{file: downloadFile, filename: `bad"name/ignored.txt`, size: int64(len("body"))} + handler := NewWorkspaceFileHandler(instanceService, fileService) + + router := workspaceFileTestRouter(10, "user") + router.GET("/api/v1/instances/:id/workspace/download", handler.Download) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/77/workspace/download?path=report.txt", nil) + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String()) + } + disposition := rec.Header().Get("Content-Disposition") + if !strings.Contains(disposition, "attachment;") || strings.Contains(disposition, `"name/`) || strings.Contains(disposition, "\r") || strings.Contains(disposition, "\n") { + t.Fatalf("Content-Disposition = %q, want safe attachment filename", disposition) + } +} + +func workspaceFileTestRouter(userID int, role string) *gin.Engine { + router := gin.New() + router.Use(func(c *gin.Context) { + c.Set("userID", userID) + c.Set("userRole", role) + c.Next() + }) + return router +} diff --git a/backend/internal/middleware/auth_middleware.go b/backend/internal/middleware/auth_middleware.go index 076e7c7..521dfe2 100644 --- a/backend/internal/middleware/auth_middleware.go +++ b/backend/internal/middleware/auth_middleware.go @@ -51,7 +51,11 @@ func Auth() gin.HandlerFunc { } // GatewayAuth accepts either a normal user access JWT or an instance lifecycle gateway token. -func GatewayAuth(instanceRepo repository.InstanceRepository) gin.HandlerFunc { +func GatewayAuth(instanceRepo repository.InstanceRepository, bindingRepos ...repository.InstanceRuntimeBindingRepository) gin.HandlerFunc { + var bindingRepo repository.InstanceRuntimeBindingRepository + if len(bindingRepos) > 0 { + bindingRepo = bindingRepos[0] + } return func(c *gin.Context) { tokenString, ok := extractToken(c) if !ok { @@ -90,11 +94,34 @@ func GatewayAuth(instanceRepo repository.InstanceRepository) gin.HandlerFunc { c.Set("userID", instance.UserID) c.Set("instanceID", instance.ID) + c.Set("instanceMode", gatewayInstanceMode(instance.InstanceMode, instance.RuntimeType)) + c.Set("runtimeType", strings.TrimSpace(instance.RuntimeType)) + if bindingRepo != nil { + if binding, err := bindingRepo.GetRunningByInstanceID(c.Request.Context(), instance.ID); err == nil && binding != nil { + c.Set("gatewayID", strings.TrimSpace(binding.GatewayID)) + c.Set("runtimePodID", binding.RuntimePodID) + } + } c.Set("gatewayAuthType", "instance") c.Next() } } +func gatewayInstanceMode(instanceMode, runtimeType string) string { + mode := strings.ToLower(strings.TrimSpace(instanceMode)) + if mode == "lite" || mode == "pro" { + return mode + } + switch strings.ToLower(strings.TrimSpace(runtimeType)) { + case "gateway": + return "lite" + case "desktop", "shell": + return "pro" + default: + return mode + } +} + func extractToken(c *gin.Context) (string, bool) { authHeader := c.GetHeader("Authorization") if authHeader != "" { diff --git a/backend/internal/models/audit_event.go b/backend/internal/models/audit_event.go index 95874c0..a367d86 100644 --- a/backend/internal/models/audit_event.go +++ b/backend/internal/models/audit_event.go @@ -10,6 +10,10 @@ type AuditEvent struct { RequestID *string `db:"request_id" json:"request_id,omitempty"` UserID *int `db:"user_id" json:"user_id,omitempty"` InstanceID *int `db:"instance_id" json:"instance_id,omitempty"` + InstanceMode *string `db:"instance_mode" json:"instance_mode,omitempty"` + RuntimeType *string `db:"runtime_type" json:"runtime_type,omitempty"` + GatewayID *string `db:"gateway_id" json:"gateway_id,omitempty"` + RuntimePodID *int64 `db:"runtime_pod_id" json:"runtime_pod_id,omitempty"` InvocationID *int `db:"invocation_id" json:"invocation_id,omitempty"` EventType string `db:"event_type" json:"event_type"` TrafficClass string `db:"traffic_class" json:"traffic_class"` diff --git a/backend/internal/models/cost_record.go b/backend/internal/models/cost_record.go index a08fcda..7bdcf69 100644 --- a/backend/internal/models/cost_record.go +++ b/backend/internal/models/cost_record.go @@ -10,6 +10,10 @@ type CostRecord struct { RequestID *string `db:"request_id" json:"request_id,omitempty"` UserID *int `db:"user_id" json:"user_id,omitempty"` InstanceID *int `db:"instance_id" json:"instance_id,omitempty"` + InstanceMode *string `db:"instance_mode" json:"instance_mode,omitempty"` + RuntimeType *string `db:"runtime_type" json:"runtime_type,omitempty"` + GatewayID *string `db:"gateway_id" json:"gateway_id,omitempty"` + RuntimePodID *int64 `db:"runtime_pod_id" json:"runtime_pod_id,omitempty"` InvocationID *int `db:"invocation_id" json:"invocation_id,omitempty"` ModelID *int `db:"model_id" json:"model_id,omitempty"` ProviderType string `db:"provider_type" json:"provider_type"` diff --git a/backend/internal/models/instance.go b/backend/internal/models/instance.go index 2675d43..0de5d32 100644 --- a/backend/internal/models/instance.go +++ b/backend/internal/models/instance.go @@ -12,6 +12,7 @@ type Instance struct { Description *string `db:"description" json:"description,omitempty"` Type string `db:"type" json:"type"` RuntimeType string `db:"runtime_type" json:"runtime_type"` + InstanceMode string `db:"instance_mode" json:"instance_mode"` Status string `db:"status" json:"status"` CPUCores float64 `db:"cpu_cores" json:"cpu_cores"` MemoryGB int `db:"memory_gb" json:"memory_gb"` @@ -26,6 +27,10 @@ type Instance struct { EnvironmentOverridesJSON *string `db:"environment_overrides_json" json:"-"` StorageClass string `db:"storage_class" json:"storage_class"` MountPath string `db:"mount_path" json:"mount_path"` + WorkspacePath *string `db:"workspace_path" json:"workspace_path,omitempty"` + WorkspaceUsageBytes int64 `db:"workspace_usage_bytes" json:"workspace_usage_bytes"` + RuntimeGeneration int `db:"runtime_generation" json:"runtime_generation"` + RuntimeErrorMessage *string `db:"runtime_error_message" json:"runtime_error_message,omitempty"` PodName *string `db:"pod_name" json:"pod_name,omitempty"` PodNamespace *string `db:"pod_namespace" json:"pod_namespace,omitempty"` PodIP *string `db:"pod_ip" json:"pod_ip,omitempty"` diff --git a/backend/internal/models/instance_external_access.go b/backend/internal/models/instance_external_access.go new file mode 100644 index 0000000..1acc5f3 --- /dev/null +++ b/backend/internal/models/instance_external_access.go @@ -0,0 +1,25 @@ +package models + +import "time" + +type InstanceExternalAccess struct { + ID int64 `db:"id,primarykey,autoincrement" json:"id"` + InstanceID int `db:"instance_id" json:"instance_id"` + Enabled bool `db:"enabled" json:"enabled"` + AuthMode string `db:"auth_mode" json:"auth_mode"` + PublicSlug *string `db:"public_slug" json:"-"` + PublicTokenHash *string `db:"public_token_hash" json:"-"` + ShortCodeHash *string `db:"short_code_hash" json:"-"` + PasswordHash *string `db:"api_key_hash" json:"-"` + PasswordValue *string `db:"password_value" json:"-"` + PasswordHint *string `db:"api_key_prefix" json:"password_hint,omitempty"` + ExpiresAt *time.Time `db:"expires_at" json:"expires_at,omitempty"` + CreatedBy *int `db:"created_by" json:"created_by,omitempty"` + LastUsedAt *time.Time `db:"last_used_at" json:"last_used_at,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (InstanceExternalAccess) TableName() string { + return "instance_external_access" +} diff --git a/backend/internal/models/instance_runtime_binding.go b/backend/internal/models/instance_runtime_binding.go new file mode 100644 index 0000000..1dff9e5 --- /dev/null +++ b/backend/internal/models/instance_runtime_binding.go @@ -0,0 +1,24 @@ +package models + +import "time" + +type InstanceRuntimeBinding struct { + ID int64 `db:"id,primarykey,autoincrement" json:"id"` + InstanceID int `db:"instance_id" json:"instance_id"` + RuntimePodID int64 `db:"runtime_pod_id" json:"runtime_pod_id"` + RuntimeType string `db:"runtime_type" json:"runtime_type"` + GatewayID string `db:"gateway_id" json:"gateway_id"` + GatewayPort int `db:"gateway_port" json:"gateway_port"` + GatewayPID *int `db:"gateway_pid" json:"gateway_pid,omitempty"` + WorkspacePath string `db:"workspace_path" json:"workspace_path"` + State string `db:"state" json:"state"` + Generation int `db:"generation" json:"generation"` + LastHealthAt *time.Time `db:"last_health_at" json:"last_health_at,omitempty"` + ErrorMessage *string `db:"error_message" json:"error_message,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (InstanceRuntimeBinding) TableName() string { + return "instance_runtime_bindings" +} diff --git a/backend/internal/models/model_invocation.go b/backend/internal/models/model_invocation.go index 6a114be..9f9f87b 100644 --- a/backend/internal/models/model_invocation.go +++ b/backend/internal/models/model_invocation.go @@ -10,6 +10,10 @@ type ModelInvocation struct { RequestID string `db:"request_id" json:"request_id"` UserID *int `db:"user_id" json:"user_id,omitempty"` InstanceID *int `db:"instance_id" json:"instance_id,omitempty"` + InstanceMode *string `db:"instance_mode" json:"instance_mode,omitempty"` + RuntimeType *string `db:"runtime_type" json:"runtime_type,omitempty"` + GatewayID *string `db:"gateway_id" json:"gateway_id,omitempty"` + RuntimePodID *int64 `db:"runtime_pod_id" json:"runtime_pod_id,omitempty"` ModelID *int `db:"model_id" json:"model_id,omitempty"` ProviderType string `db:"provider_type" json:"provider_type"` RequestedModel string `db:"requested_model" json:"requested_model"` diff --git a/backend/internal/models/risk_hit.go b/backend/internal/models/risk_hit.go index abdaefb..3bdc613 100644 --- a/backend/internal/models/risk_hit.go +++ b/backend/internal/models/risk_hit.go @@ -10,6 +10,10 @@ type RiskHit struct { RequestID *string `db:"request_id" json:"request_id,omitempty"` UserID *int `db:"user_id" json:"user_id,omitempty"` InstanceID *int `db:"instance_id" json:"instance_id,omitempty"` + InstanceMode *string `db:"instance_mode" json:"instance_mode,omitempty"` + RuntimeType *string `db:"runtime_type" json:"runtime_type,omitempty"` + GatewayID *string `db:"gateway_id" json:"gateway_id,omitempty"` + RuntimePodID *int64 `db:"runtime_pod_id" json:"runtime_pod_id,omitempty"` InvocationID *int `db:"invocation_id" json:"invocation_id,omitempty"` RuleID string `db:"rule_id" json:"rule_id"` RuleName string `db:"rule_name" json:"rule_name"` diff --git a/backend/internal/models/runtime_pod.go b/backend/internal/models/runtime_pod.go new file mode 100644 index 0000000..69c90ef --- /dev/null +++ b/backend/internal/models/runtime_pod.go @@ -0,0 +1,33 @@ +package models + +import "time" + +type RuntimePod struct { + ID int64 `db:"id,primarykey,autoincrement" json:"id"` + RuntimeType string `db:"runtime_type" json:"runtime_type"` + Namespace string `db:"namespace" json:"namespace"` + PodName string `db:"pod_name" json:"pod_name"` + PodUID *string `db:"pod_uid" json:"pod_uid,omitempty"` + PodIP *string `db:"pod_ip" json:"pod_ip,omitempty"` + NodeName *string `db:"node_name" json:"node_name,omitempty"` + DeploymentName string `db:"deployment_name" json:"deployment_name"` + ImageRef string `db:"image_ref" json:"image_ref"` + AgentEndpoint *string `db:"agent_endpoint" json:"agent_endpoint,omitempty"` + State string `db:"state" json:"state"` + Capacity int `db:"capacity" json:"capacity"` + UsedSlots int `db:"used_slots" json:"used_slots"` + Draining bool `db:"draining" json:"draining"` + CPUMillisUsed int64 `db:"cpu_millis_used" json:"cpu_millis_used"` + MemoryBytesUsed int64 `db:"memory_bytes_used" json:"memory_bytes_used"` + DiskBytesUsed int64 `db:"disk_bytes_used" json:"disk_bytes_used"` + NetworkRXBytes int64 `db:"network_rx_bytes" json:"network_rx_bytes"` + NetworkTXBytes int64 `db:"network_tx_bytes" json:"network_tx_bytes"` + MetricsJSON *string `db:"metrics_json" json:"-"` + LastSeenAt *time.Time `db:"last_seen_at" json:"last_seen_at,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (RuntimePod) TableName() string { + return "runtime_pods" +} diff --git a/backend/internal/models/runtime_rollout.go b/backend/internal/models/runtime_rollout.go new file mode 100644 index 0000000..5b7502e --- /dev/null +++ b/backend/internal/models/runtime_rollout.go @@ -0,0 +1,22 @@ +package models + +import "time" + +type RuntimeRollout struct { + ID int64 `db:"id,primarykey,autoincrement" json:"id"` + RuntimeType string `db:"runtime_type" json:"runtime_type"` + TargetImageRef string `db:"target_image_ref" json:"target_image_ref"` + Status string `db:"status" json:"status"` + BatchSize int `db:"batch_size" json:"batch_size"` + MaxUnavailable int `db:"max_unavailable" json:"max_unavailable"` + StartedBy *int `db:"started_by" json:"started_by,omitempty"` + StartedAt *time.Time `db:"started_at" json:"started_at,omitempty"` + FinishedAt *time.Time `db:"finished_at" json:"finished_at,omitempty"` + ErrorMessage *string `db:"error_message" json:"error_message,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (RuntimeRollout) TableName() string { + return "runtime_rollouts" +} diff --git a/backend/internal/models/team.go b/backend/internal/models/team.go index 7f5fc8d..080c3df 100644 --- a/backend/internal/models/team.go +++ b/backend/internal/models/team.go @@ -65,6 +65,7 @@ type TeamMember struct { DisplayName string `db:"display_name" json:"display_name"` Role string `db:"role" json:"role"` RuntimeType string `db:"runtime_type" json:"runtime_type"` + InstanceMode string `db:"instance_mode" json:"instance_mode"` Description *string `db:"description" json:"description,omitempty"` Status string `db:"status" json:"status"` CurrentTaskID *int `db:"current_task_id" json:"current_task_id,omitempty"` diff --git a/backend/internal/models/workspace_file_audit.go b/backend/internal/models/workspace_file_audit.go new file mode 100644 index 0000000..eb28539 --- /dev/null +++ b/backend/internal/models/workspace_file_audit.go @@ -0,0 +1,17 @@ +package models + +import "time" + +type WorkspaceFileAudit struct { + ID int64 `db:"id,primarykey,autoincrement" json:"id"` + InstanceID int `db:"instance_id" json:"instance_id"` + UserID int `db:"user_id" json:"user_id"` + Action string `db:"action" json:"action"` + RelativePath string `db:"relative_path" json:"relative_path"` + Bytes int64 `db:"bytes" json:"bytes"` + CreatedAt time.Time `db:"created_at" json:"created_at"` +} + +func (WorkspaceFileAudit) TableName() string { + return "workspace_file_audits" +} diff --git a/backend/internal/repository/audit_event_repository.go b/backend/internal/repository/audit_event_repository.go index 84afac6..377a364 100644 --- a/backend/internal/repository/audit_event_repository.go +++ b/backend/internal/repository/audit_event_repository.go @@ -36,6 +36,10 @@ CREATE TABLE IF NOT EXISTS audit_events ( request_id VARCHAR(100) NULL, user_id INT NULL, instance_id INT NULL, + instance_mode VARCHAR(16) NULL, + runtime_type VARCHAR(32) NULL, + gateway_id VARCHAR(128) NULL, + runtime_pod_id BIGINT NULL, invocation_id INT NULL, event_type VARCHAR(100) NOT NULL, traffic_class VARCHAR(50) NOT NULL, @@ -46,6 +50,7 @@ CREATE TABLE IF NOT EXISTS audit_events ( INDEX idx_audit_events_trace_id (trace_id), INDEX idx_audit_events_request_id (request_id), INDEX idx_audit_events_user_id (user_id), + INDEX idx_audit_events_gateway_id (gateway_id), INDEX idx_audit_events_invocation_id (invocation_id), INDEX idx_audit_events_event_type (event_type), INDEX idx_audit_events_created_at (created_at) diff --git a/backend/internal/repository/cost_record_repository.go b/backend/internal/repository/cost_record_repository.go index e305732..51bb713 100644 --- a/backend/internal/repository/cost_record_repository.go +++ b/backend/internal/repository/cost_record_repository.go @@ -37,6 +37,10 @@ CREATE TABLE IF NOT EXISTS cost_records ( request_id VARCHAR(100) NULL, user_id INT NULL, instance_id INT NULL, + instance_mode VARCHAR(16) NULL, + runtime_type VARCHAR(32) NULL, + gateway_id VARCHAR(128) NULL, + runtime_pod_id BIGINT NULL, invocation_id INT NULL, model_id INT NULL, provider_type VARCHAR(100) NOT NULL, @@ -53,6 +57,7 @@ CREATE TABLE IF NOT EXISTS cost_records ( recorded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, INDEX idx_cost_records_trace_id (trace_id), INDEX idx_cost_records_user_id (user_id), + INDEX idx_cost_records_gateway_id (gateway_id), INDEX idx_cost_records_model_id (model_id), INDEX idx_cost_records_provider_type (provider_type), INDEX idx_cost_records_recorded_at (recorded_at) diff --git a/backend/internal/repository/instance_external_access_repository.go b/backend/internal/repository/instance_external_access_repository.go new file mode 100644 index 0000000..cd99504 --- /dev/null +++ b/backend/internal/repository/instance_external_access_repository.go @@ -0,0 +1,107 @@ +package repository + +import ( + "context" + "fmt" + "time" + + "clawreef/internal/models" + "github.com/upper/db/v4" +) + +type InstanceExternalAccessRepository interface { + GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceExternalAccess, error) + GetByShortCodeHash(ctx context.Context, codeHash string) (*models.InstanceExternalAccess, error) + Upsert(ctx context.Context, access *models.InstanceExternalAccess) error + Disable(ctx context.Context, instanceID int) error + MarkUsed(ctx context.Context, id int64) error +} + +type instanceExternalAccessRepository struct { + sess db.Session +} + +func NewInstanceExternalAccessRepository(sess db.Session) InstanceExternalAccessRepository { + return &instanceExternalAccessRepository{sess: sess} +} + +func (r *instanceExternalAccessRepository) GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceExternalAccess, error) { + _ = ctx + var access models.InstanceExternalAccess + if err := r.sess.Collection(access.TableName()).Find(db.Cond{"instance_id": instanceID}).One(&access); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get instance external access: %w", err) + } + return &access, nil +} + +func (r *instanceExternalAccessRepository) GetByShortCodeHash(ctx context.Context, codeHash string) (*models.InstanceExternalAccess, error) { + _ = ctx + var access models.InstanceExternalAccess + if err := r.sess.Collection(access.TableName()).Find(db.Cond{"short_code_hash": codeHash}).One(&access); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get instance external access by short code hash: %w", err) + } + return &access, nil +} + +func (r *instanceExternalAccessRepository) Upsert(ctx context.Context, access *models.InstanceExternalAccess) error { + now := time.Now().UTC() + if access.CreatedAt.IsZero() { + access.CreatedAt = now + } + access.UpdatedAt = now + _, err := r.sess.SQL().ExecContext(ctx, ` + INSERT INTO instance_external_access ( + instance_id, enabled, auth_mode, public_slug, short_code_hash, public_token_hash, + api_key_hash, password_value, api_key_prefix, expires_at, created_by, + created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + enabled = VALUES(enabled), + auth_mode = VALUES(auth_mode), + public_slug = VALUES(public_slug), + short_code_hash = VALUES(short_code_hash), + public_token_hash = VALUES(public_token_hash), + api_key_hash = VALUES(api_key_hash), + password_value = VALUES(password_value), + api_key_prefix = VALUES(api_key_prefix), + expires_at = VALUES(expires_at), + created_by = VALUES(created_by), + updated_at = VALUES(updated_at) + `, access.InstanceID, access.Enabled, access.AuthMode, access.PublicSlug, access.ShortCodeHash, access.PublicTokenHash, + access.PasswordHash, access.PasswordValue, access.PasswordHint, access.ExpiresAt, access.CreatedBy, + access.CreatedAt, access.UpdatedAt) + if err != nil { + return fmt.Errorf("failed to upsert instance external access: %w", err) + } + return nil +} + +func (r *instanceExternalAccessRepository) Disable(ctx context.Context, instanceID int) error { + _, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE instance_external_access + SET enabled = 0, updated_at = ? + WHERE instance_id = ? + `, time.Now().UTC(), instanceID) + if err != nil { + return fmt.Errorf("failed to disable instance external access: %w", err) + } + return nil +} + +func (r *instanceExternalAccessRepository) MarkUsed(ctx context.Context, id int64) error { + _, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE instance_external_access + SET last_used_at = ?, updated_at = ? + WHERE id = ? + `, time.Now().UTC(), time.Now().UTC(), id) + if err != nil { + return fmt.Errorf("failed to mark instance external access used: %w", err) + } + return nil +} diff --git a/backend/internal/repository/instance_repository.go b/backend/internal/repository/instance_repository.go index 1102afd..1b593e3 100644 --- a/backend/internal/repository/instance_repository.go +++ b/backend/internal/repository/instance_repository.go @@ -1,13 +1,19 @@ package repository import ( + "context" + "database/sql" + "errors" "fmt" "strings" + "time" "clawreef/internal/models" "github.com/upper/db/v4" ) +var ErrStaleRuntimeGeneration = errors.New("stale runtime generation") + // InstanceRepository defines the interface for instance data operations type InstanceRepository interface { Create(instance *models.Instance) error @@ -18,8 +24,14 @@ type InstanceRepository interface { CountAll() (int, error) GetByUserID(userID int, offset, limit int) ([]models.Instance, error) CountByUserID(userID int) (int, error) + CountActiveByMode(ctx context.Context, mode string) (int, error) ExistsByUserIDAndName(userID int, name string) (bool, error) GetAllRunning() ([]models.Instance, error) + GetV2DesiredRunning(ctx context.Context, limit int) ([]models.Instance, error) + GetV2Creating(ctx context.Context, limit int) ([]models.Instance, error) + UpdateRuntimeState(ctx context.Context, id int, status string, generation int, message *string) error + SetWorkspacePath(ctx context.Context, id int, workspacePath string) error + UpdateWorkspaceUsage(ctx context.Context, id int, usageBytes int64) error Update(instance *models.Instance) error Delete(id int) error } @@ -121,6 +133,27 @@ func (r *instanceRepository) CountByUserID(userID int) (int, error) { return int(count), nil } +func (r *instanceRepository) CountActiveByMode(ctx context.Context, mode string) (int, error) { + normalized := strings.TrimSpace(strings.ToLower(mode)) + if normalized == "" { + return 0, nil + } + row, err := r.sess.SQL().QueryRowContext(ctx, ` + SELECT COUNT(*) + FROM instances + WHERE instance_mode = ? + AND status IN ('creating', 'running') + `, normalized) + if err != nil { + return 0, fmt.Errorf("failed to count active instances by mode: %w", err) + } + var count int + if err := row.Scan(&count); err != nil { + return 0, fmt.Errorf("failed to scan active instances by mode count: %w", err) + } + return count, nil +} + // ExistsByUserIDAndName checks whether the user already has an instance with the same display name. func (r *instanceRepository) ExistsByUserIDAndName(userID int, name string) (bool, error) { instances, err := r.GetByUserID(userID, 0, 1000) @@ -155,6 +188,139 @@ func (r *instanceRepository) GetAllRunning() ([]models.Instance, error) { return instances, nil } +func (r *instanceRepository) GetV2DesiredRunning(ctx context.Context, limit int) ([]models.Instance, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if limit <= 0 { + limit = 100 + } + var instances []models.Instance + query, args := buildV2SchedulerInstanceQuery(v2DesiredRunningStatuses(), limit) + iter := r.sess.SQL().IteratorContext(ctx, query, args...) + defer iter.Close() + if err := iter.All(&instances); err != nil { + return nil, fmt.Errorf("failed to get v2 desired running instances: %w", err) + } + return instances, nil +} + +func v2DesiredRunningStatuses() []string { + return []string{"creating", "running", "error"} +} + +func (r *instanceRepository) GetV2Creating(ctx context.Context, limit int) ([]models.Instance, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if limit <= 0 { + limit = 100 + } + var instances []models.Instance + query, args := buildV2SchedulerInstanceQuery([]string{"creating"}, limit) + iter := r.sess.SQL().IteratorContext(ctx, query, args...) + defer iter.Close() + if err := iter.All(&instances); err != nil { + return nil, fmt.Errorf("failed to get v2 creating instances: %w", err) + } + return instances, nil +} + +func buildV2SchedulerInstanceQuery(statuses []string, limit int) (string, []any) { + if limit <= 0 { + limit = 100 + } + if len(statuses) == 0 { + statuses = []string{"creating", "running"} + } + statusPlaceholders := strings.TrimRight(strings.Repeat("?, ", len(statuses)), ", ") + args := make([]any, 0, len(statuses)+5) + for _, status := range statuses { + args = append(args, status) + } + args = append(args, "gateway", "lite", "openclaw", "hermes", limit) + return fmt.Sprintf(` + SELECT * + FROM instances + WHERE status IN (%s) + AND runtime_type = ? + AND instance_mode = ? + AND workspace_path IS NOT NULL + AND TRIM(workspace_path) <> '' + AND type IN (?, ?) + ORDER BY id + LIMIT ? + `, statusPlaceholders), args +} + +func (r *instanceRepository) UpdateRuntimeState(ctx context.Context, id int, status string, generation int, message *string) error { + res, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE instances + SET status = ?, runtime_generation = ?, runtime_error_message = ?, updated_at = ? + WHERE id = ? AND runtime_generation <= ? + `, status, generation, message, time.Now().UTC(), id, generation) + if err != nil { + return fmt.Errorf("failed to update instance runtime state: %w", err) + } + affected, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("failed to inspect instance runtime state update: %w", err) + } + if affected == 0 { + currentGeneration, err := r.getRuntimeGeneration(ctx, id) + if err != nil { + return err + } + if currentGeneration > generation { + return ErrStaleRuntimeGeneration + } + } + return nil +} + +func (r *instanceRepository) getRuntimeGeneration(ctx context.Context, id int) (int, error) { + var currentGeneration int + row, err := r.sess.SQL().QueryRowContext(ctx, ` + SELECT runtime_generation + FROM instances + WHERE id = ? + `, id) + if err != nil { + return 0, fmt.Errorf("failed to query instance runtime generation: %w", err) + } + if err := row.Scan(¤tGeneration); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return 0, ErrStaleRuntimeGeneration + } + return 0, fmt.Errorf("failed to scan instance runtime generation: %w", err) + } + return currentGeneration, nil +} + +func (r *instanceRepository) SetWorkspacePath(ctx context.Context, id int, workspacePath string) error { + _, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE instances + SET workspace_path = ?, updated_at = ? + WHERE id = ? + `, workspacePath, time.Now().UTC(), id) + if err != nil { + return fmt.Errorf("failed to set instance workspace path: %w", err) + } + return nil +} + +func (r *instanceRepository) UpdateWorkspaceUsage(ctx context.Context, id int, usageBytes int64) error { + _, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE instances + SET workspace_usage_bytes = ?, updated_at = ? + WHERE id = ? + `, usageBytes, time.Now().UTC(), id) + if err != nil { + return fmt.Errorf("failed to update instance workspace usage: %w", err) + } + return nil +} + // Update updates an instance func (r *instanceRepository) Update(instance *models.Instance) error { err := r.sess.Collection("instances").Find(db.Cond{"id": instance.ID}).Update(instance) diff --git a/backend/internal/repository/instance_repository_test.go b/backend/internal/repository/instance_repository_test.go new file mode 100644 index 0000000..1a45955 --- /dev/null +++ b/backend/internal/repository/instance_repository_test.go @@ -0,0 +1,58 @@ +package repository + +import ( + "reflect" + "strings" + "testing" +) + +func TestBuildV2SchedulerInstanceQueryRequiresWorkspaceTypeAndStatuses(t *testing.T) { + query, args := buildV2SchedulerInstanceQuery([]string{"creating", "running"}, 25) + normalized := normalizeSQLForTest(query) + + requiredFragments := []string{ + "FROM instances", + "status IN (?, ?)", + "runtime_type = ?", + "instance_mode = ?", + "workspace_path IS NOT NULL", + "TRIM(workspace_path) <> ''", + "type IN (?, ?)", + "ORDER BY id", + "LIMIT ?", + } + for _, fragment := range requiredFragments { + if !strings.Contains(normalized, fragment) { + t.Fatalf("query %q does not contain %q", normalized, fragment) + } + } + + wantArgs := []any{"creating", "running", "gateway", "lite", "openclaw", "hermes", 25} + if !reflect.DeepEqual(args, wantArgs) { + t.Fatalf("args = %#v, want %#v", args, wantArgs) + } +} + +func TestV2DesiredRunningStatusesIncludeRecoverableErrorCandidates(t *testing.T) { + want := []string{"creating", "running", "error"} + if got := v2DesiredRunningStatuses(); !reflect.DeepEqual(got, want) { + t.Fatalf("desired running statuses = %#v, want %#v", got, want) + } +} + +func TestBuildV2SchedulerInstanceQueryDefaultsLimit(t *testing.T) { + query, args := buildV2SchedulerInstanceQuery([]string{"creating"}, 0) + normalized := normalizeSQLForTest(query) + + if !strings.Contains(normalized, "status IN (?)") { + t.Fatalf("query %q does not contain single status predicate", normalized) + } + wantArgs := []any{"creating", "gateway", "lite", "openclaw", "hermes", 100} + if !reflect.DeepEqual(args, wantArgs) { + t.Fatalf("args = %#v, want %#v", args, wantArgs) + } +} + +func normalizeSQLForTest(query string) string { + return strings.Join(strings.Fields(query), " ") +} diff --git a/backend/internal/repository/instance_runtime_binding_repository.go b/backend/internal/repository/instance_runtime_binding_repository.go new file mode 100644 index 0000000..f5d2969 --- /dev/null +++ b/backend/internal/repository/instance_runtime_binding_repository.go @@ -0,0 +1,213 @@ +package repository + +import ( + "context" + "database/sql" + "errors" + "fmt" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +type InstanceRuntimeBindingRepository interface { + Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error + GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) + GetRunningByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) + ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error) + ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error) + UpdateRunning(ctx context.Context, instanceID int, generation int, gatewayID string, port int, pid *int) error + UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error + DeleteByInstanceID(ctx context.Context, instanceID int) error + DeleteByInstanceIDAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64) error +} + +type instanceRuntimeBindingRepository struct { + sess db.Session +} + +func NewInstanceRuntimeBindingRepository(sess db.Session) InstanceRuntimeBindingRepository { + return &instanceRuntimeBindingRepository{sess: sess} +} + +func (r *instanceRuntimeBindingRepository) Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error { + if err := ctx.Err(); err != nil { + return err + } + ensureTimestamps(&binding.CreatedAt, &binding.UpdatedAt) + res, err := r.sess.Collection("instance_runtime_bindings").Insert(binding) + if err != nil { + return fmt.Errorf("failed to create instance runtime binding: %w", err) + } + if id, ok := res.ID().(int64); ok { + binding.ID = id + } + return nil +} + +func (r *instanceRuntimeBindingRepository) GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + var binding models.InstanceRuntimeBinding + if err := r.sess.Collection("instance_runtime_bindings").Find(db.Cond{"instance_id": instanceID}).One(&binding); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get instance runtime binding: %w", err) + } + return &binding, nil +} + +func (r *instanceRuntimeBindingRepository) GetRunningByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + var binding models.InstanceRuntimeBinding + if err := r.sess.Collection("instance_runtime_bindings").Find(db.Cond{"instance_id": instanceID, "state": "running"}).One(&binding); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get running instance runtime binding: %w", err) + } + return &binding, nil +} + +func (r *instanceRuntimeBindingRepository) ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + var bindings []models.InstanceRuntimeBinding + if err := r.sess.Collection("instance_runtime_bindings").Find(db.Cond{"runtime_pod_id": runtimePodID}).OrderBy("id").All(&bindings); err != nil { + return nil, fmt.Errorf("failed to list instance runtime bindings: %w", err) + } + return bindings, nil +} + +func (r *instanceRuntimeBindingRepository) ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if len(runtimePodIDs) == 0 { + return []models.InstanceRuntimeBinding{}, nil + } + var bindings []models.InstanceRuntimeBinding + if err := r.sess.Collection("instance_runtime_bindings").Find(db.Cond{"runtime_pod_id IN": runtimePodIDs}).OrderBy("runtime_pod_id", "id").All(&bindings); err != nil { + return nil, fmt.Errorf("failed to list instance runtime bindings by pods: %w", err) + } + return bindings, nil +} + +func (r *instanceRuntimeBindingRepository) UpdateRunning(ctx context.Context, instanceID int, generation int, gatewayID string, port int, pid *int) error { + now := time.Now().UTC() + res, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE instance_runtime_bindings + SET state = 'running', generation = ?, gateway_id = ?, gateway_port = ?, gateway_pid = ?, + last_health_at = ?, error_message = NULL, updated_at = ? + WHERE instance_id = ? AND generation <= ? + `, generation, gatewayID, port, pid, now, now, instanceID, generation) + if err != nil { + return fmt.Errorf("failed to update running instance runtime binding: %w", err) + } + affected, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("failed to inspect running instance runtime binding update: %w", err) + } + if affected == 0 { + currentGeneration, err := r.getGeneration(ctx, instanceID) + if err != nil { + return err + } + if currentGeneration > generation { + return ErrStaleRuntimeGeneration + } + } + return nil +} + +func (r *instanceRuntimeBindingRepository) UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error { + res, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE instance_runtime_bindings + SET state = ?, generation = ?, error_message = ?, updated_at = ? + WHERE instance_id = ? AND generation <= ? + `, state, generation, message, time.Now().UTC(), instanceID, generation) + if err != nil { + return fmt.Errorf("failed to update instance runtime binding state: %w", err) + } + affected, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("failed to inspect instance runtime binding state update: %w", err) + } + if affected == 0 { + currentGeneration, err := r.getGeneration(ctx, instanceID) + if err != nil { + return err + } + if currentGeneration > generation { + return ErrStaleRuntimeGeneration + } + } + return nil +} + +func (r *instanceRuntimeBindingRepository) getGeneration(ctx context.Context, instanceID int) (int, error) { + var currentGeneration int + row, err := r.sess.SQL().QueryRowContext(ctx, ` + SELECT generation + FROM instance_runtime_bindings + WHERE instance_id = ? + `, instanceID) + if err != nil { + return 0, fmt.Errorf("failed to query instance runtime binding generation: %w", err) + } + if err := row.Scan(¤tGeneration); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return 0, ErrStaleRuntimeGeneration + } + return 0, fmt.Errorf("failed to scan instance runtime binding generation: %w", err) + } + return currentGeneration, nil +} + +func (r *instanceRuntimeBindingRepository) DeleteByInstanceID(ctx context.Context, instanceID int) error { + _, err := r.sess.SQL().ExecContext(ctx, ` + DELETE FROM instance_runtime_bindings + WHERE instance_id = ? + `, instanceID) + if err != nil { + return fmt.Errorf("failed to delete instance runtime binding: %w", err) + } + return nil +} + +func (r *instanceRuntimeBindingRepository) DeleteByInstanceIDAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64) error { + if err := ctx.Err(); err != nil { + return err + } + return r.sess.TxContext(ctx, func(tx db.Session) error { + res, err := tx.SQL().ExecContext(ctx, ` + DELETE FROM instance_runtime_bindings + WHERE instance_id = ? AND runtime_pod_id = ? + `, instanceID, runtimePodID) + if err != nil { + return fmt.Errorf("failed to delete instance runtime binding: %w", err) + } + affected, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("failed to inspect instance runtime binding delete: %w", err) + } + if affected == 0 { + return nil + } + if _, err := tx.SQL().ExecContext(ctx, ` + UPDATE runtime_pods + SET used_slots = CASE WHEN used_slots > 0 THEN used_slots - 1 ELSE 0 END, updated_at = ? + WHERE id = ? + `, time.Now().UTC(), runtimePodID); err != nil { + return fmt.Errorf("failed to release runtime pod slot: %w", err) + } + return nil + }, nil) +} diff --git a/backend/internal/repository/model_invocation_repository.go b/backend/internal/repository/model_invocation_repository.go index c713f3f..a581cd7 100644 --- a/backend/internal/repository/model_invocation_repository.go +++ b/backend/internal/repository/model_invocation_repository.go @@ -40,6 +40,10 @@ CREATE TABLE IF NOT EXISTS model_invocations ( request_id VARCHAR(100) NOT NULL, user_id INT NULL, instance_id INT NULL, + instance_mode VARCHAR(16) NULL, + runtime_type VARCHAR(32) NULL, + gateway_id VARCHAR(128) NULL, + runtime_pod_id BIGINT NULL, model_id INT NULL, provider_type VARCHAR(100) NOT NULL, requested_model VARCHAR(255) NOT NULL, @@ -62,6 +66,7 @@ CREATE TABLE IF NOT EXISTS model_invocations ( INDEX idx_model_invocations_request_id (request_id), INDEX idx_model_invocations_user_id (user_id), INDEX idx_model_invocations_instance_id (instance_id), + INDEX idx_model_invocations_gateway_id (gateway_id), INDEX idx_model_invocations_model_id (model_id), INDEX idx_model_invocations_status (status), INDEX idx_model_invocations_created_at (created_at) diff --git a/backend/internal/repository/risk_hit_repository.go b/backend/internal/repository/risk_hit_repository.go index ecad93b..864187c 100644 --- a/backend/internal/repository/risk_hit_repository.go +++ b/backend/internal/repository/risk_hit_repository.go @@ -35,6 +35,10 @@ CREATE TABLE IF NOT EXISTS risk_hits ( request_id VARCHAR(100) NULL, user_id INT NULL, instance_id INT NULL, + instance_mode VARCHAR(16) NULL, + runtime_type VARCHAR(32) NULL, + gateway_id VARCHAR(128) NULL, + runtime_pod_id BIGINT NULL, invocation_id INT NULL, rule_id VARCHAR(100) NOT NULL, rule_name VARCHAR(255) NOT NULL, @@ -45,6 +49,7 @@ CREATE TABLE IF NOT EXISTS risk_hits ( INDEX idx_risk_hits_trace_id (trace_id), INDEX idx_risk_hits_request_id (request_id), INDEX idx_risk_hits_user_id (user_id), + INDEX idx_risk_hits_gateway_id (gateway_id), INDEX idx_risk_hits_invocation_id (invocation_id), INDEX idx_risk_hits_severity (severity), INDEX idx_risk_hits_created_at (created_at) diff --git a/backend/internal/repository/runtime_pod_repository.go b/backend/internal/repository/runtime_pod_repository.go new file mode 100644 index 0000000..e168a12 --- /dev/null +++ b/backend/internal/repository/runtime_pod_repository.go @@ -0,0 +1,224 @@ +package repository + +import ( + "context" + "fmt" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +type RuntimePodMetricsUpdate struct { + CPUMillisUsed int64 + MemoryBytesUsed int64 + DiskBytesUsed int64 + NetworkRXBytes int64 + NetworkTXBytes int64 + MetricsJSON *string + LastSeenAt *time.Time +} + +type RuntimePodRepository interface { + UpsertFromAgent(ctx context.Context, pod *models.RuntimePod) error + GetByID(ctx context.Context, id int64) (*models.RuntimePod, error) + GetByNamespaceName(ctx context.Context, namespace, podName string) (*models.RuntimePod, error) + List(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) + ListSchedulable(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) + TryClaimSlot(ctx context.Context, podID int64) (bool, error) + ReleaseSlot(ctx context.Context, podID int64) error + MarkState(ctx context.Context, podID int64, state string, draining bool) error + UpdateHeartbeat(ctx context.Context, podID int64, state string, usedSlots int, capacity int, draining bool, lastSeenAt time.Time) error + MarkUnseenUnhealthy(ctx context.Context, cutoff time.Time) error + UpdateMetrics(ctx context.Context, podID int64, metrics RuntimePodMetricsUpdate) error +} + +type runtimePodRepository struct { + sess db.Session +} + +func NewRuntimePodRepository(sess db.Session) RuntimePodRepository { + return &runtimePodRepository{sess: sess} +} + +func (r *runtimePodRepository) UpsertFromAgent(ctx context.Context, pod *models.RuntimePod) error { + ensureTimestamps(&pod.CreatedAt, &pod.UpdatedAt) + res, err := r.sess.SQL().ExecContext(ctx, ` + INSERT INTO runtime_pods ( + runtime_type, namespace, pod_name, pod_uid, pod_ip, node_name, deployment_name, + image_ref, agent_endpoint, state, capacity, used_slots, draining, cpu_millis_used, + memory_bytes_used, disk_bytes_used, network_rx_bytes, network_tx_bytes, metrics_json, + last_seen_at, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + id = LAST_INSERT_ID(id), + runtime_type = VALUES(runtime_type), + pod_uid = VALUES(pod_uid), + pod_ip = VALUES(pod_ip), + node_name = VALUES(node_name), + deployment_name = VALUES(deployment_name), + image_ref = VALUES(image_ref), + agent_endpoint = VALUES(agent_endpoint), + capacity = VALUES(capacity), + cpu_millis_used = VALUES(cpu_millis_used), + memory_bytes_used = VALUES(memory_bytes_used), + disk_bytes_used = VALUES(disk_bytes_used), + network_rx_bytes = VALUES(network_rx_bytes), + network_tx_bytes = VALUES(network_tx_bytes), + metrics_json = VALUES(metrics_json), + last_seen_at = VALUES(last_seen_at), + updated_at = VALUES(updated_at) + `, pod.RuntimeType, pod.Namespace, pod.PodName, pod.PodUID, pod.PodIP, pod.NodeName, pod.DeploymentName, + pod.ImageRef, pod.AgentEndpoint, pod.State, pod.Capacity, pod.UsedSlots, pod.Draining, pod.CPUMillisUsed, + pod.MemoryBytesUsed, pod.DiskBytesUsed, pod.NetworkRXBytes, pod.NetworkTXBytes, pod.MetricsJSON, + pod.LastSeenAt, pod.CreatedAt, pod.UpdatedAt) + if err != nil { + return fmt.Errorf("failed to upsert runtime pod: %w", err) + } + if id, err := res.LastInsertId(); err == nil { + pod.ID = id + } + return nil +} + +func (r *runtimePodRepository) GetByID(ctx context.Context, id int64) (*models.RuntimePod, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + var pod models.RuntimePod + if err := r.sess.Collection("runtime_pods").Find(db.Cond{"id": id}).One(&pod); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get runtime pod: %w", err) + } + return &pod, nil +} + +func (r *runtimePodRepository) GetByNamespaceName(ctx context.Context, namespace, podName string) (*models.RuntimePod, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + var pod models.RuntimePod + if err := r.sess.Collection("runtime_pods").Find(db.Cond{"namespace": namespace, "pod_name": podName}).One(&pod); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get runtime pod by namespace/name: %w", err) + } + return &pod, nil +} + +func (r *runtimePodRepository) List(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + cond := db.Cond{} + if runtimeType != "" { + cond["runtime_type"] = runtimeType + } + var pods []models.RuntimePod + if err := r.sess.Collection("runtime_pods").Find(cond).OrderBy("namespace", "pod_name").All(&pods); err != nil { + return nil, fmt.Errorf("failed to list runtime pods: %w", err) + } + return pods, nil +} + +func (r *runtimePodRepository) ListSchedulable(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + var pods []models.RuntimePod + iter := r.sess.SQL().IteratorContext(ctx, ` + SELECT * + FROM runtime_pods + WHERE runtime_type = ? AND state = 'ready' AND draining = 0 AND used_slots < capacity + ORDER BY used_slots, id + `, runtimeType) + defer iter.Close() + if err := iter.All(&pods); err != nil { + return nil, fmt.Errorf("failed to list schedulable runtime pods: %w", err) + } + return pods, nil +} + +func (r *runtimePodRepository) TryClaimSlot(ctx context.Context, podID int64) (bool, error) { + res, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE runtime_pods + SET used_slots = used_slots + 1, updated_at = ? + WHERE id = ? AND state = 'ready' AND draining = 0 AND used_slots < capacity + `, time.Now().UTC(), podID) + if err != nil { + return false, fmt.Errorf("failed to claim runtime pod slot: %w", err) + } + affected, err := res.RowsAffected() + if err != nil { + return false, fmt.Errorf("failed to inspect runtime pod slot claim: %w", err) + } + return affected == 1, nil +} + +func (r *runtimePodRepository) ReleaseSlot(ctx context.Context, podID int64) error { + _, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE runtime_pods + SET used_slots = CASE WHEN used_slots > 0 THEN used_slots - 1 ELSE 0 END, updated_at = ? + WHERE id = ? + `, time.Now().UTC(), podID) + if err != nil { + return fmt.Errorf("failed to release runtime pod slot: %w", err) + } + return nil +} + +func (r *runtimePodRepository) MarkState(ctx context.Context, podID int64, state string, draining bool) error { + _, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE runtime_pods + SET state = ?, draining = ?, updated_at = ? + WHERE id = ? + `, state, draining, time.Now().UTC(), podID) + if err != nil { + return fmt.Errorf("failed to mark runtime pod state: %w", err) + } + return nil +} + +func (r *runtimePodRepository) UpdateHeartbeat(ctx context.Context, podID int64, state string, usedSlots int, capacity int, draining bool, lastSeenAt time.Time) error { + _, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE runtime_pods + SET state = ?, used_slots = ?, capacity = ?, draining = ?, last_seen_at = ?, updated_at = ? + WHERE id = ? + `, state, usedSlots, capacity, draining, lastSeenAt, time.Now().UTC(), podID) + if err != nil { + return fmt.Errorf("failed to update runtime pod heartbeat: %w", err) + } + return nil +} + +func (r *runtimePodRepository) MarkUnseenUnhealthy(ctx context.Context, cutoff time.Time) error { + _, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE runtime_pods + SET state = 'unhealthy', updated_at = ? + WHERE (last_seen_at IS NULL OR last_seen_at < ?) AND state <> 'unhealthy' + `, time.Now().UTC(), cutoff) + if err != nil { + return fmt.Errorf("failed to mark unseen runtime pods unhealthy: %w", err) + } + return nil +} + +func (r *runtimePodRepository) UpdateMetrics(ctx context.Context, podID int64, metrics RuntimePodMetricsUpdate) error { + _, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE runtime_pods + SET cpu_millis_used = ?, memory_bytes_used = ?, disk_bytes_used = ?, + network_rx_bytes = ?, network_tx_bytes = ?, metrics_json = ?, + last_seen_at = ?, updated_at = ? + WHERE id = ? + `, metrics.CPUMillisUsed, metrics.MemoryBytesUsed, metrics.DiskBytesUsed, + metrics.NetworkRXBytes, metrics.NetworkTXBytes, metrics.MetricsJSON, + metrics.LastSeenAt, time.Now().UTC(), podID) + if err != nil { + return fmt.Errorf("failed to update runtime pod metrics: %w", err) + } + return nil +} diff --git a/backend/internal/repository/runtime_rollout_repository.go b/backend/internal/repository/runtime_rollout_repository.go new file mode 100644 index 0000000..43c5e0d --- /dev/null +++ b/backend/internal/repository/runtime_rollout_repository.go @@ -0,0 +1,82 @@ +package repository + +import ( + "context" + "fmt" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +type RuntimeRolloutRepository interface { + Create(ctx context.Context, rollout *models.RuntimeRollout) error + GetByID(ctx context.Context, id int64) (*models.RuntimeRollout, error) + ListActive(ctx context.Context, runtimeType string) ([]models.RuntimeRollout, error) + UpdateStatus(ctx context.Context, id int64, status string, startedAt *time.Time, finishedAt *time.Time, message *string) error +} + +type runtimeRolloutRepository struct { + sess db.Session +} + +func NewRuntimeRolloutRepository(sess db.Session) RuntimeRolloutRepository { + return &runtimeRolloutRepository{sess: sess} +} + +func (r *runtimeRolloutRepository) Create(ctx context.Context, rollout *models.RuntimeRollout) error { + if err := ctx.Err(); err != nil { + return err + } + ensureTimestamps(&rollout.CreatedAt, &rollout.UpdatedAt) + res, err := r.sess.Collection("runtime_rollouts").Insert(rollout) + if err != nil { + return fmt.Errorf("failed to create runtime rollout: %w", err) + } + if id, ok := res.ID().(int64); ok { + rollout.ID = id + } + return nil +} + +func (r *runtimeRolloutRepository) GetByID(ctx context.Context, id int64) (*models.RuntimeRollout, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + var rollout models.RuntimeRollout + if err := r.sess.Collection("runtime_rollouts").Find(db.Cond{"id": id}).One(&rollout); err != nil { + if err == db.ErrNoMoreRows { + return nil, nil + } + return nil, fmt.Errorf("failed to get runtime rollout: %w", err) + } + return &rollout, nil +} + +func (r *runtimeRolloutRepository) ListActive(ctx context.Context, runtimeType string) ([]models.RuntimeRollout, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + cond := db.Cond{"status IN": []string{"pending", "running"}} + if runtimeType != "" { + cond["runtime_type"] = runtimeType + } + var rollouts []models.RuntimeRollout + if err := r.sess.Collection("runtime_rollouts").Find(cond).OrderBy("created_at", "id").All(&rollouts); err != nil { + return nil, fmt.Errorf("failed to list active runtime rollouts: %w", err) + } + return rollouts, nil +} + +func (r *runtimeRolloutRepository) UpdateStatus(ctx context.Context, id int64, status string, startedAt *time.Time, finishedAt *time.Time, message *string) error { + _, err := r.sess.SQL().ExecContext(ctx, ` + UPDATE runtime_rollouts + SET status = ?, started_at = ?, finished_at = ?, error_message = ?, updated_at = ? + WHERE id = ? + `, status, startedAt, finishedAt, message, time.Now().UTC(), id) + if err != nil { + return fmt.Errorf("failed to update runtime rollout status: %w", err) + } + return nil +} diff --git a/backend/internal/repository/system_image_setting_repository.go b/backend/internal/repository/system_image_setting_repository.go index 32f6d97..f9f9e08 100644 --- a/backend/internal/repository/system_image_setting_repository.go +++ b/backend/internal/repository/system_image_setting_repository.go @@ -36,7 +36,7 @@ func (r *systemImageSettingRepository) ensureTable() { CREATE TABLE IF NOT EXISTS system_image_settings ( id INT AUTO_INCREMENT PRIMARY KEY, instance_type VARCHAR(50) NOT NULL, - runtime_type ENUM('desktop', 'shell') NOT NULL DEFAULT 'desktop', + runtime_type ENUM('desktop', 'shell', 'gateway') NOT NULL DEFAULT 'desktop', display_name VARCHAR(255) NOT NULL, image VARCHAR(500) NOT NULL, is_enabled BOOLEAN NOT NULL DEFAULT TRUE, @@ -52,6 +52,7 @@ CREATE TABLE IF NOT EXISTS system_image_settings ( r.ensureIsEnabledColumn() r.ensureRuntimeTypeColumn() + r.ensureRuntimeTypeAllowsGateway() r.ensureInstanceTypeIsNotUnique() r.ensureInstanceTypeIndex() } @@ -96,12 +97,35 @@ WHERE table_schema = DATABASE() } if count == 0 { - if _, err := r.sess.SQL().Exec("ALTER TABLE system_image_settings ADD COLUMN runtime_type ENUM('desktop', 'shell') NOT NULL DEFAULT 'desktop' AFTER instance_type"); err != nil { + if _, err := r.sess.SQL().Exec("ALTER TABLE system_image_settings ADD COLUMN runtime_type ENUM('desktop', 'shell', 'gateway') NOT NULL DEFAULT 'desktop' AFTER instance_type"); err != nil { panic(fmt.Errorf("failed to ensure system_image_settings.runtime_type column: %w", err)) } } } +func (r *systemImageSettingRepository) ensureRuntimeTypeAllowsGateway() { + var columnType string + row, err := r.sess.SQL().QueryRow(` +SELECT COLUMN_TYPE +FROM information_schema.columns +WHERE table_schema = DATABASE() + AND table_name = 'system_image_settings' + AND column_name = 'runtime_type' +`) + if err != nil { + panic(fmt.Errorf("failed to inspect system_image_settings runtime_type enum: %w", err)) + } + if err := row.Scan(&columnType); err != nil { + panic(fmt.Errorf("failed to scan system_image_settings runtime_type enum: %w", err)) + } + + if !strings.Contains(columnType, "'gateway'") { + if _, err := r.sess.SQL().Exec("ALTER TABLE system_image_settings MODIFY COLUMN runtime_type ENUM('desktop', 'shell', 'gateway') NOT NULL DEFAULT 'desktop'"); err != nil { + panic(fmt.Errorf("failed to allow system_image_settings.runtime_type gateway: %w", err)) + } + } +} + func (r *systemImageSettingRepository) ensureInstanceTypeIsNotUnique() { rows, err := r.sess.SQL().Query(` SELECT DISTINCT INDEX_NAME diff --git a/backend/internal/repository/workspace_file_audit_repository.go b/backend/internal/repository/workspace_file_audit_repository.go new file mode 100644 index 0000000..5cac1c1 --- /dev/null +++ b/backend/internal/repository/workspace_file_audit_repository.go @@ -0,0 +1,40 @@ +package repository + +import ( + "context" + "fmt" + "time" + + "clawreef/internal/models" + + "github.com/upper/db/v4" +) + +type WorkspaceFileAuditRepository interface { + Create(ctx context.Context, audit *models.WorkspaceFileAudit) error +} + +type workspaceFileAuditRepository struct { + sess db.Session +} + +func NewWorkspaceFileAuditRepository(sess db.Session) WorkspaceFileAuditRepository { + return &workspaceFileAuditRepository{sess: sess} +} + +func (r *workspaceFileAuditRepository) Create(ctx context.Context, audit *models.WorkspaceFileAudit) error { + if err := ctx.Err(); err != nil { + return err + } + if audit.CreatedAt.IsZero() { + audit.CreatedAt = time.Now().UTC() + } + res, err := r.sess.Collection("workspace_file_audits").Insert(audit) + if err != nil { + return fmt.Errorf("failed to create workspace file audit: %w", err) + } + if id, ok := res.ID().(int64); ok { + audit.ID = id + } + return nil +} diff --git a/backend/internal/services/instance_env.go b/backend/internal/services/instance_env.go index 98f5dfd..cdf3954 100644 --- a/backend/internal/services/instance_env.go +++ b/backend/internal/services/instance_env.go @@ -129,3 +129,21 @@ func buildInstancePodEnv(instance *models.Instance, runtimeEnv, gatewayEnv, agen return resolved, nil } + +func buildInstanceGatewayEnv(instance *models.Instance, gatewayEnv map[string]string) (map[string]string, error) { + if instance == nil { + return nil, fmt.Errorf("instance is required") + } + + overrides, err := parseEnvironmentOverridesJSON(instance.EnvironmentOverridesJSON) + if err != nil { + return nil, err + } + + resolved := mergeEnvMaps(gatewayEnv, nil) + resolved = withInstanceProxyEnv(instance.Type, instance.ID, resolved) + resolved["CLAWMANAGER_RUNTIME_TYPE"] = normalizeInstanceRuntimeType(instance.RuntimeType) + resolved = mergeEnvMaps(resolved, overrides) + + return resolved, nil +} diff --git a/backend/internal/services/instance_external_access_service.go b/backend/internal/services/instance_external_access_service.go new file mode 100644 index 0000000..e440118 --- /dev/null +++ b/backend/internal/services/instance_external_access_service.go @@ -0,0 +1,286 @@ +package services + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +const ( + ExternalAccessModeShareLink = "share_link" + ExternalAccessModePassword = "password" + + ExternalAccessExpirationPreset = "preset" + ExternalAccessExpirationCustom = "custom" + ExternalAccessExpirationPermanent = "permanent" + + ExternalAccessPreset1Hour = "1h" + ExternalAccessPreset24Hours = "24h" + ExternalAccessPreset7Days = "7d" + ExternalAccessPreset30Days = "30d" +) + +type ExternalAccessExpirationRequest struct { + Mode string `json:"expires_mode,omitempty"` + Preset string `json:"expires_preset,omitempty"` + ExpiresAt *time.Time `json:"expires_at,omitempty"` +} + +type EnableShareLinkResult struct { + Access *models.InstanceExternalAccess `json:"access"` + ShareURL string `json:"share_url,omitempty"` +} + +type PasswordExternalAccessResult struct { + Access *models.InstanceExternalAccess `json:"access"` + Password string `json:"password"` + ShareURL string `json:"share_url,omitempty"` +} + +type InstanceExternalAccessService interface { + Get(ctx context.Context, instanceID int) (*models.InstanceExternalAccess, error) + EnableShareLink(ctx context.Context, instanceID, createdBy int, expiration ExternalAccessExpirationRequest) (*EnableShareLinkResult, error) + CreatePassword(ctx context.Context, instanceID, createdBy int, expiration ExternalAccessExpirationRequest) (*PasswordExternalAccessResult, error) + Disable(ctx context.Context, instanceID int) error + ResolveShortLink(ctx context.Context, code string) (*models.InstanceExternalAccess, error) + ValidateShortLink(ctx context.Context, code, password string) (*models.InstanceExternalAccess, error) +} + +type instanceExternalAccessService struct { + repo repository.InstanceExternalAccessRepository +} + +func NewInstanceExternalAccessService(repo repository.InstanceExternalAccessRepository) InstanceExternalAccessService { + return &instanceExternalAccessService{repo: repo} +} + +func (s *instanceExternalAccessService) Get(ctx context.Context, instanceID int) (*models.InstanceExternalAccess, error) { + if s == nil || s.repo == nil { + return nil, fmt.Errorf("instance external access repository is not configured") + } + return s.repo.GetByInstanceID(ctx, instanceID) +} + +func (s *instanceExternalAccessService) EnableShareLink(ctx context.Context, instanceID, createdBy int, expiration ExternalAccessExpirationRequest) (*EnableShareLinkResult, error) { + if s == nil || s.repo == nil { + return nil, fmt.Errorf("instance external access repository is not configured") + } + code, codeHash, err := generateShortCode() + if err != nil { + return nil, err + } + expiresAt, err := resolveExternalAccessExpiresAt(expiration) + if err != nil { + return nil, err + } + access := &models.InstanceExternalAccess{ + InstanceID: instanceID, + Enabled: true, + AuthMode: ExternalAccessModeShareLink, + ShortCodeHash: &codeHash, + PublicSlug: &code, + PublicTokenHash: nil, + ExpiresAt: expiresAt, + CreatedBy: &createdBy, + PasswordHash: nil, + PasswordValue: nil, + PasswordHint: nil, + LastUsedAt: nil, + } + if err := s.repo.Upsert(ctx, access); err != nil { + return nil, err + } + saved, err := s.repo.GetByInstanceID(ctx, instanceID) + if err != nil { + return nil, err + } + return &EnableShareLinkResult{ + Access: saved, + ShareURL: shortExternalAccessURL(code), + }, nil +} + +func (s *instanceExternalAccessService) CreatePassword(ctx context.Context, instanceID, createdBy int, expiration ExternalAccessExpirationRequest) (*PasswordExternalAccessResult, error) { + if s == nil || s.repo == nil { + return nil, fmt.Errorf("instance external access repository is not configured") + } + code, codeHash, err := generateShortCode() + if err != nil { + return nil, err + } + password, err := randomToken("pwd", 16) + if err != nil { + return nil, err + } + expiresAt, err := resolveExternalAccessExpiresAt(expiration) + if err != nil { + return nil, err + } + passwordHash := hashExternalSecret(password) + hint := password + if len(hint) > 12 { + hint = hint[:12] + } + access := &models.InstanceExternalAccess{ + InstanceID: instanceID, + Enabled: true, + AuthMode: ExternalAccessModePassword, + PublicSlug: &code, + ShortCodeHash: &codeHash, + PasswordHash: &passwordHash, + PasswordValue: &password, + PasswordHint: &hint, + ExpiresAt: expiresAt, + CreatedBy: &createdBy, + PublicTokenHash: nil, + LastUsedAt: nil, + } + if err := s.repo.Upsert(ctx, access); err != nil { + return nil, err + } + saved, err := s.repo.GetByInstanceID(ctx, instanceID) + if err != nil { + return nil, err + } + return &PasswordExternalAccessResult{Access: saved, Password: password, ShareURL: shortExternalAccessURL(code)}, nil +} + +func (s *instanceExternalAccessService) Disable(ctx context.Context, instanceID int) error { + if s == nil || s.repo == nil { + return fmt.Errorf("instance external access repository is not configured") + } + return s.repo.Disable(ctx, instanceID) +} + +func (s *instanceExternalAccessService) ResolveShortLink(ctx context.Context, code string) (*models.InstanceExternalAccess, error) { + if s == nil || s.repo == nil { + return nil, fmt.Errorf("instance external access repository is not configured") + } + code = strings.TrimSpace(code) + if code == "" { + return nil, fmt.Errorf("short link code is required") + } + access, err := s.repo.GetByShortCodeHash(ctx, hashExternalSecret(code)) + if err != nil { + return nil, err + } + if access == nil || !access.Enabled { + return nil, fmt.Errorf("external access is not enabled") + } + if access.ExpiresAt != nil && time.Now().UTC().After(*access.ExpiresAt) { + return nil, fmt.Errorf("external access has expired") + } + return access, nil +} + +func (s *instanceExternalAccessService) ValidateShortLink(ctx context.Context, code, password string) (*models.InstanceExternalAccess, error) { + access, err := s.ResolveShortLink(ctx, code) + if err != nil { + return nil, err + } + switch access.AuthMode { + case ExternalAccessModeShareLink: + // The short code itself is the share-link capability. A matching hash is enough. + case ExternalAccessModePassword: + if access.PasswordHash == nil || *access.PasswordHash != hashExternalSecret(strings.TrimSpace(password)) { + return nil, fmt.Errorf("invalid external access password") + } + default: + return nil, fmt.Errorf("unsupported external access mode") + } + if err := s.repo.MarkUsed(ctx, access.ID); err != nil { + return nil, err + } + return access, nil +} + +func resolveExternalAccessExpiresAt(expiration ExternalAccessExpirationRequest) (*time.Time, error) { + mode := strings.TrimSpace(expiration.Mode) + if mode == "" { + mode = ExternalAccessExpirationPreset + } + switch mode { + case ExternalAccessExpirationPermanent: + return nil, nil + case ExternalAccessExpirationCustom: + if expiration.ExpiresAt == nil { + return nil, fmt.Errorf("custom external access expiration requires expires_at") + } + value := expiration.ExpiresAt.UTC() + return &value, nil + case ExternalAccessExpirationPreset: + preset := strings.TrimSpace(expiration.Preset) + if preset == "" { + preset = ExternalAccessPreset24Hours + } + duration, err := externalAccessPresetDuration(preset) + if err != nil { + return nil, err + } + value := time.Now().UTC().Add(duration) + return &value, nil + default: + return nil, fmt.Errorf("unsupported external access expiration mode") + } +} + +func externalAccessPresetDuration(preset string) (time.Duration, error) { + switch strings.TrimSpace(preset) { + case ExternalAccessPreset1Hour: + return time.Hour, nil + case ExternalAccessPreset24Hours: + return 24 * time.Hour, nil + case ExternalAccessPreset7Days: + return 7 * 24 * time.Hour, nil + case ExternalAccessPreset30Days: + return 30 * 24 * time.Hour, nil + default: + return 0, fmt.Errorf("unsupported external access expiration preset") + } +} + +func generateShortCode() (code, codeHash string, err error) { + code, err = randomToken("sl", 12) + if err != nil { + return "", "", err + } + return code, hashExternalSecret(code), nil +} + +func shortExternalAccessURL(code string) string { + return fmt.Sprintf("/s/%s/", strings.Trim(strings.TrimSpace(code), "/")) +} + +func ExternalAccessShareURL(access *models.InstanceExternalAccess) string { + if access == nil || !access.Enabled || access.PublicSlug == nil { + return "" + } + return shortExternalAccessURL(*access.PublicSlug) +} + +func ExternalAccessPassword(access *models.InstanceExternalAccess) string { + if access == nil || !access.Enabled || access.AuthMode != ExternalAccessModePassword || access.PasswordValue == nil { + return "" + } + return *access.PasswordValue +} + +func randomToken(prefix string, bytesLen int) (string, error) { + buf := make([]byte, bytesLen) + if _, err := rand.Read(buf); err != nil { + return "", fmt.Errorf("failed to generate token: %w", err) + } + return prefix + "_" + hex.EncodeToString(buf), nil +} + +func hashExternalSecret(value string) string { + sum := sha256.Sum256([]byte(strings.TrimSpace(value))) + return hex.EncodeToString(sum[:]) +} diff --git a/backend/internal/services/instance_external_access_service_test.go b/backend/internal/services/instance_external_access_service_test.go new file mode 100644 index 0000000..5d9bb23 --- /dev/null +++ b/backend/internal/services/instance_external_access_service_test.go @@ -0,0 +1,304 @@ +package services + +import ( + "context" + "strings" + "testing" + "time" + + "clawreef/internal/models" +) + +func TestInstanceExternalAccessShareLinkValidation(t *testing.T) { + repo := newFakeExternalAccessRepo() + service := NewInstanceExternalAccessService(repo) + + result, err := service.EnableShareLink(context.Background(), 42, 7, ExternalAccessExpirationRequest{Mode: ExternalAccessExpirationPermanent}) + if err != nil { + t.Fatalf("EnableShareLink failed: %v", err) + } + if !strings.HasPrefix(result.ShareURL, "/s/") || strings.Contains(result.ShareURL, "token=") || strings.Contains(result.ShareURL, "/api/v1/") || strings.Contains(result.ShareURL, "/proxy") { + t.Fatalf("unexpected short share URL: %q", result.ShareURL) + } + code := strings.Trim(strings.TrimPrefix(result.ShareURL, "/s/"), "/") + if code == "" { + t.Fatalf("short code missing from URL %q", result.ShareURL) + } + if result.Access.ShortCodeHash == nil || *result.Access.ShortCodeHash == code { + t.Fatalf("expected stored short code to be hashed") + } + if result.Access.AuthMode != ExternalAccessModeShareLink { + t.Fatalf("auth mode = %q, want %q", result.Access.AuthMode, ExternalAccessModeShareLink) + } + + access, err := service.ValidateShortLink(context.Background(), code, "") + if err != nil { + t.Fatalf("ValidateShortLink failed: %v", err) + } + if access.InstanceID != 42 { + t.Fatalf("validated instance ID = %d, want 42", access.InstanceID) + } + if repo.markUsedCount != 1 { + t.Fatalf("mark used count = %d, want 1", repo.markUsedCount) + } + + if _, err := service.ValidateShortLink(context.Background(), "wrong-code", ""); err == nil { + t.Fatalf("expected wrong short code to fail") + } +} + +func TestInstanceExternalAccessShareLinkPresetExpiration(t *testing.T) { + repo := newFakeExternalAccessRepo() + service := NewInstanceExternalAccessService(repo) + before := time.Now().UTC().Add(24 * time.Hour) + + result, err := service.EnableShareLink(context.Background(), 42, 7, ExternalAccessExpirationRequest{ + Mode: ExternalAccessExpirationPreset, + Preset: ExternalAccessPreset24Hours, + }) + if err != nil { + t.Fatalf("EnableShareLink failed: %v", err) + } + if result.Access.ExpiresAt == nil { + t.Fatal("preset expiration must set expires_at") + } + after := time.Now().UTC().Add(24 * time.Hour) + if result.Access.ExpiresAt.Before(before) || result.Access.ExpiresAt.After(after.Add(time.Second)) { + t.Fatalf("preset expires_at = %v, want around 24h from now", result.Access.ExpiresAt) + } +} + +func TestInstanceExternalAccessShareLinkCustomAndPermanentExpiration(t *testing.T) { + repo := newFakeExternalAccessRepo() + service := NewInstanceExternalAccessService(repo) + customAt := time.Now().UTC().Add(3 * time.Hour).Truncate(time.Second) + + custom, err := service.EnableShareLink(context.Background(), 42, 7, ExternalAccessExpirationRequest{ + Mode: ExternalAccessExpirationCustom, + ExpiresAt: &customAt, + }) + if err != nil { + t.Fatalf("custom EnableShareLink failed: %v", err) + } + if custom.Access.ExpiresAt == nil || !custom.Access.ExpiresAt.Equal(customAt) { + t.Fatalf("custom expires_at = %v, want %v", custom.Access.ExpiresAt, customAt) + } + + permanent, err := service.EnableShareLink(context.Background(), 42, 7, ExternalAccessExpirationRequest{Mode: ExternalAccessExpirationPermanent}) + if err != nil { + t.Fatalf("permanent EnableShareLink failed: %v", err) + } + if permanent.Access.ExpiresAt != nil { + t.Fatalf("permanent expires_at = %v, want nil", permanent.Access.ExpiresAt) + } +} + +func TestInstanceExternalAccessPasswordDisable(t *testing.T) { + repo := newFakeExternalAccessRepo() + service := NewInstanceExternalAccessService(repo) + + result, err := service.CreatePassword(context.Background(), 100, 9, ExternalAccessExpirationRequest{Mode: ExternalAccessExpirationPermanent}) + if err != nil { + t.Fatalf("CreatePassword failed: %v", err) + } + if !strings.HasPrefix(result.Password, "pwd_") { + t.Fatalf("unexpected password prefix: %q", result.Password) + } + if result.Access.PasswordHash == nil || *result.Access.PasswordHash == result.Password { + t.Fatalf("expected stored password to be hashed") + } + if result.Access.PasswordHint == nil || !strings.HasPrefix(result.Password, *result.Access.PasswordHint) { + t.Fatalf("stored password hint does not match generated password") + } + if result.Access.AuthMode != ExternalAccessModePassword { + t.Fatalf("auth mode = %q, want %q", result.Access.AuthMode, ExternalAccessModePassword) + } + if !strings.HasPrefix(result.ShareURL, "/s/") || strings.Contains(result.ShareURL, "token=") || strings.Contains(result.ShareURL, "/api/v1/") { + t.Fatalf("unexpected password short URL: %q", result.ShareURL) + } + code := strings.Trim(strings.TrimPrefix(result.ShareURL, "/s/"), "/") + + if _, err := service.ValidateShortLink(context.Background(), code, result.Password); err != nil { + t.Fatalf("ValidateShortLink with password failed: %v", err) + } + + if err := service.Disable(context.Background(), 100); err != nil { + t.Fatalf("Disable failed: %v", err) + } + if _, err := service.ValidateShortLink(context.Background(), code, result.Password); err == nil { + t.Fatalf("expected disabled password access to fail") + } +} + +func TestInstanceExternalAccessRestoresShareURLAfterRefresh(t *testing.T) { + repo := newFakeExternalAccessRepo() + service := NewInstanceExternalAccessService(repo) + + result, err := service.CreatePassword(context.Background(), 100, 9, ExternalAccessExpirationRequest{Mode: ExternalAccessExpirationPermanent}) + if err != nil { + t.Fatalf("CreatePassword failed: %v", err) + } + reloaded, err := service.Get(context.Background(), 100) + if err != nil { + t.Fatalf("Get failed: %v", err) + } + if got := ExternalAccessShareURL(reloaded); got != result.ShareURL { + t.Fatalf("ExternalAccessShareURL after reload = %q, want %q", got, result.ShareURL) + } +} + +func TestInstanceExternalAccessRestoresPasswordAfterRefresh(t *testing.T) { + repo := newFakeExternalAccessRepo() + service := NewInstanceExternalAccessService(repo) + + result, err := service.CreatePassword(context.Background(), 100, 9, ExternalAccessExpirationRequest{Mode: ExternalAccessExpirationPermanent}) + if err != nil { + t.Fatalf("CreatePassword failed: %v", err) + } + reloaded, err := service.Get(context.Background(), 100) + if err != nil { + t.Fatalf("Get failed: %v", err) + } + if got := ExternalAccessPassword(reloaded); got != result.Password { + t.Fatalf("ExternalAccessPassword after reload = %q, want generated password", got) + } +} + +func TestInstanceExternalAccessExpiration(t *testing.T) { + repo := newFakeExternalAccessRepo() + service := NewInstanceExternalAccessService(repo) + expiredAt := time.Now().UTC().Add(-time.Minute) + + result, err := service.EnableShareLink(context.Background(), 77, 3, ExternalAccessExpirationRequest{ + Mode: ExternalAccessExpirationCustom, + ExpiresAt: &expiredAt, + }) + if err != nil { + t.Fatalf("EnableShareLink failed: %v", err) + } + code := strings.Trim(strings.TrimPrefix(result.ShareURL, "/s/"), "/") + + if _, err := service.ValidateShortLink(context.Background(), code, ""); err == nil { + t.Fatalf("expected expired public access to fail") + } +} + +type fakeExternalAccessRepo struct { + byInstance map[int]*models.InstanceExternalAccess + byCodeHash map[string]*models.InstanceExternalAccess + nextID int64 + markUsedCount int +} + +func newFakeExternalAccessRepo() *fakeExternalAccessRepo { + return &fakeExternalAccessRepo{ + byInstance: make(map[int]*models.InstanceExternalAccess), + byCodeHash: make(map[string]*models.InstanceExternalAccess), + nextID: 1, + } +} + +func (r *fakeExternalAccessRepo) GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceExternalAccess, error) { + _ = ctx + return cloneExternalAccess(r.byInstance[instanceID]), nil +} + +func (r *fakeExternalAccessRepo) GetByShortCodeHash(ctx context.Context, codeHash string) (*models.InstanceExternalAccess, error) { + _ = ctx + return cloneExternalAccess(r.byCodeHash[codeHash]), nil +} + +func (r *fakeExternalAccessRepo) Upsert(ctx context.Context, access *models.InstanceExternalAccess) error { + _ = ctx + saved := cloneExternalAccess(access) + if saved.ID == 0 { + if existing := r.byInstance[saved.InstanceID]; existing != nil { + saved.ID = existing.ID + } else { + saved.ID = r.nextID + r.nextID++ + } + } + if existing := r.byInstance[saved.InstanceID]; existing != nil && existing.ShortCodeHash != nil { + delete(r.byCodeHash, *existing.ShortCodeHash) + } + r.byInstance[saved.InstanceID] = cloneExternalAccess(saved) + if saved.ShortCodeHash != nil { + r.byCodeHash[*saved.ShortCodeHash] = cloneExternalAccess(saved) + } + return nil +} + +func (r *fakeExternalAccessRepo) Disable(ctx context.Context, instanceID int) error { + _ = ctx + if existing := r.byInstance[instanceID]; existing != nil { + existing.Enabled = false + if existing.ShortCodeHash != nil { + if byCode := r.byCodeHash[*existing.ShortCodeHash]; byCode != nil { + byCode.Enabled = false + } + } + } + return nil +} + +func (r *fakeExternalAccessRepo) MarkUsed(ctx context.Context, id int64) error { + _ = ctx + r.markUsedCount++ + now := time.Now().UTC() + for _, access := range r.byInstance { + if access.ID == id { + access.LastUsedAt = &now + } + } + for _, access := range r.byCodeHash { + if access.ID == id { + access.LastUsedAt = &now + } + } + return nil +} + +func cloneExternalAccess(access *models.InstanceExternalAccess) *models.InstanceExternalAccess { + if access == nil { + return nil + } + clone := *access + if access.PublicSlug != nil { + value := *access.PublicSlug + clone.PublicSlug = &value + } + if access.PublicTokenHash != nil { + value := *access.PublicTokenHash + clone.PublicTokenHash = &value + } + if access.ShortCodeHash != nil { + value := *access.ShortCodeHash + clone.ShortCodeHash = &value + } + if access.PasswordHash != nil { + value := *access.PasswordHash + clone.PasswordHash = &value + } + if access.PasswordValue != nil { + value := *access.PasswordValue + clone.PasswordValue = &value + } + if access.PasswordHint != nil { + value := *access.PasswordHint + clone.PasswordHint = &value + } + if access.ExpiresAt != nil { + value := *access.ExpiresAt + clone.ExpiresAt = &value + } + if access.CreatedBy != nil { + value := *access.CreatedBy + clone.CreatedBy = &value + } + if access.LastUsedAt != nil { + value := *access.LastUsedAt + clone.LastUsedAt = &value + } + return &clone +} diff --git a/backend/internal/services/instance_proxy_service.go b/backend/internal/services/instance_proxy_service.go index 8e3e05d..9143255 100644 --- a/backend/internal/services/instance_proxy_service.go +++ b/backend/internal/services/instance_proxy_service.go @@ -4,16 +4,20 @@ import ( "bytes" "context" "crypto/tls" + "errors" "fmt" "io" + "net" "net/http" "net/url" + "os" "strconv" "strings" "sync" "time" "clawreef/internal/models" + "clawreef/internal/repository" "clawreef/internal/services/k8s" "github.com/gorilla/websocket" @@ -21,14 +25,19 @@ import ( // InstanceProxyService handles proxying requests to instance pods type InstanceProxyService struct { - serviceService *k8s.ServiceService - accessService *InstanceAccessService - httpClient *http.Client - serviceCache map[serviceCacheKey]serviceCacheEntry - serviceLookups map[serviceCacheKey]*serviceLookupCall - cacheMu sync.RWMutex - lookupMu sync.Mutex - serviceTTL time.Duration + serviceService *k8s.ServiceService + accessService *InstanceAccessService + instanceRepo repository.InstanceRepository + runtimePodRepo repository.RuntimePodRepository + bindingRepo repository.InstanceRuntimeBindingRepository + httpClient *http.Client + openClawGatewayToken string + openClawProxyOrigin string + serviceCache map[serviceCacheKey]serviceCacheEntry + serviceLookups map[serviceCacheKey]*serviceLookupCall + cacheMu sync.RWMutex + lookupMu sync.Mutex + serviceTTL time.Duration } type serviceCacheKey struct { @@ -50,8 +59,20 @@ type serviceLookupCall struct { const defaultServiceCacheTTL = 30 * time.Second +var ErrInstanceGatewayUnavailable = errors.New("instance gateway is not available") + +type InstanceProxyServiceOption func(*InstanceProxyService) + +func WithInstanceProxyRuntimeRepositories(instanceRepo repository.InstanceRepository, runtimePodRepo repository.RuntimePodRepository, bindingRepo repository.InstanceRuntimeBindingRepository) InstanceProxyServiceOption { + return func(s *InstanceProxyService) { + s.instanceRepo = instanceRepo + s.runtimePodRepo = runtimePodRepo + s.bindingRepo = bindingRepo + } +} + // NewInstanceProxyService creates a new instance proxy service -func NewInstanceProxyService(accessService *InstanceAccessService) *InstanceProxyService { +func NewInstanceProxyService(accessService *InstanceAccessService, options ...InstanceProxyServiceOption) *InstanceProxyService { transport := &http.Transport{ Proxy: http.ProxyFromEnvironment, TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, @@ -63,7 +84,7 @@ func NewInstanceProxyService(accessService *InstanceAccessService) *InstanceProx ForceAttemptHTTP2: true, } - return &InstanceProxyService{ + service := &InstanceProxyService{ serviceService: k8s.NewServiceService(), accessService: accessService, httpClient: &http.Client{ @@ -73,10 +94,18 @@ func NewInstanceProxyService(accessService *InstanceAccessService) *InstanceProx return http.ErrUseLastResponse }, }, - serviceCache: make(map[serviceCacheKey]serviceCacheEntry), - serviceLookups: make(map[serviceCacheKey]*serviceLookupCall), - serviceTTL: defaultServiceCacheTTL, + openClawGatewayToken: strings.TrimSpace(os.Getenv("OPENCLAW_GATEWAY_TOKEN")), + openClawProxyOrigin: resolveOpenClawProxyOriginFromEnv(), + serviceCache: make(map[serviceCacheKey]serviceCacheEntry), + serviceLookups: make(map[serviceCacheKey]*serviceLookupCall), + serviceTTL: defaultServiceCacheTTL, } + for _, option := range options { + if option != nil { + option(service) + } + } + return service } // ProxyRequest proxies a request to an instance @@ -102,27 +131,22 @@ func (s *InstanceProxyService) ProxyRequest(ctx context.Context, instanceID int, return fmt.Errorf("token does not match instance") } - // Extract the actual path from the request (remove the proxy prefix) - targetPath := s.extractTargetPath(r.URL.Path, instanceID, accessToken.InstanceType) - targetPort := s.resolveTargetPort(accessToken.InstanceType, accessToken.TargetPort, targetPath) - shouldRewriteHTML := s.shouldRewriteHTML(accessToken.InstanceType) + effectiveRequestPath := canonicalProxyEntryRequestPath(r.URL.Path, accessToken, instanceID) - // Get service info for the instance (create if not exists) - serviceInfo, err := s.getOrCreateService(ctx, accessToken.UserID, instanceID, targetPort) - if err != nil { - return fmt.Errorf("failed to get or create service: %w", err) - } + // Extract the actual path from the request (remove the proxy prefix) + targetPath := s.extractTargetPath(effectiveRequestPath, instanceID, accessToken.InstanceType) + targetPort := s.resolveTargetPort(accessToken.InstanceType, accessToken.TargetPort, targetPath) + shouldRewriteHTML := s.shouldRewriteHTMLForProxy(instanceID, accessToken.InstanceType) // Build target URL - targetURL := &url.URL{ - Scheme: s.resolveTargetScheme(accessToken.InstanceType, false), - Host: s.resolveProxyHost(ctx, accessToken.UserID, instanceID, serviceInfo), - Path: targetPath, + targetURL, err := s.resolveHTTPProxyTarget(ctx, accessToken, instanceID, targetPort, targetPath, effectiveRequestPath) + if err != nil { + return err } // Copy query parameters (excluding token) queryParams := r.URL.Query() - queryParams.Del("token") + removeProxyAccessTokenQuery(queryParams, token) if len(queryParams) > 0 { targetURL.RawQuery = queryParams.Encode() } @@ -148,6 +172,9 @@ func (s *InstanceProxyService) ProxyRequest(ctx context.Context, instanceID int, proxyReq.Header.Set("X-Forwarded-Host", r.Host) proxyReq.Header.Set("X-Forwarded-Proto", requestScheme(r)) proxyReq.Header.Set("X-Forwarded-Prefix", fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID)) + if token := s.managedRuntimeGatewayBearerToken(ctx, instanceID, accessToken.InstanceType); token != "" { + proxyReq.Header.Set("Authorization", "Bearer "+token) + } if shouldRewriteHTML { proxyReq.Header.Del("Accept-Encoding") } @@ -179,7 +206,7 @@ func (s *InstanceProxyService) ProxyRequest(ctx context.Context, instanceID int, return fmt.Errorf("failed to close upstream html body: %w", closeErr) } - modifiedBody := injectProxyBase(string(body), fmt.Sprintf("/api/v1/instances/%d/proxy/", instanceID)) + modifiedBody := injectProxyBase(string(body), proxyBaseForRequestPath(effectiveRequestPath, instanceID)) resp.Body = io.NopCloser(bytes.NewReader([]byte(modifiedBody))) resp.ContentLength = int64(len(modifiedBody)) resp.Header.Set("Content-Length", strconv.Itoa(len(modifiedBody))) @@ -238,22 +265,14 @@ func (s *InstanceProxyService) ProxyWebSocket(ctx context.Context, instanceID in targetPath := s.extractTargetPath(r.URL.Path, instanceID, accessToken.InstanceType) targetPort := s.resolveTargetPort(accessToken.InstanceType, accessToken.TargetPort, targetPath) - // Get service info for the instance - serviceInfo, err := s.getOrCreateService(ctx, accessToken.UserID, instanceID, targetPort) + targetURL, err := s.resolveWebSocketProxyTarget(ctx, accessToken, instanceID, targetPort, targetPath, r.URL.Path) if err != nil { - return fmt.Errorf("failed to get or create service: %w", err) - } - - // WebSocket upstream uses ws/wss explicitly. - targetURL := &url.URL{ - Scheme: s.resolveTargetScheme(accessToken.InstanceType, true), - Host: s.resolveProxyHost(ctx, accessToken.UserID, instanceID, serviceInfo), - Path: targetPath, + return err } // Copy query parameters (excluding token) queryParams := r.URL.Query() - queryParams.Del("token") + removeProxyAccessTokenQuery(queryParams, token) if len(queryParams) > 0 { targetURL.RawQuery = queryParams.Encode() } @@ -270,8 +289,14 @@ func (s *InstanceProxyService) ProxyWebSocket(ctx context.Context, instanceID in upstreamHeader.Del("Sec-Websocket-Key") upstreamHeader.Del("Sec-Websocket-Version") upstreamHeader.Del("Sec-Websocket-Extensions") + upstreamHeader.Set("X-Forwarded-For", r.RemoteAddr) + upstreamHeader.Set("X-Forwarded-Host", r.Host) upstreamHeader.Set("X-Forwarded-Proto", requestScheme(r)) upstreamHeader.Set("X-Forwarded-Prefix", fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID)) + if token := s.managedRuntimeGatewayBearerToken(ctx, instanceID, accessToken.InstanceType); token != "" { + upstreamHeader.Set("Authorization", "Bearer "+token) + upstreamHeader.Set("Origin", s.openClawWebSocketOrigin(targetURL)) + } dialer := websocket.Dialer{ Proxy: http.ProxyFromEnvironment, @@ -364,6 +389,112 @@ func (s *InstanceProxyService) removeHopByHopHeaders(header http.Header) { } } +func (s *InstanceProxyService) managedRuntimeGatewayBearerToken(ctx context.Context, instanceID int, instanceType string) string { + if s == nil || s.instanceRepo == nil { + return "" + } + normalizedType, managedType := NormalizeV2RuntimeType(instanceType) + if !managedType { + return "" + } + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil || instance == nil { + return "" + } + if runtimeType, ok := v2RuntimeTypeForInstance(instance); ok && runtimeType == normalizedType { + if instance.AccessToken != nil && strings.TrimSpace(*instance.AccessToken) != "" { + return strings.TrimSpace(*instance.AccessToken) + } + } + if normalizedType == RuntimeTypeOpenClaw && s.openClawGatewayToken != "" { + return s.openClawGatewayToken + } + return "" +} + +func (s *InstanceProxyService) resolveHTTPProxyTarget(ctx context.Context, accessToken *AccessToken, instanceID int, targetPort int32, targetPath, requestPath string) (*url.URL, error) { + if targetURL, ok, err := s.resolveV2ProxyTarget(ctx, accessToken, instanceID, targetPath, requestPath, false); ok || err != nil { + return targetURL, err + } + serviceInfo, err := s.getOrCreateService(ctx, accessToken.UserID, instanceID, targetPort) + if err != nil { + return nil, fmt.Errorf("failed to get or create service: %w", err) + } + return &url.URL{ + Scheme: s.resolveTargetScheme(accessToken.InstanceType, false), + Host: s.resolveProxyHost(ctx, accessToken.UserID, instanceID, serviceInfo), + Path: targetPath, + }, nil +} + +func (s *InstanceProxyService) resolveWebSocketProxyTarget(ctx context.Context, accessToken *AccessToken, instanceID int, targetPort int32, targetPath, requestPath string) (*url.URL, error) { + if targetURL, ok, err := s.resolveV2ProxyTarget(ctx, accessToken, instanceID, targetPath, requestPath, true); ok || err != nil { + return targetURL, err + } + serviceInfo, err := s.getOrCreateService(ctx, accessToken.UserID, instanceID, targetPort) + if err != nil { + return nil, fmt.Errorf("failed to get or create service: %w", err) + } + return &url.URL{ + Scheme: s.resolveTargetScheme(accessToken.InstanceType, true), + Host: s.resolveProxyHost(ctx, accessToken.UserID, instanceID, serviceInfo), + Path: targetPath, + }, nil +} + +func (s *InstanceProxyService) resolveV2ProxyTarget(ctx context.Context, accessToken *AccessToken, instanceID int, targetPath, requestPath string, websocket bool) (*url.URL, bool, error) { + if s.instanceRepo == nil || s.bindingRepo == nil || s.runtimePodRepo == nil { + return nil, false, nil + } + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil { + return nil, false, fmt.Errorf("failed to get instance for proxy: %w", err) + } + if instance == nil { + return nil, false, ErrInstanceGatewayUnavailable + } + if instance.UserID != accessToken.UserID { + return nil, false, fmt.Errorf("token does not match instance owner") + } + if _, ok := v2RuntimeTypeForInstance(instance); !ok { + return nil, false, nil + } + if !strings.EqualFold(strings.TrimSpace(instance.Status), "running") { + return nil, true, ErrInstanceGatewayUnavailable + } + + binding, err := s.bindingRepo.GetRunningByInstanceID(ctx, instanceID) + if err != nil { + return nil, true, fmt.Errorf("%w: %v", ErrInstanceGatewayUnavailable, err) + } + if binding == nil { + return nil, true, ErrInstanceGatewayUnavailable + } + if binding.Generation != instance.RuntimeGeneration { + return nil, true, ErrInstanceGatewayUnavailable + } + pod, err := s.runtimePodRepo.GetByID(ctx, binding.RuntimePodID) + if err != nil { + return nil, true, fmt.Errorf("%w: %v", ErrInstanceGatewayUnavailable, err) + } + if pod == nil || pod.PodIP == nil || strings.TrimSpace(*pod.PodIP) == "" || binding.GatewayPort <= 0 { + return nil, true, ErrInstanceGatewayUnavailable + } + scheme := "http" + if websocket { + scheme = "ws" + } + upstreamPath := stripInstanceProxyPrefix(targetPath, instanceID) + if shouldPreserveOpenClawControlUIPath(instance) { + upstreamPath = openClawControlUIRequestPath(requestPath, instanceID) + } + return &url.URL{ + Scheme: scheme, + Host: net.JoinHostPort(strings.TrimSpace(*pod.PodIP), strconv.Itoa(binding.GatewayPort)), + Path: upstreamPath, + }, true, nil +} + // getOrCreateService gets service info or creates the service if it doesn't exist func (s *InstanceProxyService) getOrCreateService(ctx context.Context, userID, instanceID int, targetPort int32) (*k8s.ServiceInfo, error) { cacheKey := serviceCacheKey{ @@ -445,13 +576,78 @@ func (s *InstanceProxyService) extractTargetPath(requestPath string, instanceID return requestPath } +func stripInstanceProxyPrefix(requestPath string, instanceID int) string { + prefix := fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID) + if strings.HasPrefix(requestPath, prefix) { + path := strings.TrimPrefix(requestPath, prefix) + if path == "" { + return "/" + } + return path + } + return requestPath +} + +func canonicalProxyEntryRequestPath(requestPath string, accessToken *AccessToken, instanceID int) string { + prefix := fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID) + path := strings.TrimSpace(requestPath) + if path != prefix && path != prefix+"/" { + return requestPath + } + if accessToken == nil || strings.TrimSpace(accessToken.AccessURL) == "" { + return requestPath + } + parsed, err := url.Parse(accessToken.AccessURL) + if err != nil { + return requestPath + } + entryPath := strings.TrimSpace(parsed.Path) + if entryPath == "" || entryPath == prefix || entryPath == prefix+"/" { + return requestPath + } + if strings.HasPrefix(entryPath, prefix+"/") { + return entryPath + } + return requestPath +} + +func shouldPreserveOpenClawControlUIPath(instance *models.Instance) bool { + if instance == nil || !strings.EqualFold(strings.TrimSpace(instance.Type), RuntimeTypeOpenClaw) { + return false + } + _, ok := v2RuntimeTypeForInstance(instance) + return ok +} + +func openClawControlUIRequestPath(requestPath string, instanceID int) string { + prefix := fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID) + path := strings.TrimSpace(requestPath) + if path == "" || path == prefix { + return prefix + "/" + } + if strings.HasPrefix(path, prefix) { + return path + } + if strings.HasPrefix(path, "/") { + return prefix + path + } + return prefix + "/" + path +} + // GetProxyURL generates a proxy URL for frontend func (s *InstanceProxyService) GetProxyURL(instanceID int, token string) string { - if token == "" { - return fmt.Sprintf("/api/v1/instances/%d/proxy/", instanceID) - } + return proxyURLWithPath(instanceID, "/", token) +} - return fmt.Sprintf("/api/v1/instances/%d/proxy/?token=%s", instanceID, token) +// GetProxyURLForInstance generates the best frontend entry URL for an instance. +func (s *InstanceProxyService) GetProxyURLForInstance(instance *models.Instance, token string) string { + if instance == nil { + return "" + } + if runtimeType, ok := v2RuntimeTypeForInstance(instance); ok && runtimeType == RuntimeTypeHermes { + return proxyURLWithPath(instance.ID, "/chat", token) + } + return proxyURLWithPath(instance.ID, "/", token) } // GetTargetPortForInstance returns the service target port used by the instance type. @@ -526,6 +722,18 @@ func (s *InstanceProxyService) shouldRewriteHTML(instanceType string) bool { return !usesWebtopImage(instanceType) } +func (s *InstanceProxyService) shouldRewriteHTMLForProxy(instanceID int, instanceType string) bool { + if s != nil && s.instanceRepo != nil && strings.EqualFold(strings.TrimSpace(instanceType), RuntimeTypeHermes) { + instance, err := s.instanceRepo.GetByID(instanceID) + if err == nil && instance != nil { + if runtimeType, ok := v2RuntimeTypeForInstance(instance); ok && runtimeType == RuntimeTypeHermes { + return true + } + } + } + return s.shouldRewriteHTML(instanceType) +} + func (s *InstanceProxyService) getCachedService(key serviceCacheKey) *k8s.ServiceInfo { s.cacheMu.RLock() entry, ok := s.serviceCache[key] @@ -593,6 +801,114 @@ func injectProxyBase(html, proxyBase string) string { return baseTag + html } +func proxyURLWithPath(instanceID int, targetPath, token string) string { + path := strings.TrimSpace(targetPath) + if path == "" || path == "/" { + path = "/" + } else { + path = "/" + strings.TrimLeft(path, "/") + if !strings.HasSuffix(path, "/") { + path += "/" + } + } + + raw := fmt.Sprintf("/api/v1/instances/%d/proxy%s", instanceID, path) + if token == "" { + return raw + } + return fmt.Sprintf("%s?token=%s", raw, url.QueryEscape(token)) +} + +func removeProxyAccessTokenQuery(query url.Values, accessToken string) { + values := query["token"] + if len(values) == 0 { + return + } + filtered := values[:0] + for _, value := range values { + if value != accessToken { + filtered = append(filtered, value) + } + } + if len(filtered) == 0 { + query.Del("token") + return + } + query["token"] = filtered +} + +func proxyBaseForRequestPath(requestPath string, instanceID int) string { + prefix := fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID) + path := strings.TrimSpace(requestPath) + if strings.HasPrefix(path, prefix) { + path = strings.TrimPrefix(path, prefix) + } + if path == "" || path == "/" { + return fmt.Sprintf("%s/", prefix) + } + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + if !strings.HasSuffix(path, "/") { + lastSlash := strings.LastIndex(path, "/") + if lastSlash >= 0 { + path = path[:lastSlash+1] + } else { + path = "/" + } + } + return prefix + path +} + +func websocketUpstreamOrigin(targetURL *url.URL) string { + if targetURL == nil { + return "" + } + scheme := targetURL.Scheme + switch scheme { + case "ws": + scheme = "http" + case "wss": + scheme = "https" + } + if scheme == "" || targetURL.Host == "" { + return "" + } + return scheme + "://" + targetURL.Host +} + +func (s *InstanceProxyService) openClawWebSocketOrigin(targetURL *url.URL) string { + if s != nil && s.openClawProxyOrigin != "" { + return s.openClawProxyOrigin + } + return websocketUpstreamOrigin(targetURL) +} + +func resolveOpenClawProxyOriginFromEnv() string { + for _, key := range []string{ + "OPENCLAW_PROXY_ORIGIN", + "CLAWMANAGER_TEAM_MANAGER_BASE_URL", + "CLAWMANAGER_BACKEND_URL", + } { + if origin := originFromURLString(os.Getenv(key)); origin != "" { + return origin + } + } + return "" +} + +func originFromURLString(rawURL string) string { + value := strings.TrimSpace(rawURL) + if value == "" { + return "" + } + parsed, err := url.Parse(value) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return "" + } + return parsed.Scheme + "://" + parsed.Host +} + func (s *InstanceProxyService) rewriteRedirectLocation(instanceID int, location string) string { if strings.HasPrefix(location, "/") && !strings.HasPrefix(location, "/api/v1/instances/") { return fmt.Sprintf("/api/v1/instances/%d/proxy%s", instanceID, location) diff --git a/backend/internal/services/instance_proxy_service_v2_test.go b/backend/internal/services/instance_proxy_service_v2_test.go new file mode 100644 index 0000000..d497eeb --- /dev/null +++ b/backend/internal/services/instance_proxy_service_v2_test.go @@ -0,0 +1,692 @@ +package services + +import ( + "errors" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "strings" + "testing" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" + + "github.com/gorilla/websocket" +) + +func TestInstanceProxyServiceUsesRuntimeBindingForV2(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/instances/123/proxy/apps/openclaw" { + t.Fatalf("unexpected upstream path %s", r.URL.Path) + } + if got := r.Header.Get("X-Forwarded-Prefix"); got != "/api/v1/instances/123/proxy" { + t.Fatalf("X-Forwarded-Prefix = %q", got) + } + if got := r.Header.Get("Authorization"); got != "Bearer internal-openclaw-token" { + t.Fatalf("Authorization = %q", got) + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + })) + defer upstream.Close() + + podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL) + workspacePath := "/workspaces/openclaw/user-45/instance-123" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[123] = &models.Instance{ + ID: 123, + UserID: 45, + Type: "openclaw", + RuntimeType: "gateway", + Status: "running", + WorkspacePath: &workspacePath, + RuntimeGeneration: 3, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[123] = &models.InstanceRuntimeBinding{ + InstanceID: 123, + RuntimePodID: 9, + GatewayPort: gatewayPort, + State: "running", + Generation: 3, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, PodIP: &podIP, State: "ready"}, + }, + } + accessService := NewInstanceAccessService() + defer accessService.Stop() + token, err := accessService.GenerateToken(45, 123, "openclaw", "/api/v1/instances/123/proxy/", 3000, time.Hour) + if err != nil { + t.Fatalf("GenerateToken returned error: %v", err) + } + service := NewInstanceProxyService(accessService) + service.instanceRepo = instanceRepo + service.bindingRepo = bindingRepo + service.runtimePodRepo = podRepo + service.httpClient = upstream.Client() + service.openClawGatewayToken = "internal-openclaw-token" + + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/123/proxy/apps/openclaw?token="+url.QueryEscape(token.Token), nil) + rec := httptest.NewRecorder() + + if err := service.ProxyRequest(req.Context(), 123, token.Token, rec, req); err != nil { + t.Fatalf("ProxyRequest returned error: %v", err) + } + if rec.Code != http.StatusOK || rec.Body.String() != "ok" { + t.Fatalf("unexpected proxy response %d %q", rec.Code, rec.Body.String()) + } +} + +func TestInstanceProxyServiceInjectsInstanceTokenForHermesLite(t *testing.T) { + instanceToken := "igt_hermes_instance" + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/chat" { + t.Fatalf("unexpected upstream path %s", r.URL.Path) + } + if r.URL.RawQuery != "" { + t.Fatalf("unexpected upstream query %q", r.URL.RawQuery) + } + if got := r.Header.Get("Authorization"); got != "Bearer "+instanceToken { + t.Fatalf("Authorization = %q", got) + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + })) + defer upstream.Close() + + podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL) + workspacePath := "/workspaces/hermes/user-45/instance-127" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[127] = &models.Instance{ + ID: 127, + UserID: 45, + Type: "hermes", + RuntimeType: "gateway", + InstanceMode: InstanceModeLite, + Status: "running", + AccessToken: &instanceToken, + WorkspacePath: &workspacePath, + RuntimeGeneration: 5, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[127] = &models.InstanceRuntimeBinding{ + InstanceID: 127, + RuntimePodID: 10, + GatewayPort: gatewayPort, + State: "running", + Generation: 5, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 10: {ID: 10, PodIP: &podIP, State: "ready"}, + }, + } + accessService := NewInstanceAccessService() + defer accessService.Stop() + token, err := accessService.GenerateToken(45, 127, "hermes", "/api/v1/instances/127/proxy/", 3000, time.Hour) + if err != nil { + t.Fatalf("GenerateToken returned error: %v", err) + } + service := NewInstanceProxyService(accessService) + service.instanceRepo = instanceRepo + service.bindingRepo = bindingRepo + service.runtimePodRepo = podRepo + service.httpClient = upstream.Client() + + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/127/proxy/chat?token="+url.QueryEscape(token.Token), nil) + rec := httptest.NewRecorder() + + if err := service.ProxyRequest(req.Context(), 127, token.Token, rec, req); err != nil { + t.Fatalf("ProxyRequest returned error: %v", err) + } + if rec.Code != http.StatusOK || rec.Body.String() != "ok" { + t.Fatalf("unexpected proxy response %d %q", rec.Code, rec.Body.String()) + } +} + +func TestInstanceProxyServiceUsesHermesLiteAccessURLForRootEntry(t *testing.T) { + instanceToken := "igt_hermes_instance" + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/chat/" { + t.Fatalf("unexpected upstream path %s", r.URL.Path) + } + if got := r.Header.Get("Authorization"); got != "Bearer "+instanceToken { + t.Fatalf("Authorization = %q", got) + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("Hermesok")) + })) + defer upstream.Close() + + podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL) + workspacePath := "/workspaces/hermes/user-45/instance-131" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[131] = &models.Instance{ + ID: 131, + UserID: 45, + Type: "hermes", + RuntimeType: "gateway", + InstanceMode: InstanceModeLite, + Status: "running", + AccessToken: &instanceToken, + WorkspacePath: &workspacePath, + RuntimeGeneration: 5, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[131] = &models.InstanceRuntimeBinding{ + InstanceID: 131, + RuntimePodID: 14, + GatewayPort: gatewayPort, + State: "running", + Generation: 5, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 14: {ID: 14, PodIP: &podIP, State: "ready"}, + }, + } + accessService := NewInstanceAccessService() + defer accessService.Stop() + token, err := accessService.GenerateToken(45, 131, "hermes", "/api/v1/instances/131/proxy/chat/", 3000, time.Hour) + if err != nil { + t.Fatalf("GenerateToken returned error: %v", err) + } + service := NewInstanceProxyService(accessService) + service.instanceRepo = instanceRepo + service.bindingRepo = bindingRepo + service.runtimePodRepo = podRepo + service.httpClient = upstream.Client() + + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/131/proxy/?token="+url.QueryEscape(token.Token), nil) + rec := httptest.NewRecorder() + + if err := service.ProxyRequest(req.Context(), 131, token.Token, rec, req); err != nil { + t.Fatalf("ProxyRequest returned error: %v", err) + } + if rec.Code != http.StatusOK { + t.Fatalf("unexpected proxy response %d %q", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), ``) { + t.Fatalf("expected Hermes Lite HTML to include chat proxy base, got %q", rec.Body.String()) + } +} + +func TestInstanceProxyServicePreservesHermesRuntimeQueryToken(t *testing.T) { + instanceToken := "igt_hermes_instance" + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/ws" { + t.Fatalf("unexpected upstream path %s", r.URL.Path) + } + if got := r.URL.Query().Get("token"); got != "hermes-session-token" { + t.Fatalf("upstream token query = %q", got) + } + if got := r.Header.Get("Authorization"); got != "Bearer "+instanceToken { + t.Fatalf("Authorization = %q", got) + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + })) + defer upstream.Close() + + podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL) + workspacePath := "/workspaces/hermes/user-45/instance-130" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[130] = &models.Instance{ + ID: 130, + UserID: 45, + Type: "hermes", + RuntimeType: "gateway", + InstanceMode: InstanceModeLite, + Status: "running", + AccessToken: &instanceToken, + WorkspacePath: &workspacePath, + RuntimeGeneration: 5, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[130] = &models.InstanceRuntimeBinding{ + InstanceID: 130, + RuntimePodID: 13, + GatewayPort: gatewayPort, + State: "running", + Generation: 5, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 13: {ID: 13, PodIP: &podIP, State: "ready"}, + }, + } + accessService := NewInstanceAccessService() + defer accessService.Stop() + token, err := accessService.GenerateToken(45, 130, "hermes", "/api/v1/instances/130/proxy/chat/", 3000, time.Hour) + if err != nil { + t.Fatalf("GenerateToken returned error: %v", err) + } + service := NewInstanceProxyService(accessService) + service.instanceRepo = instanceRepo + service.bindingRepo = bindingRepo + service.runtimePodRepo = podRepo + service.httpClient = upstream.Client() + + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/130/proxy/api/ws?token=hermes-session-token", nil) + rec := httptest.NewRecorder() + + if err := service.ProxyRequest(req.Context(), 130, token.Token, rec, req); err != nil { + t.Fatalf("ProxyRequest returned error: %v", err) + } + if rec.Code != http.StatusOK || rec.Body.String() != "ok" { + t.Fatalf("unexpected proxy response %d %q", rec.Code, rec.Body.String()) + } +} + +func TestInstanceProxyServiceRewritesHermesLiteHTMLBase(t *testing.T) { + instanceToken := "igt_hermes_instance" + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + t.Fatalf("unexpected upstream path %s", r.URL.Path) + } + if got := r.Header.Get("Authorization"); got != "Bearer "+instanceToken { + t.Fatalf("Authorization = %q", got) + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("Hermesok")) + })) + defer upstream.Close() + + podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL) + workspacePath := "/workspaces/hermes/user-45/instance-128" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[128] = &models.Instance{ + ID: 128, + UserID: 45, + Type: "hermes", + RuntimeType: "gateway", + InstanceMode: InstanceModeLite, + Status: "running", + AccessToken: &instanceToken, + WorkspacePath: &workspacePath, + RuntimeGeneration: 5, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[128] = &models.InstanceRuntimeBinding{ + InstanceID: 128, + RuntimePodID: 11, + GatewayPort: gatewayPort, + State: "running", + Generation: 5, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 11: {ID: 11, PodIP: &podIP, State: "ready"}, + }, + } + accessService := NewInstanceAccessService() + defer accessService.Stop() + token, err := accessService.GenerateToken(45, 128, "hermes", "/api/v1/instances/128/proxy/", 3000, time.Hour) + if err != nil { + t.Fatalf("GenerateToken returned error: %v", err) + } + service := NewInstanceProxyService(accessService) + service.instanceRepo = instanceRepo + service.bindingRepo = bindingRepo + service.runtimePodRepo = podRepo + service.httpClient = upstream.Client() + + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/128/proxy/?token="+url.QueryEscape(token.Token), nil) + rec := httptest.NewRecorder() + + if err := service.ProxyRequest(req.Context(), 128, token.Token, rec, req); err != nil { + t.Fatalf("ProxyRequest returned error: %v", err) + } + if rec.Code != http.StatusOK { + t.Fatalf("unexpected proxy response %d %q", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), ``) { + t.Fatalf("expected Hermes Lite HTML to include proxy base, got %q", rec.Body.String()) + } +} + +func TestInstanceProxyServiceProxiesHermesLiteWebSocket(t *testing.T) { + instanceToken := "igt_hermes_instance" + upgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }} + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/ws" { + t.Fatalf("unexpected upstream websocket path %s", r.URL.Path) + } + if got := r.URL.Query().Get("token"); got != "hermes-ws-token" { + t.Fatalf("upstream websocket token query = %q", got) + } + if got := r.Header.Get("Authorization"); got != "Bearer "+instanceToken { + t.Fatalf("Authorization = %q", got) + } + if got := r.Header.Get("X-Forwarded-Prefix"); got != "/api/v1/instances/129/proxy" { + t.Fatalf("X-Forwarded-Prefix = %q", got) + } + if got := r.Header.Get("X-Forwarded-Host"); got == "" { + t.Fatalf("X-Forwarded-Host is empty") + } + if got := r.Header.Get("X-Forwarded-For"); got == "" { + t.Fatalf("X-Forwarded-For is empty") + } + if got, want := r.Header.Get("Origin"), "http://"+r.Host; got != want { + t.Fatalf("Origin = %q, want %q", got, want) + } + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + t.Fatalf("upstream websocket upgrade failed: %v", err) + } + defer conn.Close() + messageType, message, err := conn.ReadMessage() + if err != nil { + t.Fatalf("upstream websocket read failed: %v", err) + } + if string(message) != "ping" { + t.Fatalf("upstream websocket message = %q", message) + } + if err := conn.WriteMessage(messageType, []byte("pong")); err != nil { + t.Fatalf("upstream websocket write failed: %v", err) + } + })) + defer upstream.Close() + + podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL) + workspacePath := "/workspaces/hermes/user-45/instance-129" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[129] = &models.Instance{ + ID: 129, + UserID: 45, + Type: "hermes", + RuntimeType: "gateway", + InstanceMode: InstanceModeLite, + Status: "running", + AccessToken: &instanceToken, + WorkspacePath: &workspacePath, + RuntimeGeneration: 5, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[129] = &models.InstanceRuntimeBinding{ + InstanceID: 129, + RuntimePodID: 12, + GatewayPort: gatewayPort, + State: "running", + Generation: 5, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 12: {ID: 12, PodIP: &podIP, State: "ready"}, + }, + } + accessService := NewInstanceAccessService() + defer accessService.Stop() + token, err := accessService.GenerateToken(45, 129, "hermes", "/api/v1/instances/129/proxy/", 3000, time.Hour) + if err != nil { + t.Fatalf("GenerateToken returned error: %v", err) + } + service := NewInstanceProxyService(accessService) + service.instanceRepo = instanceRepo + service.bindingRepo = bindingRepo + service.runtimePodRepo = podRepo + + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := service.ProxyWebSocket(r.Context(), 129, token.Token, w, r); err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + } + })) + defer proxy.Close() + + wsURL := "ws" + strings.TrimPrefix(proxy.URL, "http") + "/api/v1/instances/129/proxy/ws?token=hermes-ws-token" + clientConn, _, err := websocket.DefaultDialer.Dial(wsURL, nil) + if err != nil { + t.Fatalf("client websocket dial failed: %v", err) + } + defer clientConn.Close() + if err := clientConn.WriteMessage(websocket.TextMessage, []byte("ping")); err != nil { + t.Fatalf("client websocket write failed: %v", err) + } + _, message, err := clientConn.ReadMessage() + if err != nil { + t.Fatalf("client websocket read failed: %v", err) + } + if string(message) != "pong" { + t.Fatalf("client websocket message = %q", message) + } +} + +func TestInstanceProxyServiceUsesBaseProxyEntryForV2OpenClaw(t *testing.T) { + workspacePath := "/workspaces/openclaw/user-45/instance-123" + accessService := NewInstanceAccessService() + t.Cleanup(accessService.Stop) + service := NewInstanceProxyService(accessService) + got := service.GetProxyURLForInstance(&models.Instance{ + ID: 123, + Type: "openclaw", + RuntimeType: "gateway", + WorkspacePath: &workspacePath, + }, "token+with/slash") + + want := "/api/v1/instances/123/proxy/?token=token%2Bwith%2Fslash" + if got != want { + t.Fatalf("GetProxyURLForInstance() = %q, want %q", got, want) + } +} + +func TestInstanceProxyServiceUsesChatEntryForHermesLite(t *testing.T) { + workspacePath := "/workspaces/hermes/user-45/instance-123" + accessService := NewInstanceAccessService() + t.Cleanup(accessService.Stop) + service := NewInstanceProxyService(accessService) + got := service.GetProxyURLForInstance(&models.Instance{ + ID: 123, + Type: "hermes", + RuntimeType: "gateway", + InstanceMode: InstanceModeLite, + WorkspacePath: &workspacePath, + }, "token+with/slash") + + want := "/api/v1/instances/123/proxy/chat/?token=token%2Bwith%2Fslash" + if got != want { + t.Fatalf("GetProxyURLForInstance() = %q, want %q", got, want) + } +} + +func TestProxyBaseForRequestPathUsesSubdirectory(t *testing.T) { + got := proxyBaseForRequestPath("/api/v1/instances/123/proxy/__openclaw__/canvas/", 123) + want := "/api/v1/instances/123/proxy/__openclaw__/canvas/" + if got != want { + t.Fatalf("proxyBaseForRequestPath() = %q, want %q", got, want) + } + + got = proxyBaseForRequestPath("/api/v1/instances/123/proxy/__openclaw__/canvas/app.js", 123) + if got != want { + t.Fatalf("proxyBaseForRequestPath(file) = %q, want %q", got, want) + } +} + +func TestResolveOpenClawProxyOriginFromEnvUsesInternalOrigin(t *testing.T) { + t.Setenv("OPENCLAW_PROXY_ORIGIN", "") + t.Setenv("CLAWMANAGER_BACKEND_URL", "") + t.Setenv("CLAWMANAGER_TEAM_MANAGER_BASE_URL", "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001/api") + + got := resolveOpenClawProxyOriginFromEnv() + want := "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + if got != want { + t.Fatalf("resolveOpenClawProxyOriginFromEnv() = %q, want %q", got, want) + } +} + +func TestOpenClawWebSocketOriginPrefersConfiguredProxyOrigin(t *testing.T) { + accessService := NewInstanceAccessService() + t.Cleanup(accessService.Stop) + service := NewInstanceProxyService(accessService) + service.openClawProxyOrigin = "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + + got := service.openClawWebSocketOrigin(&url.URL{Scheme: "ws", Host: "10.42.0.63:20000"}) + want := "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + if got != want { + t.Fatalf("openClawWebSocketOrigin() = %q, want %q", got, want) + } +} + +func TestWebSocketUpstreamOriginFallsBackToGatewayHost(t *testing.T) { + tests := []struct { + name string + url *url.URL + want string + }{ + { + name: "plain websocket", + url: &url.URL{Scheme: "ws", Host: "10.42.0.63:20000"}, + want: "http://10.42.0.63:20000", + }, + { + name: "tls websocket", + url: &url.URL{Scheme: "wss", Host: "gateway.example.test"}, + want: "https://gateway.example.test", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := websocketUpstreamOrigin(tt.url); got != tt.want { + t.Fatalf("websocketUpstreamOrigin() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestInstanceProxyServiceRejectsStoppedV2InstanceWithStaleBinding(t *testing.T) { + workspacePath := "/workspaces/openclaw/user-45/instance-125" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[125] = &models.Instance{ + ID: 125, + UserID: 45, + Type: "openclaw", + RuntimeType: "gateway", + Status: "stopped", + WorkspacePath: &workspacePath, + RuntimeGeneration: 4, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[125] = &models.InstanceRuntimeBinding{ + InstanceID: 125, + RuntimePodID: 9, + GatewayPort: 20025, + State: "running", + Generation: 4, + } + podIP := "10.42.0.125" + service, token := newV2ProxyTestService(t, instanceRepo, bindingRepo, &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, PodIP: &podIP}, + }, + }, 45, 125, "openclaw") + + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/125/proxy/", nil) + err := service.ProxyRequest(req.Context(), 125, token.Token, httptest.NewRecorder(), req) + if !errors.Is(err, ErrInstanceGatewayUnavailable) { + t.Fatalf("ProxyRequest error = %v, want ErrInstanceGatewayUnavailable", err) + } +} + +func TestInstanceProxyServiceRejectsV2BindingGenerationMismatch(t *testing.T) { + workspacePath := "/workspaces/hermes/user-45/instance-126" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[126] = &models.Instance{ + ID: 126, + UserID: 45, + Type: "hermes", + RuntimeType: "gateway", + Status: "running", + WorkspacePath: &workspacePath, + RuntimeGeneration: 8, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[126] = &models.InstanceRuntimeBinding{ + InstanceID: 126, + RuntimePodID: 10, + GatewayPort: 20026, + State: "running", + Generation: 7, + } + podIP := "10.42.0.126" + service, token := newV2ProxyTestService(t, instanceRepo, bindingRepo, &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 10: {ID: 10, PodIP: &podIP}, + }, + }, 45, 126, "hermes") + + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/126/proxy/", nil) + err := service.ProxyRequest(req.Context(), 126, token.Token, httptest.NewRecorder(), req) + if !errors.Is(err, ErrInstanceGatewayUnavailable) { + t.Fatalf("ProxyRequest error = %v, want ErrInstanceGatewayUnavailable", err) + } +} + +func TestInstanceProxyServiceReturnsUnavailableWhenV2BindingMissing(t *testing.T) { + workspacePath := "/workspaces/hermes/user-45/instance-124" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[124] = &models.Instance{ + ID: 124, + UserID: 45, + Type: "hermes", + RuntimeType: "gateway", + Status: "running", + WorkspacePath: &workspacePath, + } + accessService := NewInstanceAccessService() + defer accessService.Stop() + token, err := accessService.GenerateToken(45, 124, "hermes", "/api/v1/instances/124/proxy/", 3000, time.Hour) + if err != nil { + t.Fatalf("GenerateToken returned error: %v", err) + } + service := NewInstanceProxyService(accessService) + service.instanceRepo = instanceRepo + service.bindingRepo = newFakeRuntimeBindingRepo() + service.runtimePodRepo = &fakeRuntimePodRepo{} + + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/124/proxy/", nil) + rec := httptest.NewRecorder() + + err = service.ProxyRequest(req.Context(), 124, token.Token, rec, req) + if !errors.Is(err, ErrInstanceGatewayUnavailable) { + t.Fatalf("ProxyRequest error = %v, want ErrInstanceGatewayUnavailable", err) + } +} + +func newV2ProxyTestService(t *testing.T, instanceRepo repository.InstanceRepository, bindingRepo repository.InstanceRuntimeBindingRepository, podRepo repository.RuntimePodRepository, userID int, instanceID int, instanceType string) (*InstanceProxyService, *AccessToken) { + t.Helper() + accessService := NewInstanceAccessService() + t.Cleanup(accessService.Stop) + token, err := accessService.GenerateToken(userID, instanceID, instanceType, "/api/v1/instances/"+strconv.Itoa(instanceID)+"/proxy/", 3000, time.Hour) + if err != nil { + t.Fatalf("GenerateToken returned error: %v", err) + } + service := NewInstanceProxyService(accessService) + service.instanceRepo = instanceRepo + service.bindingRepo = bindingRepo + service.runtimePodRepo = podRepo + return service, token +} + +func splitURLHostPortForProxyTest(t *testing.T, rawURL string) (string, int) { + t.Helper() + parsed, err := url.Parse(rawURL) + if err != nil { + t.Fatalf("parse upstream URL: %v", err) + } + host, portText, err := net.SplitHostPort(parsed.Host) + if err != nil { + t.Fatalf("split upstream host port: %v", err) + } + port, err := strconv.Atoi(portText) + if err != nil { + t.Fatalf("parse upstream port: %v", err) + } + return host, port +} diff --git a/backend/internal/services/instance_service.go b/backend/internal/services/instance_service.go index db53c9d..0729a9b 100644 --- a/backend/internal/services/instance_service.go +++ b/backend/internal/services/instance_service.go @@ -6,6 +6,8 @@ import ( "encoding/hex" "encoding/json" "fmt" + "os" + "path" "strconv" "strings" "time" @@ -79,11 +81,13 @@ func (s *instanceService) ValidateCreateRequests(userID int, requests []CreateIn currentGPU := 0 existingNames := map[string]struct{}{} for _, existing := range existingInstances { - currentCPU += existing.CPUCores - currentMemory += existing.MemoryGB - currentStorage += existing.DiskGB - if existing.GPUEnabled { - currentGPU += existing.GPUCount + if instanceModeUsesDedicatedResources(modeForExistingInstance(&existing)) { + currentCPU += existing.CPUCores + currentMemory += existing.MemoryGB + currentStorage += existing.DiskGB + if existing.GPUEnabled { + currentGPU += existing.GPUCount + } } existingNames[strings.TrimSpace(strings.ToLower(existing.Name))] = struct{}{} } @@ -102,11 +106,13 @@ func (s *instanceService) ValidateCreateRequests(userID int, requests []CreateIn return fmt.Errorf("instance name already exists") } requestNames[normalizedName] = struct{}{} - requestedCPU += req.CPUCores - requestedMemory += req.MemoryGB - requestedStorage += req.DiskGB - if req.GPUEnabled { - requestedGPU += req.GPUCount + if instanceModeUsesDedicatedResources(resolveCreateInstanceMode(req)) { + requestedCPU += req.CPUCores + requestedMemory += req.MemoryGB + requestedStorage += req.DiskGB + if req.GPUEnabled { + requestedGPU += req.GPUCount + } } } @@ -130,8 +136,10 @@ func (s *instanceService) ValidateCreateRequests(userID int, requests []CreateIn type CreateInstanceRequest struct { Name string `json:"name" validate:"required,min=3,max=50"` Description *string `json:"description,omitempty"` - Type string `json:"type" validate:"required,oneof=openclaw ubuntu debian centos custom webtop hermes"` - RuntimeType string `json:"runtime_type" validate:"omitempty,oneof=desktop shell"` + Type string `json:"type" validate:"required,oneof=openclaw hermes"` + Mode string `json:"mode" validate:"omitempty,oneof=lite pro"` + InstanceMode string `json:"instance_mode" validate:"omitempty,oneof=lite pro"` + RuntimeType string `json:"runtime_type" validate:"omitempty,oneof=gateway desktop shell"` CPUCores float64 `json:"cpu_cores" validate:"required,min=0.1,max=32"` MemoryGB int `json:"memory_gb" validate:"required,min=1,max=128"` DiskGB int `json:"disk_gb" validate:"required,min=10,max=1000"` @@ -159,6 +167,14 @@ type TeamInstanceConfig struct { SharedUmask string } +type instanceModeLimitConfig struct { + Capacity *int + MaxCPU *float64 + MaxMemoryGB *int + MaxStorageGB *int + MaxGPUCount *int +} + // UpdateInstanceRequest holds data for updating an instance type UpdateInstanceRequest struct { Name *string `json:"name,omitempty" validate:"omitempty,min=3,max=50"` @@ -167,14 +183,17 @@ type UpdateInstanceRequest struct { // InstanceStatus holds the status of an instance type InstanceStatus struct { - InstanceID int `json:"instance_id"` - Status string `json:"status"` - PodName *string `json:"pod_name,omitempty"` - PodNamespace *string `json:"pod_namespace,omitempty"` - PodIP *string `json:"pod_ip,omitempty"` - PodStatus string `json:"pod_status,omitempty"` - CreatedAt time.Time `json:"created_at"` - StartedAt *time.Time `json:"started_at,omitempty"` + InstanceID int `json:"instance_id"` + Status string `json:"status"` + Availability string `json:"availability,omitempty"` + AgentType string `json:"agent_type,omitempty"` + WorkspaceUsageBytes int64 `json:"workspace_usage_bytes,omitempty"` + PodName *string `json:"pod_name,omitempty"` + PodNamespace *string `json:"pod_namespace,omitempty"` + PodIP *string `json:"pod_ip,omitempty"` + PodStatus string `json:"pod_status,omitempty"` + CreatedAt time.Time `json:"created_at"` + StartedAt *time.Time `json:"started_at,omitempty"` } // instanceService implements InstanceService @@ -184,7 +203,12 @@ type instanceService struct { llmModelRepo repository.LLMModelRepository openClawConfigService OpenClawConfigService allowPrivilegedPods bool + runtimePodRepo repository.RuntimePodRepository + bindingRepo repository.InstanceRuntimeBindingRepository + agentClient RuntimeAgentClient + workspaceRoot string podService *k8s.PodService + deploymentService *k8s.InstanceDeploymentService pvcService *k8s.PVCService serviceService *k8s.ServiceService networkPolicyService *k8s.NetworkPolicyService @@ -203,6 +227,17 @@ func WithPrivilegedInstancePods(allowed bool) InstanceServiceOption { } } +func WithV2RuntimeLifecycle(runtimePodRepo repository.RuntimePodRepository, bindingRepo repository.InstanceRuntimeBindingRepository, agentClient RuntimeAgentClient, workspaceRoot string) InstanceServiceOption { + return func(s *instanceService) { + s.runtimePodRepo = runtimePodRepo + s.bindingRepo = bindingRepo + s.agentClient = agentClient + if strings.TrimSpace(workspaceRoot) != "" { + s.workspaceRoot = strings.TrimSpace(workspaceRoot) + } + } +} + // NewInstanceService creates a new instance service func NewInstanceService(instanceRepo repository.InstanceRepository, quotaRepo repository.QuotaRepository, llmModelRepo repository.LLMModelRepository, openClawConfigService OpenClawConfigService, options ...InstanceServiceOption) InstanceService { service := &instanceService{ @@ -210,7 +245,9 @@ func NewInstanceService(instanceRepo repository.InstanceRepository, quotaRepo re quotaRepo: quotaRepo, llmModelRepo: llmModelRepo, openClawConfigService: openClawConfigService, + workspaceRoot: "/workspaces", podService: k8s.NewPodService(), + deploymentService: k8s.NewInstanceDeploymentService(), pvcService: k8s.NewPVCService(), serviceService: k8s.NewServiceService(), networkPolicyService: k8s.NewNetworkPolicyService(), @@ -227,6 +264,7 @@ func NewInstanceService(instanceRepo repository.InstanceRepository, quotaRepo re func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models.Instance, error) { ctx := context.Background() req.Name = strings.TrimSpace(req.Name) + req.Type = strings.ToLower(strings.TrimSpace(req.Type)) environmentOverrides, err := normalizeEnvironmentOverrides(req.EnvironmentOverrides) if err != nil { return nil, err @@ -245,6 +283,11 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models if quota == nil { return nil, fmt.Errorf("user quota not found") } + instanceMode := resolveCreateInstanceMode(req) + modeRuntimeType, _ := RuntimeTypeForInstanceMode(instanceMode) + if !hasExplicitCreateInstanceMode(req) && normalizeInstanceRuntimeType(req.RuntimeType) == RuntimeBackendShell { + modeRuntimeType = RuntimeBackendShell + } // Check instance count limit currentCount, err := s.instanceRepo.CountByUserID(userID) @@ -266,11 +309,13 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models currentStorage := 0 currentGPU := 0 for _, existing := range existingInstances { - currentCPU += existing.CPUCores - currentMemory += existing.MemoryGB - currentStorage += existing.DiskGB - if existing.GPUEnabled { - currentGPU += existing.GPUCount + if instanceModeUsesDedicatedResources(modeForExistingInstance(&existing)) { + currentCPU += existing.CPUCores + currentMemory += existing.MemoryGB + currentStorage += existing.DiskGB + if existing.GPUEnabled { + currentGPU += existing.GPUCount + } } } @@ -282,43 +327,58 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models return nil, fmt.Errorf("instance name already exists") } - // Check CPU limit - if currentCPU+req.CPUCores > quota.MaxCPUCores { - return nil, fmt.Errorf("CPU cores exceed quota: current %v, requested %v, max %v", currentCPU, req.CPUCores, quota.MaxCPUCores) - } - - // Check memory limit - if currentMemory+req.MemoryGB > quota.MaxMemoryGB { - return nil, fmt.Errorf("memory exceed quota: current %dGB, requested %dGB, max %dGB", currentMemory, req.MemoryGB, quota.MaxMemoryGB) - } - - // Check storage limit - if currentStorage+req.DiskGB > quota.MaxStorageGB { - return nil, fmt.Errorf("storage exceed quota: current %dGB, requested %dGB, max %dGB", currentStorage, req.DiskGB, quota.MaxStorageGB) - } - - // Check GPU limit requestedGPU := 0 if req.GPUEnabled { requestedGPU = req.GPUCount } - if currentGPU+requestedGPU > quota.MaxGPUCount { - return nil, fmt.Errorf("GPU count exceed quota: current %d, requested %d, max %d", currentGPU, requestedGPU, quota.MaxGPUCount) + if instanceModeUsesDedicatedResources(instanceMode) { + // Check CPU limit + if currentCPU+req.CPUCores > quota.MaxCPUCores { + return nil, fmt.Errorf("CPU cores exceed quota: current %v, requested %v, max %v", currentCPU, req.CPUCores, quota.MaxCPUCores) + } + + // Check memory limit + if currentMemory+req.MemoryGB > quota.MaxMemoryGB { + return nil, fmt.Errorf("memory exceed quota: current %dGB, requested %dGB, max %dGB", currentMemory, req.MemoryGB, quota.MaxMemoryGB) + } + + // Check storage limit + if currentStorage+req.DiskGB > quota.MaxStorageGB { + return nil, fmt.Errorf("storage exceed quota: current %dGB, requested %dGB, max %dGB", currentStorage, req.DiskGB, quota.MaxStorageGB) + } + + // Check GPU limit + if currentGPU+requestedGPU > quota.MaxGPUCount { + return nil, fmt.Errorf("GPU count exceed quota: current %d, requested %d, max %d", currentGPU, requestedGPU, quota.MaxGPUCount) + } + } + if err := s.enforceInstanceModeLimits(ctx, instanceMode, req.CPUCores, req.MemoryGB, req.DiskGB, requestedGPU); err != nil { + return nil, err + } + if runtimeType, isV2 := NormalizeV2RuntimeType(req.Type); isV2 && instanceMode == InstanceModeLite { + return s.createV2Instance(ctx, userID, req, runtimeType, environmentOverridesJSON) } runtimeConfig := buildRuntimeConfig(req.Type, req.OSType, req.OSVersion, req.ImageRegistry, req.ImageTag) runtimeType := normalizeInstanceRuntimeType(req.RuntimeType) + if modeRuntimeType != "" { + runtimeType = modeRuntimeType + } if (req.ImageRegistry == nil || strings.TrimSpace(*req.ImageRegistry) == "") && (req.ImageTag == nil || strings.TrimSpace(*req.ImageTag) == "") { if selection, ok := runtimeImageOverride(req.Type); ok { image := selection.Image req.ImageRegistry = &image req.ImageTag = nil - runtimeType = normalizeInstanceRuntimeType(selection.RuntimeType) + if modeRuntimeType == "" { + runtimeType = normalizeInstanceRuntimeType(selection.RuntimeType) + } runtimeConfig = buildRuntimeConfig(req.Type, req.OSType, req.OSVersion, req.ImageRegistry, req.ImageTag) } } else if req.ImageRegistry != nil { if selection, ok := runtimeImageOverrideForImage(req.Type, *req.ImageRegistry); ok { - runtimeType = normalizeInstanceRuntimeType(selection.RuntimeType) + if modeRuntimeType == "" { + runtimeType = normalizeInstanceRuntimeType(selection.RuntimeType) + } } } @@ -334,6 +394,7 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models Description: req.Description, Type: req.Type, RuntimeType: runtimeType, + InstanceMode: InstanceModeForRuntimeType(runtimeType), Status: "creating", CPUCores: req.CPUCores, MemoryGB: req.MemoryGB, @@ -506,18 +567,29 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models SecurityMode: s.securityModeForInstance(instance.Type), } - pod, err := s.podService.CreatePod(ctx, podConfig) - if err != nil { - // Rollback: delete PVC and instance record - s.pvcService.DeletePVC(ctx, userID, instance.ID) - if bootstrapSnapshot != nil { - _ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err) - } - s.instanceRepo.Delete(instance.ID) - return nil, fmt.Errorf("failed to create pod: %w", err) - } - + var workloadNamespace string + var workloadName string if instanceUsesDesktopRuntime(instance) { + if s.deploymentService == nil { + s.pvcService.DeletePVC(ctx, userID, instance.ID) + if bootstrapSnapshot != nil { + _ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, fmt.Errorf("instance deployment service is not configured")) + } + s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("instance deployment service is not configured") + } + deployment, err := s.deploymentService.EnsureDeployment(ctx, podConfig, 1) + if err != nil { + s.pvcService.DeletePVC(ctx, userID, instance.ID) + if bootstrapSnapshot != nil { + _ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err) + } + s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to create deployment: %w", err) + } + workloadNamespace = deployment.Namespace + workloadName = deployment.Name + // Create Service for browser desktop access. serviceConfig := k8s.ServiceConfig{ InstanceID: instance.ID, @@ -529,8 +601,8 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models serviceInfo, err := s.serviceService.CreateService(ctx, serviceConfig) if err != nil { - // Rollback: delete pod, PVC and instance record - s.podService.DeletePod(ctx, userID, instance.ID) + // Rollback: delete Deployment, PVC and instance record. + _ = s.deploymentService.DeleteDeployment(ctx, userID, instance.ID) s.pvcService.DeletePVC(ctx, userID, instance.ID) if bootstrapSnapshot != nil { _ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err) @@ -541,12 +613,25 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models fmt.Printf("Instance %d: Service created successfully (ClusterIP: %s)\n", instance.ID, serviceInfo.ClusterIP) } else { + pod, err := s.podService.CreatePod(ctx, podConfig) + if err != nil { + // Rollback: delete PVC and instance record. + s.pvcService.DeletePVC(ctx, userID, instance.ID) + if bootstrapSnapshot != nil { + _ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err) + } + s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to create pod: %w", err) + } + workloadNamespace = pod.Namespace + workloadName = pod.Name fmt.Printf("Instance %d: Shell runtime selected, skipping desktop service creation\n", instance.ID) } - // Update instance with pod info - podNamespace := pod.Namespace - podName := pod.Name + // Update instance with initial workload info. For Pro instances this is the + // stable Deployment name; sync later records the active Pod name/IP. + podNamespace := workloadNamespace + podName := workloadName instance.PodNamespace = &podNamespace instance.PodName = &podName instance.Status = "creating" @@ -576,6 +661,54 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models return instance, nil } +func (s *instanceService) createV2Instance(ctx context.Context, userID int, req CreateInstanceRequest, runtimeType string, environmentOverridesJSON *string) (*models.Instance, error) { + now := time.Now() + workspaceRoot := s.runtimeWorkspaceRoot() + instance := &models.Instance{ + UserID: userID, + Name: strings.TrimSpace(req.Name), + Description: trimOptionalString(req.Description), + Type: runtimeType, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + Status: "creating", + CPUCores: req.CPUCores, + MemoryGB: req.MemoryGB, + DiskGB: req.DiskGB, + GPUEnabled: req.GPUEnabled, + GPUCount: req.GPUCount, + OSType: req.OSType, + OSVersion: req.OSVersion, + ImageRegistry: req.ImageRegistry, + ImageTag: req.ImageTag, + EnvironmentOverridesJSON: environmentOverridesJSON, + StorageClass: strings.TrimSpace(req.StorageClass), + MountPath: workspaceRoot, + RuntimeGeneration: 1, + CreatedAt: now, + UpdatedAt: now, + StartedAt: &now, + } + + if err := s.instanceRepo.Create(instance); err != nil { + return nil, fmt.Errorf("failed to create instance record: %w", err) + } + + workspacePath, err := ensureRuntimeWorkspaceDirectories(workspaceRoot, runtimeType, userID, instance.ID) + if err != nil { + _ = s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to create instance workspace: %w", err) + } + if err := s.instanceRepo.SetWorkspacePath(ctx, instance.ID, workspacePath); err != nil { + _ = s.instanceRepo.Delete(instance.ID) + return nil, fmt.Errorf("failed to persist instance workspace path: %w", err) + } + instance.WorkspacePath = &workspacePath + + GetHub().BroadcastInstanceStatus(userID, instance) + return instance, nil +} + // GetByID gets an instance by ID func (s *instanceService) GetByID(id int) (*models.Instance, error) { return s.instanceRepo.GetByID(id) @@ -626,6 +759,13 @@ func (s *instanceService) Start(instanceID int) error { if instance.Status == "running" { return fmt.Errorf("instance is already running") } + if err := s.enforceInstanceModeLimits(ctx, modeForExistingInstance(instance), instance.CPUCores, instance.MemoryGB, instance.DiskGB, instance.GPUCount); err != nil { + return err + } + + if runtimeType, ok := v2RuntimeTypeForInstance(instance); ok { + return s.startV2Instance(ctx, instance, runtimeType) + } if _, err := s.ensureGatewayToken(instance); err != nil { return fmt.Errorf("failed to provision instance gateway token: %w", err) @@ -686,12 +826,19 @@ func (s *instanceService) Start(instanceID int) error { SecurityMode: s.securityModeForInstance(instance.Type), } - pod, err := s.podService.CreatePod(ctx, podConfig) - if err != nil { - return fmt.Errorf("failed to create pod: %w", err) - } - + var workloadNamespace string + var workloadName string if instanceUsesDesktopRuntime(instance) { + if s.deploymentService == nil { + return fmt.Errorf("instance deployment service is not configured") + } + deployment, err := s.deploymentService.EnsureDeployment(ctx, podConfig, 1) + if err != nil { + return fmt.Errorf("failed to ensure deployment: %w", err) + } + workloadNamespace = deployment.Namespace + workloadName = deployment.Name + // Ensure Service exists (create if not exists) serviceExists, _ := s.serviceService.ServiceExists(ctx, instance.UserID, instance.ID) if !serviceExists { @@ -708,12 +855,19 @@ func (s *instanceService) Start(instanceID int) error { // Don't fail if service creation fails, pod is already running } } + } else { + pod, err := s.podService.CreatePod(ctx, podConfig) + if err != nil { + return fmt.Errorf("failed to create pod: %w", err) + } + workloadNamespace = pod.Namespace + workloadName = pod.Name } // Update instance status now := time.Now() - podNamespace := pod.Namespace - podName := pod.Name + podNamespace := workloadNamespace + podName := workloadName instance.PodNamespace = &podNamespace instance.PodName = &podName instance.Status = "creating" @@ -792,6 +946,25 @@ func (s *instanceService) buildGatewayEnv(instance *models.Instance) (map[string }, nil } +func (s *instanceService) BuildGatewayEnv(instance *models.Instance) (map[string]string, error) { + if instance == nil || !supportsManagedRuntimeIntegration(instance.Type) { + return s.buildGatewayEnv(instance) + } + if instance.AccessToken == nil || strings.TrimSpace(*instance.AccessToken) == "" { + if s == nil || s.instanceRepo == nil { + return nil, fmt.Errorf("instance repository is not configured") + } + if _, err := s.ensureGatewayToken(instance); err != nil { + return nil, err + } + } + gatewayEnv, err := s.buildGatewayEnv(instance) + if err != nil { + return nil, err + } + return buildInstanceGatewayEnv(instance, gatewayEnv) +} + func (s *instanceService) ensureAgentBootstrapToken(instance *models.Instance) (string, error) { if instance.AgentBootstrapToken != nil && strings.TrimSpace(*instance.AgentBootstrapToken) != "" { return strings.TrimSpace(*instance.AgentBootstrapToken), nil @@ -964,6 +1137,130 @@ func mergeEnvMaps(base map[string]string, overlay map[string]string) map[string] return merged } +func (s *instanceService) startV2Instance(ctx context.Context, instance *models.Instance, runtimeType string) error { + if err := s.ensureV2Workspace(ctx, instance, runtimeType); err != nil { + return err + } + nextGeneration := instance.RuntimeGeneration + 1 + if nextGeneration <= 0 { + nextGeneration = 1 + } + if err := s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "creating", nextGeneration, nil); err != nil { + return fmt.Errorf("failed to mark v2 instance creating: %w", err) + } + instance.Status = "creating" + instance.RuntimeGeneration = nextGeneration + instance.RuntimeErrorMessage = nil + now := time.Now() + instance.StartedAt = &now + instance.UpdatedAt = now + GetHub().BroadcastInstanceStatus(instance.UserID, instance) + return nil +} + +func (s *instanceService) stopV2Instance(ctx context.Context, instance *models.Instance) error { + if err := s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "stopped", instance.RuntimeGeneration, nil); err != nil { + return fmt.Errorf("failed to mark v2 instance stopped: %w", err) + } + now := time.Now() + instance.Status = "stopped" + instance.StoppedAt = &now + instance.PodName = nil + instance.PodNamespace = nil + instance.PodIP = nil + instance.UpdatedAt = now + GetHub().BroadcastInstanceStatus(instance.UserID, instance) + return s.cleanupV2GatewayBinding(ctx, instance) +} + +func (s *instanceService) deleteV2Instance(ctx context.Context, instance *models.Instance) error { + if instance.Status != "deleting" { + now := time.Now() + instance.Status = "deleting" + instance.UpdatedAt = now + if err := s.instanceRepo.Update(instance); err != nil { + return fmt.Errorf("failed to mark v2 instance as deleting: %w", err) + } + GetHub().BroadcastInstanceStatus(instance.UserID, instance) + } + + cleanupErr := s.cleanupV2GatewayBinding(ctx, instance) + if cleanupErr != nil { + return cleanupErr + } + if err := s.instanceRepo.Delete(instance.ID); err != nil { + return fmt.Errorf("failed to delete v2 instance record: %w", err) + } + return nil +} + +func (s *instanceService) cleanupV2GatewayBinding(ctx context.Context, instance *models.Instance) error { + if s.bindingRepo == nil { + return nil + } + binding, err := s.bindingRepo.GetByInstanceID(ctx, instance.ID) + if err != nil { + return fmt.Errorf("failed to get v2 runtime binding: %w", err) + } + if binding == nil { + return nil + } + + if s.runtimePodRepo != nil { + pod, podErr := s.runtimePodRepo.GetByID(ctx, binding.RuntimePodID) + if podErr != nil { + return fmt.Errorf("failed to get runtime pod %d for v2 cleanup: %w", binding.RuntimePodID, podErr) + } else if pod == nil { + return fmt.Errorf("runtime pod %d is not available for v2 cleanup", binding.RuntimePodID) + } else if pod != nil && pod.AgentEndpoint != nil && strings.TrimSpace(*pod.AgentEndpoint) != "" && s.agentClient != nil && binding.GatewayID != "" { + if err := s.agentClient.DeleteGateway(ctx, strings.TrimSpace(*pod.AgentEndpoint), binding.GatewayID); err != nil { + return fmt.Errorf("failed to delete v2 gateway: %w", err) + } + } + } + + if err := s.bindingRepo.DeleteByInstanceIDAndReleaseSlot(ctx, instance.ID, binding.RuntimePodID); err != nil { + return fmt.Errorf("failed to delete v2 runtime binding and release slot: %w", err) + } + return nil +} + +func (s *instanceService) ensureV2Workspace(ctx context.Context, instance *models.Instance, runtimeType string) error { + if instance.WorkspacePath != nil && strings.TrimSpace(*instance.WorkspacePath) != "" { + return nil + } + workspacePath, err := ensureRuntimeWorkspaceDirectories(s.runtimeWorkspaceRoot(), runtimeType, instance.UserID, instance.ID) + if err != nil { + return fmt.Errorf("failed to create instance workspace: %w", err) + } + if err := s.instanceRepo.SetWorkspacePath(ctx, instance.ID, workspacePath); err != nil { + return fmt.Errorf("failed to persist instance workspace path: %w", err) + } + instance.WorkspacePath = &workspacePath + return nil +} + +func ensureRuntimeWorkspaceDirectories(root, runtimeType string, userID, instanceID int) (string, error) { + workspacePath := RuntimeWorkspacePathWithRoot(root, runtimeType, userID, instanceID) + if err := os.MkdirAll(workspacePath, 0750); err != nil { + return "", err + } + + // Allow the isolated gateway UID to traverse to its own workspace without + // granting read/list access to sibling user or instance directories. + userRoot := path.Dir(workspacePath) + runtimeRoot := path.Dir(userRoot) + for _, dir := range []string{runtimeRoot, userRoot} { + if err := os.Chmod(dir, 0711); err != nil { + return "", err + } + } + if err := os.Chmod(workspacePath, 0750); err != nil { + return "", err + } + return workspacePath, nil +} + // Stop stops an instance func (s *instanceService) Stop(instanceID int) error { ctx := context.Background() @@ -977,13 +1274,29 @@ func (s *instanceService) Stop(instanceID int) error { return fmt.Errorf("instance not found") } + if _, ok := v2RuntimeTypeForInstance(instance); ok { + return s.stopV2Instance(ctx, instance) + } + if instance.Status != "running" { return fmt.Errorf("instance is not running") } - // Delete pod - if err := s.podService.DeletePod(ctx, instance.UserID, instance.ID); err != nil { - return fmt.Errorf("failed to delete pod: %w", err) + if instanceUsesDesktopRuntime(instance) { + if s.deploymentService == nil { + return fmt.Errorf("instance deployment service is not configured") + } + if err := s.deploymentService.ScaleDeployment(ctx, instance.UserID, instance.ID, 0); err != nil { + fmt.Printf("Warning: failed to stop deployment for instance %d, falling back to pod delete: %v\n", instance.ID, err) + if podErr := s.podService.DeletePod(ctx, instance.UserID, instance.ID); podErr != nil { + return fmt.Errorf("failed to stop deployment: %w", err) + } + } + } else { + // Delete shell pod + if err := s.podService.DeletePod(ctx, instance.UserID, instance.ID); err != nil { + return fmt.Errorf("failed to delete pod: %w", err) + } } // Update instance status @@ -1029,6 +1342,10 @@ func (s *instanceService) Delete(instanceID int) error { return fmt.Errorf("instance not found") } + if _, ok := v2RuntimeTypeForInstance(instance); ok { + return s.deleteV2Instance(context.Background(), instance) + } + if instance.Status != "deleting" { now := time.Now() instance.Status = "deleting" @@ -1300,6 +1617,18 @@ func (s *instanceService) GetInstanceStatus(instanceID int) (*InstanceStatus, er return nil, fmt.Errorf("instance not found") } + if runtimeType, ok := v2RuntimeTypeForInstance(instance); ok { + return &InstanceStatus{ + InstanceID: instance.ID, + Status: instance.Status, + Availability: s.v2InstanceAvailability(ctx, instance), + AgentType: runtimeType, + WorkspaceUsageBytes: instance.WorkspaceUsageBytes, + CreatedAt: instance.CreatedAt, + StartedAt: instance.StartedAt, + }, nil + } + status := &InstanceStatus{ InstanceID: instance.ID, Status: instance.Status, @@ -1334,6 +1663,13 @@ func (s *instanceService) ForceSyncInstance(instanceID int) error { return fmt.Errorf("instance not found") } + if _, ok := v2RuntimeTypeForInstance(instance); ok { + return nil + } + if instanceUsesDesktopRuntime(instance) { + return s.forceSyncDeploymentInstance(ctx, instance) + } + fmt.Printf("Force syncing instance %d (current status: %s, user: %d)\n", instanceID, instance.Status, instance.UserID) // First try direct lookup by instance ID @@ -1452,6 +1788,60 @@ func (s *instanceService) ForceSyncInstance(instanceID int) error { return nil } +func (s *instanceService) forceSyncDeploymentInstance(ctx context.Context, instance *models.Instance) error { + if s.deploymentService == nil { + return fmt.Errorf("instance deployment service is not configured") + } + deployment, err := s.deploymentService.GetDeployment(ctx, instance.UserID, instance.ID) + if err != nil { + if instance.Status == "running" || instance.Status == "creating" { + nextStatus := "stopped" + if instance.Status == "creating" { + nextStatus = "error" + } + instance.Status = nextStatus + instance.PodName = nil + instance.PodNamespace = nil + instance.PodIP = nil + instance.UpdatedAt = time.Now() + if err := s.instanceRepo.Update(instance); err != nil { + return fmt.Errorf("failed to update instance status: %w", err) + } + GetHub().BroadcastInstanceStatus(instance.UserID, instance) + } + return nil + } + + needsUpdate := false + desiredStatus := mapDeploymentToInstanceStatus(deployment) + if instance.Status != desiredStatus { + instance.Status = desiredStatus + needsUpdate = true + } + if pod, podErr := s.deploymentService.GetActivePod(ctx, instance.UserID, instance.ID); podErr == nil && pod != nil { + if pod.Status.PodIP != "" && (instance.PodIP == nil || *instance.PodIP != pod.Status.PodIP) { + instance.PodIP = &pod.Status.PodIP + needsUpdate = true + } + if instance.PodName == nil || *instance.PodName != pod.Name { + instance.PodName = &pod.Name + needsUpdate = true + } + if instance.PodNamespace == nil || *instance.PodNamespace != pod.Namespace { + instance.PodNamespace = &pod.Namespace + needsUpdate = true + } + } + if needsUpdate { + instance.UpdatedAt = time.Now() + if err := s.instanceRepo.Update(instance); err != nil { + return fmt.Errorf("failed to update instance: %w", err) + } + GetHub().BroadcastInstanceStatus(instance.UserID, instance) + } + return nil +} + func additionalServicePorts(primaryPort int32) []int32 { if primaryPort == 3000 || primaryPort == 8082 { return []int32{3000, 8082} @@ -1462,10 +1852,12 @@ func additionalServicePorts(primaryPort int32) []int32 { func normalizeInstanceRuntimeType(runtimeType string) string { switch strings.ToLower(strings.TrimSpace(runtimeType)) { + case RuntimeBackendGateway: + return RuntimeBackendGateway case "shell": - return "shell" + return RuntimeBackendShell default: - return "desktop" + return RuntimeBackendDesktop } } @@ -1473,5 +1865,182 @@ func instanceUsesDesktopRuntime(instance *models.Instance) bool { if instance == nil { return true } - return normalizeInstanceRuntimeType(instance.RuntimeType) == "desktop" + return normalizeInstanceRuntimeType(instance.RuntimeType) == RuntimeBackendDesktop +} + +func resolveCreateInstanceMode(req CreateInstanceRequest) string { + if mode, ok := NormalizeInstanceMode(req.Mode); ok { + return mode + } + if mode, ok := NormalizeInstanceMode(req.InstanceMode); ok { + return mode + } + if strings.TrimSpace(req.RuntimeType) == "" { + return InstanceModeLite + } + return InstanceModeForRuntimeType(normalizeInstanceRuntimeType(req.RuntimeType)) +} + +func hasExplicitCreateInstanceMode(req CreateInstanceRequest) bool { + if _, ok := NormalizeInstanceMode(req.Mode); ok { + return true + } + if _, ok := NormalizeInstanceMode(req.InstanceMode); ok { + return true + } + return false +} + +func modeForExistingInstance(instance *models.Instance) string { + if instance == nil { + return InstanceModeLite + } + if mode, ok := NormalizeInstanceMode(instance.InstanceMode); ok { + return mode + } + return InstanceModeForRuntimeType(normalizeInstanceRuntimeType(instance.RuntimeType)) +} + +func instanceModeUsesDedicatedResources(mode string) bool { + normalized, ok := NormalizeInstanceMode(mode) + return ok && normalized == InstanceModePro +} + +func (s *instanceService) enforceInstanceModeLimits(ctx context.Context, mode string, cpuCores float64, memoryGB, storageGB, gpuCount int) error { + normalizedMode, ok := NormalizeInstanceMode(mode) + if !ok { + return fmt.Errorf("unsupported instance mode %q", mode) + } + limits := loadInstanceModeLimitConfig(normalizedMode) + if limits.Capacity != nil { + if *limits.Capacity <= 0 { + return fmt.Errorf("%s instance mode is disabled", normalizedMode) + } + if s == nil || s.instanceRepo == nil { + return fmt.Errorf("instance repository is not configured") + } + count, err := s.instanceRepo.CountActiveByMode(ctx, normalizedMode) + if err != nil { + return err + } + if count >= *limits.Capacity { + return fmt.Errorf("%s instance capacity reached: %d/%d", normalizedMode, count, *limits.Capacity) + } + } + if !instanceModeUsesDedicatedResources(normalizedMode) { + return nil + } + if limits.MaxCPU != nil && cpuCores > *limits.MaxCPU { + return fmt.Errorf("%s CPU cores exceed mode limit: requested %g, max %g", normalizedMode, cpuCores, *limits.MaxCPU) + } + if limits.MaxMemoryGB != nil && memoryGB > *limits.MaxMemoryGB { + return fmt.Errorf("%s memory exceeds mode limit: requested %dGB, max %dGB", normalizedMode, memoryGB, *limits.MaxMemoryGB) + } + if limits.MaxStorageGB != nil && storageGB > *limits.MaxStorageGB { + return fmt.Errorf("%s storage exceeds mode limit: requested %dGB, max %dGB", normalizedMode, storageGB, *limits.MaxStorageGB) + } + if limits.MaxGPUCount != nil && gpuCount > *limits.MaxGPUCount { + return fmt.Errorf("%s GPU count exceeds mode limit: requested %d, max %d", normalizedMode, gpuCount, *limits.MaxGPUCount) + } + return nil +} + +func loadInstanceModeLimitConfig(mode string) instanceModeLimitConfig { + prefix := "CLAWMANAGER_" + strings.ToUpper(mode) + "_" + return instanceModeLimitConfig{ + Capacity: optionalIntEnv(prefix + "CAPACITY"), + MaxCPU: optionalFloatEnv(prefix + "MAX_CPU_CORES"), + MaxMemoryGB: optionalIntEnv(prefix + "MAX_MEMORY_GB"), + MaxStorageGB: optionalIntEnv(prefix + "MAX_STORAGE_GB"), + MaxGPUCount: optionalIntEnv(prefix + "MAX_GPU_COUNT"), + } +} + +func optionalIntEnv(key string) *int { + value := strings.TrimSpace(os.Getenv(key)) + if value == "" { + return nil + } + parsed, err := strconv.Atoi(value) + if err != nil { + return nil + } + return &parsed +} + +func optionalFloatEnv(key string) *float64 { + value := strings.TrimSpace(os.Getenv(key)) + if value == "" { + return nil + } + parsed, err := strconv.ParseFloat(value, 64) + if err != nil { + return nil + } + return &parsed +} + +func (s *instanceService) runtimeWorkspaceRoot() string { + if s != nil && strings.TrimSpace(s.workspaceRoot) != "" { + return strings.TrimSpace(s.workspaceRoot) + } + return "/workspaces" +} + +func v2RuntimeTypeForInstance(instance *models.Instance) (string, bool) { + if instance == nil { + return "", false + } + runtimeType, ok := NormalizeV2RuntimeType(instance.Type) + if !ok { + return "", false + } + if strings.EqualFold(strings.TrimSpace(instance.RuntimeType), RuntimeBackendGateway) { + return runtimeType, true + } + if mode, ok := NormalizeInstanceMode(instance.InstanceMode); ok && mode == InstanceModeLite { + return runtimeType, true + } + return "", false +} + +func availabilityForStatus(status string) string { + switch strings.ToLower(strings.TrimSpace(status)) { + case "running": + return "available" + case "creating": + return "starting" + default: + return "unavailable" + } +} + +func (s *instanceService) v2InstanceAvailability(ctx context.Context, instance *models.Instance) string { + base := availabilityForStatus(instance.Status) + if base != "available" { + return base + } + if s == nil || s.bindingRepo == nil || s.runtimePodRepo == nil { + return "unavailable" + } + binding, err := s.bindingRepo.GetRunningByInstanceID(ctx, instance.ID) + if err != nil || binding == nil || binding.GatewayPort <= 0 { + return "unavailable" + } + pod, err := s.runtimePodRepo.GetByID(ctx, binding.RuntimePodID) + if err != nil || pod == nil || pod.PodIP == nil || strings.TrimSpace(*pod.PodIP) == "" { + return "unavailable" + } + return "available" +} + +func trimOptionalString(value *string) *string { + if value == nil { + return nil + } + trimmed := strings.TrimSpace(*value) + if trimmed == "" { + return nil + } + return &trimmed } diff --git a/backend/internal/services/instance_service_test.go b/backend/internal/services/instance_service_test.go index a91656c..1064c0d 100644 --- a/backend/internal/services/instance_service_test.go +++ b/backend/internal/services/instance_service_test.go @@ -80,6 +80,79 @@ func TestBuildGatewayEnvInjectsGatewayModelCatalog(t *testing.T) { } } +func TestBuildGatewayEnvEnsuresMissingGatewayToken(t *testing.T) { + t.Setenv("CLAWMANAGER_LLM_GATEWAY_BASE_URL", "http://gateway.example/api/v1/gateway/llm") + + instanceRepo := &stubGatewayEnvInstanceRepository{fakeRuntimeInstanceRepo: newFakeRuntimeInstanceRepo()} + service := &instanceService{ + instanceRepo: instanceRepo, + llmModelRepo: &stubLLMModelRepository{ + active: []models.LLMModel{{DisplayName: "auto"}}, + }, + } + instance := &models.Instance{ + ID: 68, + UserID: 1, + Type: "openclaw", + } + + env, err := service.BuildGatewayEnv(instance) + if err != nil { + t.Fatalf("BuildGatewayEnv returned error: %v", err) + } + if instance.AccessToken == nil || *instance.AccessToken == "" { + t.Fatal("BuildGatewayEnv did not provision instance access token") + } + if env["CLAWMANAGER_LLM_API_KEY"] != *instance.AccessToken { + t.Fatalf("CLAWMANAGER_LLM_API_KEY = %q, want provisioned token %q", env["CLAWMANAGER_LLM_API_KEY"], *instance.AccessToken) + } + if got := instanceRepo.updated[68]; got == nil || got.AccessToken == nil || *got.AccessToken != *instance.AccessToken { + t.Fatalf("repository update did not persist provisioned token: %#v", instanceRepo.updated[68]) + } +} + +func TestBuildGatewayEnvMergesEnvironmentOverrides(t *testing.T) { + t.Setenv("CLAWMANAGER_LLM_GATEWAY_BASE_URL", "http://gateway.example/api/v1/gateway/llm") + token := "igt_test_token" + raw, err := marshalEnvironmentOverrides(map[string]string{ + "CLAWMANAGER_TEAM_ENABLED": "true", + "CLAWMANAGER_TEAM_MEMBER_ID": "lite-worker", + "CUSTOM_GATEWAY_ENV": "enabled", + }) + if err != nil { + t.Fatalf("marshalEnvironmentOverrides returned error: %v", err) + } + service := &instanceService{ + llmModelRepo: &stubLLMModelRepository{ + active: []models.LLMModel{{DisplayName: "auto"}}, + }, + } + + env, err := service.BuildGatewayEnv(&models.Instance{ + ID: 88, + Type: "openclaw", + RuntimeType: RuntimeBackendGateway, + AccessToken: &token, + EnvironmentOverridesJSON: raw, + }) + if err != nil { + t.Fatalf("BuildGatewayEnv returned error: %v", err) + } + + if env["CLAWMANAGER_TEAM_ENABLED"] != "true" || env["CLAWMANAGER_TEAM_MEMBER_ID"] != "lite-worker" { + t.Fatalf("expected Team environment overrides to be merged into gateway env, got %#v", env) + } + if env["CUSTOM_GATEWAY_ENV"] != "enabled" { + t.Fatalf("expected custom gateway environment override to be merged, got %#v", env) + } + if env["CLAWMANAGER_LLM_API_KEY"] != token { + t.Fatalf("expected gateway token env to remain available") + } + if env["CLAWMANAGER_RUNTIME_TYPE"] != RuntimeBackendGateway { + t.Fatalf("expected runtime type marker %q, got %q", RuntimeBackendGateway, env["CLAWMANAGER_RUNTIME_TYPE"]) + } +} + func TestBuildGatewayEnvSkipsUnmanagedRuntime(t *testing.T) { token := "igt_test_token" service := &instanceService{} @@ -96,6 +169,20 @@ func TestBuildGatewayEnvSkipsUnmanagedRuntime(t *testing.T) { } } +type stubGatewayEnvInstanceRepository struct { + *fakeRuntimeInstanceRepo + updated map[int]*models.Instance +} + +func (r *stubGatewayEnvInstanceRepository) Update(instance *models.Instance) error { + if r.updated == nil { + r.updated = map[int]*models.Instance{} + } + copy := *instance + r.updated[instance.ID] = © + return nil +} + func TestBuildAgentEnvInjectsHermesAgentConfig(t *testing.T) { t.Setenv("CLAWMANAGER_AGENT_CONTROL_BASE_URL", "http://agent-control.example") diff --git a/backend/internal/services/instance_service_v2_test.go b/backend/internal/services/instance_service_v2_test.go new file mode 100644 index 0000000..479790f --- /dev/null +++ b/backend/internal/services/instance_service_v2_test.go @@ -0,0 +1,751 @@ +package services + +import ( + "context" + "errors" + "os" + "path" + "strings" + "testing" + + "clawreef/internal/models" +) + +func TestInstanceServiceCreateV2CreatesWorkspaceOnly(t *testing.T) { + workspaceRoot := strings.ReplaceAll(t.TempDir(), "\\", "/") + instanceRepo := newV2LifecycleInstanceRepo() + service := &instanceService{ + instanceRepo: instanceRepo, + quotaRepo: v2LifecycleQuotaRepo{}, + workspaceRoot: workspaceRoot, + } + + instance, err := service.Create(45, CreateInstanceRequest{ + Name: "OpenClaw Dev", + Type: " OpenClaw ", + CPUCores: 2, + MemoryGB: 4, + DiskGB: 20, + OSType: "openclaw", + OSVersion: "latest", + }) + if err != nil { + t.Fatalf("Create returned error: %v", err) + } + + expectedWorkspacePath := RuntimeWorkspacePathWithRoot(workspaceRoot, "openclaw", 45, instance.ID) + if instance.Type != "openclaw" { + t.Fatalf("instance type = %q, want openclaw", instance.Type) + } + if instance.RuntimeType != "gateway" { + t.Fatalf("runtime type = %q, want gateway", instance.RuntimeType) + } + if instance.InstanceMode != InstanceModeLite { + t.Fatalf("instance mode = %q, want lite", instance.InstanceMode) + } + if instance.Status != "creating" { + t.Fatalf("status = %q, want creating", instance.Status) + } + if instance.RuntimeGeneration != 1 { + t.Fatalf("runtime generation = %d, want 1", instance.RuntimeGeneration) + } + if instance.WorkspacePath == nil || *instance.WorkspacePath != expectedWorkspacePath { + t.Fatalf("workspace path = %v, want %q", instance.WorkspacePath, expectedWorkspacePath) + } + if _, err := os.Stat(expectedWorkspacePath); err != nil { + t.Fatalf("expected workspace directory to exist: %v", err) + } + if os.PathSeparator == '/' { + assertDirMode(t, path.Dir(path.Dir(expectedWorkspacePath)), 0711) + assertDirMode(t, path.Dir(expectedWorkspacePath), 0711) + assertDirMode(t, expectedWorkspacePath, 0750) + } + if instance.PodName != nil || instance.PodNamespace != nil || instance.PodIP != nil { + t.Fatalf("V2 create must not set per-instance pod fields: %#v", instance) + } + if len(instanceRepo.created) != 1 { + t.Fatalf("created instance records = %d, want 1", len(instanceRepo.created)) + } +} + +func TestInstanceServiceCreateLiteSkipsPerInstanceResourceQuota(t *testing.T) { + workspaceRoot := strings.ReplaceAll(t.TempDir(), "\\", "/") + instanceRepo := newV2LifecycleInstanceRepo() + service := &instanceService{ + instanceRepo: instanceRepo, + quotaRepo: fixedV2QuotaRepo{cpu: 1, memory: 1, storage: 10, gpu: 0}, + workspaceRoot: workspaceRoot, + } + + instance, err := service.Create(45, CreateInstanceRequest{ + Name: "Lite Gateway", + Type: "openclaw", + Mode: InstanceModeLite, + RuntimeType: RuntimeBackendGateway, + CPUCores: 8, + MemoryGB: 32, + DiskGB: 200, + GPUEnabled: true, + GPUCount: 1, + OSType: "openclaw", + OSVersion: "latest", + StorageClass: "manual", + }) + if err != nil { + t.Fatalf("Create lite returned error: %v", err) + } + if instance.InstanceMode != InstanceModeLite || instance.RuntimeType != RuntimeBackendGateway { + t.Fatalf("created runtime = mode %q type %q, want lite gateway", instance.InstanceMode, instance.RuntimeType) + } +} + +func TestInstanceServiceCreateProEnforcesPerInstanceResourceQuota(t *testing.T) { + instanceRepo := newV2LifecycleInstanceRepo() + service := &instanceService{ + instanceRepo: instanceRepo, + quotaRepo: fixedV2QuotaRepo{cpu: 1, memory: 1, storage: 10, gpu: 0}, + pvcService: nil, + deploymentService: nil, + serviceService: nil, + networkPolicyService: nil, + openClawConfigService: nil, + } + + _, err := service.Create(45, CreateInstanceRequest{ + Name: "Pro Desktop", + Type: "openclaw", + Mode: InstanceModePro, + RuntimeType: RuntimeBackendDesktop, + CPUCores: 8, + MemoryGB: 32, + DiskGB: 200, + OSType: "openclaw", + OSVersion: "latest", + }) + if err == nil || !strings.Contains(err.Error(), "CPU cores exceed quota") { + t.Fatalf("Create pro error = %v, want CPU quota failure", err) + } +} + +func assertDirMode(t *testing.T, dir string, want os.FileMode) { + t.Helper() + info, err := os.Stat(dir) + if err != nil { + t.Fatalf("stat %s: %v", dir, err) + } + if got := info.Mode().Perm(); got != want { + t.Fatalf("mode %s = %04o, want %04o", dir, got, want) + } +} + +func TestInstanceServiceStartV2MarksCreatingWithNextGeneration(t *testing.T) { + workspacePath := "/workspaces/hermes/user-45/instance-77" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[77] = &models.Instance{ + ID: 77, + UserID: 45, + Type: "hermes", + RuntimeType: "gateway", + Status: "stopped", + WorkspacePath: &workspacePath, + RuntimeGeneration: 4, + } + service := &instanceService{instanceRepo: instanceRepo} + + if err := service.Start(77); err != nil { + t.Fatalf("Start returned error: %v", err) + } + + state := instanceRepo.runtimeStates[77] + if state.status != "creating" || state.generation != 5 { + t.Fatalf("runtime state = %#v, want creating generation 5", state) + } +} + +func TestInstanceServiceStopV2DeletesGatewayBindingAndReleasesSlot(t *testing.T) { + workspacePath := "/workspaces/openclaw/user-45/instance-88" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[88] = &models.Instance{ + ID: 88, + UserID: 45, + Type: "openclaw", + RuntimeType: "gateway", + Status: "running", + WorkspacePath: &workspacePath, + RuntimeGeneration: 3, + } + endpoint := "http://agent.local:19090" + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, AgentEndpoint: &endpoint}, + }, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[88] = &models.InstanceRuntimeBinding{ + InstanceID: 88, + RuntimePodID: 9, + GatewayID: "gw-88", + GatewayPort: 20018, + State: "running", + Generation: 3, + } + agent := &fakeRuntimeAgentClient{} + service := &instanceService{ + instanceRepo: instanceRepo, + runtimePodRepo: podRepo, + bindingRepo: bindingRepo, + agentClient: agent, + workspaceRoot: "/workspaces", + } + + if err := service.Stop(88); err != nil { + t.Fatalf("Stop returned error: %v", err) + } + + if len(agent.deleteRequests) != 1 || agent.deleteRequests[0].endpoint != endpoint || agent.deleteRequests[0].gatewayID != "gw-88" { + t.Fatalf("delete gateway requests = %#v", agent.deleteRequests) + } + if bindingRepo.deleteAndReleaseCalls[88] != 1 { + t.Fatalf("binding delete and release calls = %d, want 1", bindingRepo.deleteAndReleaseCalls[88]) + } + state := instanceRepo.runtimeStates[88] + if state.status != "stopped" || state.generation != 3 { + t.Fatalf("runtime state = %#v, want stopped generation 3", state) + } +} + +func TestInstanceServiceStopV2KeepsBindingWhenAgentDeleteFails(t *testing.T) { + workspacePath := "/workspaces/openclaw/user-45/instance-188" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[188] = &models.Instance{ + ID: 188, + UserID: 45, + Type: "openclaw", + RuntimeType: "gateway", + Status: "running", + WorkspacePath: &workspacePath, + RuntimeGeneration: 6, + } + endpoint := "http://agent.local:19090" + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 19: {ID: 19, AgentEndpoint: &endpoint}, + }, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[188] = &models.InstanceRuntimeBinding{ + InstanceID: 188, + RuntimePodID: 19, + GatewayID: "gw-188", + GatewayPort: 20018, + State: "running", + Generation: 6, + } + agent := &fakeRuntimeAgentClient{deleteErr: errors.New("agent delete failed")} + service := &instanceService{ + instanceRepo: instanceRepo, + runtimePodRepo: podRepo, + bindingRepo: bindingRepo, + agentClient: agent, + } + + err := service.Stop(188) + if err == nil || !strings.Contains(err.Error(), "failed to delete v2 gateway") { + t.Fatalf("Stop error = %v, want gateway delete failure", err) + } + if bindingRepo.bindings[188] == nil { + t.Fatal("binding was deleted after gateway delete failed") + } + if bindingRepo.deleteAndReleaseCalls[188] != 0 { + t.Fatalf("delete and release calls = %d, want 0", bindingRepo.deleteAndReleaseCalls[188]) + } + state := instanceRepo.runtimeStates[188] + if state.status != "stopped" || state.generation != 6 { + t.Fatalf("runtime state = %#v, want stopped generation 6", state) + } +} + +func TestInstanceServiceStopV2KeepsBindingWhenRuntimePodLookupFails(t *testing.T) { + workspacePath := "/workspaces/openclaw/user-45/instance-189" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[189] = &models.Instance{ + ID: 189, + UserID: 45, + Type: "openclaw", + RuntimeType: "gateway", + Status: "running", + WorkspacePath: &workspacePath, + RuntimeGeneration: 6, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[189] = &models.InstanceRuntimeBinding{ + InstanceID: 189, + RuntimePodID: 29, + GatewayID: "gw-189", + GatewayPort: 20019, + State: "running", + Generation: 6, + } + service := &instanceService{ + instanceRepo: instanceRepo, + runtimePodRepo: &fakeRuntimePodRepo{getErr: errors.New("pod lookup failed")}, + bindingRepo: bindingRepo, + agentClient: &fakeRuntimeAgentClient{}, + } + + err := service.Stop(189) + if err == nil || !strings.Contains(err.Error(), "failed to get runtime pod") { + t.Fatalf("Stop error = %v, want pod lookup failure", err) + } + if bindingRepo.bindings[189] == nil { + t.Fatal("binding was deleted after pod lookup failed") + } + if bindingRepo.deleteAndReleaseCalls[189] != 0 { + t.Fatalf("delete and release calls = %d, want 0", bindingRepo.deleteAndReleaseCalls[189]) + } + state := instanceRepo.runtimeStates[189] + if state.status != "stopped" || state.generation != 6 { + t.Fatalf("runtime state = %#v, want stopped generation 6", state) + } +} + +func TestInstanceServiceRestartV2RecreatesGatewayViaScheduler(t *testing.T) { + workspacePath := "/workspaces/openclaw/user-45/instance-89" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[89] = &models.Instance{ + ID: 89, + UserID: 45, + Type: "openclaw", + RuntimeType: "gateway", + Status: "running", + WorkspacePath: &workspacePath, + RuntimeGeneration: 8, + } + endpoint := "http://agent.local:19090" + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 11: {ID: 11, AgentEndpoint: &endpoint}, + }, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[89] = &models.InstanceRuntimeBinding{ + InstanceID: 89, + RuntimePodID: 11, + GatewayID: "gw-89", + GatewayPort: 20019, + State: "running", + Generation: 8, + } + agent := &fakeRuntimeAgentClient{} + service := &instanceService{ + instanceRepo: instanceRepo, + runtimePodRepo: podRepo, + bindingRepo: bindingRepo, + agentClient: agent, + } + + if err := service.Restart(89); err != nil { + t.Fatalf("Restart returned error: %v", err) + } + + if len(agent.deleteRequests) != 1 || agent.deleteRequests[0].gatewayID != "gw-89" { + t.Fatalf("delete gateway requests = %#v", agent.deleteRequests) + } + if bindingRepo.deleteAndReleaseCalls[89] != 1 { + t.Fatalf("binding delete and release calls = %d, want 1", bindingRepo.deleteAndReleaseCalls[89]) + } + state := instanceRepo.runtimeStates[89] + if state.status != "creating" || state.generation != 9 { + t.Fatalf("runtime state = %#v, want creating generation 9", state) + } +} + +func TestInstanceServiceDeleteV2MissingBindingStillDeletesInstance(t *testing.T) { + workspacePath := "/workspaces/hermes/user-45/instance-90" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[90] = &models.Instance{ + ID: 90, + UserID: 45, + Type: "hermes", + RuntimeType: "gateway", + Status: "stopped", + WorkspacePath: &workspacePath, + RuntimeGeneration: 2, + } + service := &instanceService{ + instanceRepo: instanceRepo, + runtimePodRepo: &fakeRuntimePodRepo{}, + bindingRepo: newFakeRuntimeBindingRepo(), + agentClient: &fakeRuntimeAgentClient{}, + } + + if err := service.Delete(90); err != nil { + t.Fatalf("Delete returned error: %v", err) + } + if len(instanceRepo.deleted) != 1 || instanceRepo.deleted[0] != 90 { + t.Fatalf("deleted instances = %#v, want [90]", instanceRepo.deleted) + } +} + +func TestInstanceServiceDeleteV2KeepsInstanceAndBindingWhenAgentDeleteFails(t *testing.T) { + workspacePath := "/workspaces/hermes/user-45/instance-190" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[190] = &models.Instance{ + ID: 190, + UserID: 45, + Type: "hermes", + RuntimeType: "gateway", + Status: "running", + WorkspacePath: &workspacePath, + RuntimeGeneration: 2, + } + endpoint := "http://agent.local:19090" + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 39: {ID: 39, AgentEndpoint: &endpoint}, + }, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[190] = &models.InstanceRuntimeBinding{ + InstanceID: 190, + RuntimePodID: 39, + GatewayID: "gw-190", + GatewayPort: 20090, + State: "running", + Generation: 2, + } + service := &instanceService{ + instanceRepo: instanceRepo, + runtimePodRepo: podRepo, + bindingRepo: bindingRepo, + agentClient: &fakeRuntimeAgentClient{deleteErr: errors.New("agent delete failed")}, + } + + err := service.Delete(190) + if err == nil || !strings.Contains(err.Error(), "failed to delete v2 gateway") { + t.Fatalf("Delete error = %v, want gateway delete failure", err) + } + if len(instanceRepo.deleted) != 0 { + t.Fatalf("deleted instances = %#v, want none", instanceRepo.deleted) + } + if instanceRepo.byID[190] == nil { + t.Fatal("instance was removed after cleanup failure") + } + if bindingRepo.bindings[190] == nil { + t.Fatal("binding was removed after cleanup failure") + } + if bindingRepo.deleteAndReleaseCalls[190] != 0 { + t.Fatalf("delete and release calls = %d, want 0", bindingRepo.deleteAndReleaseCalls[190]) + } +} + +func TestInstanceServiceV2StatusRequiresRunningBindingAndPodIP(t *testing.T) { + workspacePath := "/workspaces/openclaw/user-45/instance-92" + instanceRepo := newV2LifecycleInstanceRepo() + instanceRepo.byID[92] = &models.Instance{ + ID: 92, + UserID: 45, + Type: "openclaw", + RuntimeType: "gateway", + Status: "running", + WorkspacePath: &workspacePath, + WorkspaceUsageBytes: 2048, + RuntimeGeneration: 1, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[92] = &models.InstanceRuntimeBinding{ + InstanceID: 92, + RuntimePodID: 12, + GatewayPort: 20020, + State: "running", + } + service := &instanceService{ + instanceRepo: instanceRepo, + runtimePodRepo: &fakeRuntimePodRepo{pods: map[int64]*models.RuntimePod{12: {ID: 12}}}, + bindingRepo: bindingRepo, + } + + status, err := service.GetInstanceStatus(92) + if err != nil { + t.Fatalf("GetInstanceStatus returned error: %v", err) + } + if status.Availability != "unavailable" { + t.Fatalf("availability without pod IP = %q, want unavailable", status.Availability) + } + + podIP := "10.42.0.92" + service.runtimePodRepo = &fakeRuntimePodRepo{pods: map[int64]*models.RuntimePod{12: {ID: 12, PodIP: &podIP}}} + status, err = service.GetInstanceStatus(92) + if err != nil { + t.Fatalf("GetInstanceStatus returned error after pod IP: %v", err) + } + if status.Availability != "available" { + t.Fatalf("availability with running binding and pod IP = %q, want available", status.Availability) + } +} + +func TestSyncInstanceSkipsV2RuntimeGatewayInstances(t *testing.T) { + workspacePath := "/workspaces/openclaw/user-45/instance-91" + instanceRepo := newV2LifecycleInstanceRepo() + service := &SyncService{instanceRepo: instanceRepo} + instance := &models.Instance{ + ID: 91, + UserID: 45, + Type: "openclaw", + RuntimeType: "gateway", + InstanceMode: InstanceModeLite, + Status: "running", + WorkspacePath: &workspacePath, + } + + service.syncInstance(context.Background(), instance) + + if len(instanceRepo.updated) != 0 { + t.Fatalf("sync should not update V2 gateway instance via pod status, got %#v", instanceRepo.updated) + } +} + +func TestV2RuntimeTypeForInstanceRequiresLiteGatewayBoundary(t *testing.T) { + workspacePath := "/workspaces/openclaw/user-45/instance-191" + lite := &models.Instance{ + ID: 191, + Type: "openclaw", + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + WorkspacePath: &workspacePath, + } + if runtimeType, ok := v2RuntimeTypeForInstance(lite); !ok || runtimeType != RuntimeTypeOpenClaw { + t.Fatalf("lite gateway instance was not recognized: %q %v", runtimeType, ok) + } + + pro := &models.Instance{ + ID: 192, + Type: "openclaw", + RuntimeType: RuntimeBackendDesktop, + InstanceMode: InstanceModePro, + WorkspacePath: &workspacePath, + } + if runtimeType, ok := v2RuntimeTypeForInstance(pro); ok { + t.Fatalf("pro desktop instance must not be scheduler-managed, got %q", runtimeType) + } +} + +func TestInstanceModeCapacityCanDisablePro(t *testing.T) { + t.Setenv("CLAWMANAGER_PRO_CAPACITY", "0") + service := &instanceService{instanceRepo: newV2LifecycleInstanceRepo()} + + err := service.enforceInstanceModeLimits(context.Background(), InstanceModePro, 2, 4, 20, 0) + if err == nil || !strings.Contains(err.Error(), "pro instance mode is disabled") { + t.Fatalf("capacity error = %v, want pro disabled", err) + } +} + +func TestInstanceModeResourceLimitRejectsOversizedPro(t *testing.T) { + t.Setenv("CLAWMANAGER_PRO_MAX_CPU_CORES", "1.5") + service := &instanceService{instanceRepo: newV2LifecycleInstanceRepo()} + + err := service.enforceInstanceModeLimits(context.Background(), InstanceModePro, 2, 4, 20, 0) + if err == nil || !strings.Contains(err.Error(), "pro CPU cores exceed mode limit") { + t.Fatalf("resource limit error = %v, want pro CPU limit", err) + } +} + +func TestInstanceModeResourceLimitAllowsOversizedLite(t *testing.T) { + t.Setenv("CLAWMANAGER_LITE_MAX_CPU_CORES", "1.5") + service := &instanceService{instanceRepo: newV2LifecycleInstanceRepo()} + + err := service.enforceInstanceModeLimits(context.Background(), InstanceModeLite, 2, 4, 20, 0) + if err != nil { + t.Fatalf("resource limit error = %v, want lite to skip dedicated resource limits", err) + } +} + +type v2LifecycleInstanceRepo struct { + byID map[int]*models.Instance + created []models.Instance + updated []models.Instance + deleted []int + workspacePath map[int]string + runtimeStates map[int]v2RuntimeStateRecord + nextID int +} + +type v2RuntimeStateRecord struct { + status string + generation int + message *string +} + +func newV2LifecycleInstanceRepo() *v2LifecycleInstanceRepo { + return &v2LifecycleInstanceRepo{ + byID: map[int]*models.Instance{}, + workspacePath: map[int]string{}, + runtimeStates: map[int]v2RuntimeStateRecord{}, + nextID: 1, + } +} + +func (r *v2LifecycleInstanceRepo) Create(instance *models.Instance) error { + if instance.ID == 0 { + instance.ID = r.nextID + r.nextID++ + } + copy := *instance + r.byID[instance.ID] = © + r.created = append(r.created, copy) + return nil +} + +func (r *v2LifecycleInstanceRepo) GetByID(id int) (*models.Instance, error) { + return r.byID[id], nil +} + +func (r *v2LifecycleInstanceRepo) GetByAccessToken(string) (*models.Instance, error) { + return nil, nil +} + +func (r *v2LifecycleInstanceRepo) GetByAgentBootstrapToken(string) (*models.Instance, error) { + return nil, nil +} + +func (r *v2LifecycleInstanceRepo) GetAll(offset, limit int) ([]models.Instance, error) { + return nil, nil +} + +func (r *v2LifecycleInstanceRepo) CountAll() (int, error) { return len(r.byID), nil } + +func (r *v2LifecycleInstanceRepo) GetByUserID(userID int, offset, limit int) ([]models.Instance, error) { + instances := make([]models.Instance, 0) + for _, instance := range r.byID { + if instance.UserID == userID { + instances = append(instances, *instance) + } + } + return instances, nil +} + +func (r *v2LifecycleInstanceRepo) CountByUserID(userID int) (int, error) { + count := 0 + for _, instance := range r.byID { + if instance.UserID == userID { + count++ + } + } + return count, nil +} + +func (r *v2LifecycleInstanceRepo) CountActiveByMode(ctx context.Context, mode string) (int, error) { + count := 0 + for _, instance := range r.byID { + if instance.InstanceMode == mode && (instance.Status == "creating" || instance.Status == "running") { + count++ + } + } + return count, nil +} + +func (r *v2LifecycleInstanceRepo) ExistsByUserIDAndName(userID int, name string) (bool, error) { + normalized := strings.TrimSpace(strings.ToLower(name)) + for _, instance := range r.byID { + if instance.UserID == userID && strings.TrimSpace(strings.ToLower(instance.Name)) == normalized { + return true, nil + } + } + return false, nil +} + +func (r *v2LifecycleInstanceRepo) GetAllRunning() ([]models.Instance, error) { + instances := make([]models.Instance, 0, len(r.byID)) + for _, instance := range r.byID { + instances = append(instances, *instance) + } + return instances, nil +} + +func (r *v2LifecycleInstanceRepo) GetV2DesiredRunning(context.Context, int) ([]models.Instance, error) { + return nil, nil +} + +func (r *v2LifecycleInstanceRepo) GetV2Creating(context.Context, int) ([]models.Instance, error) { + return nil, nil +} + +func (r *v2LifecycleInstanceRepo) UpdateRuntimeState(ctx context.Context, id int, status string, generation int, message *string) error { + instance := r.byID[id] + if instance != nil { + instance.Status = status + instance.RuntimeGeneration = generation + instance.RuntimeErrorMessage = message + } + r.runtimeStates[id] = v2RuntimeStateRecord{status: status, generation: generation, message: message} + return nil +} + +func (r *v2LifecycleInstanceRepo) SetWorkspacePath(ctx context.Context, id int, workspacePath string) error { + instance := r.byID[id] + if instance != nil { + instance.WorkspacePath = &workspacePath + } + r.workspacePath[id] = workspacePath + return nil +} + +func (r *v2LifecycleInstanceRepo) UpdateWorkspaceUsage(context.Context, int, int64) error { + return nil +} + +func (r *v2LifecycleInstanceRepo) Update(instance *models.Instance) error { + copy := *instance + r.byID[instance.ID] = © + r.updated = append(r.updated, copy) + return nil +} + +func (r *v2LifecycleInstanceRepo) Delete(id int) error { + delete(r.byID, id) + r.deleted = append(r.deleted, id) + return nil +} + +type v2LifecycleQuotaRepo struct{} + +func (v2LifecycleQuotaRepo) Create(*models.UserQuota) error { return nil } +func (v2LifecycleQuotaRepo) GetByUserID(userID int) (*models.UserQuota, error) { + return &models.UserQuota{ + UserID: userID, + MaxInstances: 100, + MaxCPUCores: 100, + MaxMemoryGB: 1000, + MaxStorageGB: 1000, + MaxGPUCount: 8, + }, nil +} +func (v2LifecycleQuotaRepo) Update(*models.UserQuota) error { return nil } +func (v2LifecycleQuotaRepo) DeleteByUserID(int) error { return nil } +func (v2LifecycleQuotaRepo) CreateDefaultQuota(userID int) (*models.UserQuota, error) { + return v2LifecycleQuotaRepo{}.GetByUserID(userID) +} + +type fixedV2QuotaRepo struct { + cpu float64 + memory int + storage int + gpu int +} + +func (fixedV2QuotaRepo) Create(*models.UserQuota) error { return nil } +func (r fixedV2QuotaRepo) GetByUserID(userID int) (*models.UserQuota, error) { + return &models.UserQuota{ + UserID: userID, + MaxInstances: 100, + MaxCPUCores: r.cpu, + MaxMemoryGB: r.memory, + MaxStorageGB: r.storage, + MaxGPUCount: r.gpu, + }, nil +} +func (fixedV2QuotaRepo) Update(*models.UserQuota) error { return nil } +func (fixedV2QuotaRepo) DeleteByUserID(int) error { return nil } +func (r fixedV2QuotaRepo) CreateDefaultQuota(userID int) (*models.UserQuota, error) { + return r.GetByUserID(userID) +} diff --git a/backend/internal/services/instance_visibility_test.go b/backend/internal/services/instance_visibility_test.go index 7c9e289..7dd9981 100644 --- a/backend/internal/services/instance_visibility_test.go +++ b/backend/internal/services/instance_visibility_test.go @@ -1,159 +1,178 @@ -package services - -import ( - "testing" - - "clawreef/internal/models" -) - -// stubInstanceVisibilityRepo is a minimal InstanceRepository used by the -// visibility tests below. It only implements the read paths exercised by -// GetByUserID / GetAllInstances; every other method panics so that any -// accidental call surfaces as a test failure instead of a silent stub. -type stubInstanceVisibilityRepo struct { - all []models.Instance -} - -func (r *stubInstanceVisibilityRepo) byUser(userID int) []models.Instance { - out := make([]models.Instance, 0, len(r.all)) - for _, inst := range r.all { - if inst.UserID == userID { - out = append(out, inst) - } - } - return out -} - -func (r *stubInstanceVisibilityRepo) Create(*models.Instance) error { panic("not used") } -func (r *stubInstanceVisibilityRepo) GetByID(int) (*models.Instance, error) { - panic("not used") -} -func (r *stubInstanceVisibilityRepo) GetByAccessToken(string) (*models.Instance, error) { - panic("not used") -} -func (r *stubInstanceVisibilityRepo) GetByAgentBootstrapToken(string) (*models.Instance, error) { - panic("not used") -} -func (r *stubInstanceVisibilityRepo) GetAll(offset, limit int) ([]models.Instance, error) { - return paginate(r.all, offset, limit), nil -} -func (r *stubInstanceVisibilityRepo) CountAll() (int, error) { return len(r.all), nil } -func (r *stubInstanceVisibilityRepo) GetByUserID(userID, offset, limit int) ([]models.Instance, error) { - return paginate(r.byUser(userID), offset, limit), nil -} -func (r *stubInstanceVisibilityRepo) CountByUserID(userID int) (int, error) { - return len(r.byUser(userID)), nil -} -func (r *stubInstanceVisibilityRepo) ExistsByUserIDAndName(int, string) (bool, error) { - return false, nil -} -func (r *stubInstanceVisibilityRepo) GetAllRunning() ([]models.Instance, error) { - return nil, nil -} -func (r *stubInstanceVisibilityRepo) Update(*models.Instance) error { panic("not used") } -func (r *stubInstanceVisibilityRepo) Delete(int) error { panic("not used") } - -func paginate(items []models.Instance, offset, limit int) []models.Instance { - if offset >= len(items) { - return []models.Instance{} - } - end := offset + limit - if end > len(items) { - end = len(items) - } - out := make([]models.Instance, end-offset) - copy(out, items[offset:end]) - return out -} - -// TestGetByUserIDFiltersByCaller locks in the workspace-view contract: the -// caller-scoped listing must return only the caller's own instances, -// regardless of any role the caller may hold elsewhere. Regression guard -// for the admin-cross-user-leakage bug in which role=admin widened this -// endpoint to every user's instances. -func TestGetByUserIDFiltersByCaller(t *testing.T) { - t.Parallel() - - repo := &stubInstanceVisibilityRepo{ - all: []models.Instance{ - {ID: 1, UserID: 10, Name: "alice-1"}, - {ID: 2, UserID: 10, Name: "alice-2"}, - {ID: 3, UserID: 20, Name: "bob-1"}, - }, - } - svc := &instanceService{instanceRepo: repo} - - instances, total, err := svc.GetByUserID(10, 0, 50) - if err != nil { - t.Fatalf("GetByUserID returned error: %v", err) - } - if total != 2 { - t.Fatalf("expected total=2 for alice, got %d", total) - } - if len(instances) != 2 || instances[0].Name != "alice-1" || instances[1].Name != "alice-2" { - t.Fatalf("expected alice's instances only, got %+v", instances) - } - - instances, total, err = svc.GetByUserID(20, 0, 50) - if err != nil { - t.Fatalf("GetByUserID returned error: %v", err) - } - if total != 1 { - t.Fatalf("expected total=1 for bob, got %d", total) - } - if len(instances) != 1 || instances[0].Name != "bob-1" { - t.Fatalf("expected bob's instance only, got %+v", instances) - } - - // A user with no instances sees an empty list, never someone else's. - instances, total, err = svc.GetByUserID(99, 0, 50) - if err != nil { - t.Fatalf("GetByUserID returned error: %v", err) - } - if total != 0 || len(instances) != 0 { - t.Fatalf("expected empty listing for user 99, got total=%d instances=%+v", total, instances) - } -} - -// TestGetAllInstancesReturnsEveryUser verifies the admin-console surface: -// cross-user listing, with pagination, independent of caller identity (the -// admin middleware gates reachability at the route layer, not here). -func TestGetAllInstancesReturnsEveryUser(t *testing.T) { - t.Parallel() - - repo := &stubInstanceVisibilityRepo{ - all: []models.Instance{ - {ID: 1, UserID: 10, Name: "alice-1"}, - {ID: 2, UserID: 10, Name: "alice-2"}, - {ID: 3, UserID: 20, Name: "bob-1"}, - }, - } - svc := &instanceService{instanceRepo: repo} - - instances, total, err := svc.GetAllInstances(0, 50) - if err != nil { - t.Fatalf("GetAllInstances returned error: %v", err) - } - if total != 3 { - t.Fatalf("expected total=3 across users, got %d", total) - } - if len(instances) != 3 { - t.Fatalf("expected 3 instances, got %d", len(instances)) - } - - // Pagination still respected. - page1, _, err := svc.GetAllInstances(0, 2) - if err != nil { - t.Fatalf("GetAllInstances page1 error: %v", err) - } - if len(page1) != 2 { - t.Fatalf("expected page1 size=2, got %d", len(page1)) - } - page2, _, err := svc.GetAllInstances(2, 2) - if err != nil { - t.Fatalf("GetAllInstances page2 error: %v", err) - } - if len(page2) != 1 || page2[0].Name != "bob-1" { - t.Fatalf("expected page2=[bob-1], got %+v", page2) - } -} +package services + +import ( + "context" + "testing" + + "clawreef/internal/models" +) + +// stubInstanceVisibilityRepo is a minimal InstanceRepository used by the +// visibility tests below. It only implements the read paths exercised by +// GetByUserID / GetAllInstances; every other method panics so that any +// accidental call surfaces as a test failure instead of a silent stub. +type stubInstanceVisibilityRepo struct { + all []models.Instance +} + +func (r *stubInstanceVisibilityRepo) byUser(userID int) []models.Instance { + out := make([]models.Instance, 0, len(r.all)) + for _, inst := range r.all { + if inst.UserID == userID { + out = append(out, inst) + } + } + return out +} + +func (r *stubInstanceVisibilityRepo) Create(*models.Instance) error { panic("not used") } +func (r *stubInstanceVisibilityRepo) GetByID(int) (*models.Instance, error) { + panic("not used") +} +func (r *stubInstanceVisibilityRepo) GetByAccessToken(string) (*models.Instance, error) { + panic("not used") +} +func (r *stubInstanceVisibilityRepo) GetByAgentBootstrapToken(string) (*models.Instance, error) { + panic("not used") +} +func (r *stubInstanceVisibilityRepo) GetAll(offset, limit int) ([]models.Instance, error) { + return paginate(r.all, offset, limit), nil +} +func (r *stubInstanceVisibilityRepo) CountAll() (int, error) { return len(r.all), nil } +func (r *stubInstanceVisibilityRepo) GetByUserID(userID, offset, limit int) ([]models.Instance, error) { + return paginate(r.byUser(userID), offset, limit), nil +} +func (r *stubInstanceVisibilityRepo) CountByUserID(userID int) (int, error) { + return len(r.byUser(userID)), nil +} +func (r *stubInstanceVisibilityRepo) CountActiveByMode(context.Context, string) (int, error) { + return 0, nil +} +func (r *stubInstanceVisibilityRepo) ExistsByUserIDAndName(int, string) (bool, error) { + return false, nil +} +func (r *stubInstanceVisibilityRepo) GetAllRunning() ([]models.Instance, error) { + return nil, nil +} +func (r *stubInstanceVisibilityRepo) GetV2DesiredRunning(context.Context, int) ([]models.Instance, error) { + panic("not used") +} +func (r *stubInstanceVisibilityRepo) GetV2Creating(context.Context, int) ([]models.Instance, error) { + panic("not used") +} +func (r *stubInstanceVisibilityRepo) UpdateRuntimeState(context.Context, int, string, int, *string) error { + panic("not used") +} +func (r *stubInstanceVisibilityRepo) SetWorkspacePath(context.Context, int, string) error { + panic("not used") +} +func (r *stubInstanceVisibilityRepo) UpdateWorkspaceUsage(context.Context, int, int64) error { + panic("not used") +} +func (r *stubInstanceVisibilityRepo) Update(*models.Instance) error { panic("not used") } +func (r *stubInstanceVisibilityRepo) Delete(int) error { panic("not used") } + +func paginate(items []models.Instance, offset, limit int) []models.Instance { + if offset >= len(items) { + return []models.Instance{} + } + end := offset + limit + if end > len(items) { + end = len(items) + } + out := make([]models.Instance, end-offset) + copy(out, items[offset:end]) + return out +} + +// TestGetByUserIDFiltersByCaller locks in the workspace-view contract: the +// caller-scoped listing must return only the caller's own instances, +// regardless of any role the caller may hold elsewhere. Regression guard +// for the admin-cross-user-leakage bug in which role=admin widened this +// endpoint to every user's instances. +func TestGetByUserIDFiltersByCaller(t *testing.T) { + t.Parallel() + + repo := &stubInstanceVisibilityRepo{ + all: []models.Instance{ + {ID: 1, UserID: 10, Name: "alice-1"}, + {ID: 2, UserID: 10, Name: "alice-2"}, + {ID: 3, UserID: 20, Name: "bob-1"}, + }, + } + svc := &instanceService{instanceRepo: repo} + + instances, total, err := svc.GetByUserID(10, 0, 50) + if err != nil { + t.Fatalf("GetByUserID returned error: %v", err) + } + if total != 2 { + t.Fatalf("expected total=2 for alice, got %d", total) + } + if len(instances) != 2 || instances[0].Name != "alice-1" || instances[1].Name != "alice-2" { + t.Fatalf("expected alice's instances only, got %+v", instances) + } + + instances, total, err = svc.GetByUserID(20, 0, 50) + if err != nil { + t.Fatalf("GetByUserID returned error: %v", err) + } + if total != 1 { + t.Fatalf("expected total=1 for bob, got %d", total) + } + if len(instances) != 1 || instances[0].Name != "bob-1" { + t.Fatalf("expected bob's instance only, got %+v", instances) + } + + // A user with no instances sees an empty list, never someone else's. + instances, total, err = svc.GetByUserID(99, 0, 50) + if err != nil { + t.Fatalf("GetByUserID returned error: %v", err) + } + if total != 0 || len(instances) != 0 { + t.Fatalf("expected empty listing for user 99, got total=%d instances=%+v", total, instances) + } +} + +// TestGetAllInstancesReturnsEveryUser verifies the admin-console surface: +// cross-user listing, with pagination, independent of caller identity (the +// admin middleware gates reachability at the route layer, not here). +func TestGetAllInstancesReturnsEveryUser(t *testing.T) { + t.Parallel() + + repo := &stubInstanceVisibilityRepo{ + all: []models.Instance{ + {ID: 1, UserID: 10, Name: "alice-1"}, + {ID: 2, UserID: 10, Name: "alice-2"}, + {ID: 3, UserID: 20, Name: "bob-1"}, + }, + } + svc := &instanceService{instanceRepo: repo} + + instances, total, err := svc.GetAllInstances(0, 50) + if err != nil { + t.Fatalf("GetAllInstances returned error: %v", err) + } + if total != 3 { + t.Fatalf("expected total=3 across users, got %d", total) + } + if len(instances) != 3 { + t.Fatalf("expected 3 instances, got %d", len(instances)) + } + + // Pagination still respected. + page1, _, err := svc.GetAllInstances(0, 2) + if err != nil { + t.Fatalf("GetAllInstances page1 error: %v", err) + } + if len(page1) != 2 { + t.Fatalf("expected page1 size=2, got %d", len(page1)) + } + page2, _, err := svc.GetAllInstances(2, 2) + if err != nil { + t.Fatalf("GetAllInstances page2 error: %v", err) + } + if len(page2) != 1 || page2[0].Name != "bob-1" { + t.Fatalf("expected page2=[bob-1], got %+v", page2) + } +} diff --git a/backend/internal/services/k8s/client.go b/backend/internal/services/k8s/client.go index 8c8bd20..23f41cd 100644 --- a/backend/internal/services/k8s/client.go +++ b/backend/internal/services/k8s/client.go @@ -29,12 +29,15 @@ const ( // Client wraps the Kubernetes client type Client struct { - Clientset kubernetes.Interface - Config *rest.Config - Namespace string - StorageClass string - HostPathPrefix string - Mode ConnectionMode + Clientset kubernetes.Interface + Config *rest.Config + Namespace string + StorageClass string + HostPathPrefix string + WorkspaceRoot string + WorkspaceNFSServer string + WorkspaceNFSPath string + Mode ConnectionMode } var ( @@ -97,12 +100,15 @@ func Initialize(cfg *config.Config) error { } globalClient = &Client{ - Clientset: clientset, - Config: restConfig, - Namespace: cfg.GetNamespace(), - StorageClass: cfg.GetStorageClass(), - HostPathPrefix: cfg.GetHostPathPrefix(), - Mode: detectedMode, + Clientset: clientset, + Config: restConfig, + Namespace: cfg.GetNamespace(), + StorageClass: cfg.GetStorageClass(), + HostPathPrefix: cfg.GetHostPathPrefix(), + WorkspaceRoot: cfg.Runtime.WorkspaceRoot, + WorkspaceNFSServer: cfg.Runtime.WorkspaceNFSServer, + WorkspaceNFSPath: cfg.Runtime.WorkspaceNFSPath, + Mode: detectedMode, } return nil diff --git a/backend/internal/services/k8s/instance_deployment_service.go b/backend/internal/services/k8s/instance_deployment_service.go new file mode 100644 index 0000000..12186bc --- /dev/null +++ b/backend/internal/services/k8s/instance_deployment_service.go @@ -0,0 +1,376 @@ +package k8s + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +const instanceDeploymentRestartedAtAnnotation = "clawmanager.io/restarted-at" + +type InstanceDeploymentService struct { + client *Client + namespaceService *NamespaceService +} + +func NewInstanceDeploymentService() *InstanceDeploymentService { + return &InstanceDeploymentService{ + client: globalClient, + namespaceService: NewNamespaceService(), + } +} + +func InstanceDeploymentName(instanceID int) string { + return sanitizeK8sName(fmt.Sprintf("clawreef-%d-deployment", instanceID)) +} + +func (s *InstanceDeploymentService) EnsureDeployment(ctx context.Context, config PodConfig, replicas int32) (*appsv1.Deployment, error) { + if s.client == nil { + return nil, fmt.Errorf("k8s client not initialized") + } + if _, err := s.namespaceService.EnsureNamespace(ctx, config.UserID); err != nil { + return nil, fmt.Errorf("failed to ensure namespace: %w", err) + } + + desired := BuildInstanceDeployment(s.client, config, replicas) + deployments := s.client.Clientset.AppsV1().Deployments(desired.Namespace) + existing, err := deployments.Get(ctx, desired.Name, metav1.GetOptions{}) + if errors.IsNotFound(err) { + created, createErr := deployments.Create(ctx, desired, metav1.CreateOptions{}) + if createErr != nil { + return nil, fmt.Errorf("failed to create instance deployment %s/%s: %w", desired.Namespace, desired.Name, createErr) + } + return created, nil + } + if err != nil { + return nil, fmt.Errorf("failed to get instance deployment %s/%s: %w", desired.Namespace, desired.Name, err) + } + if !reflect.DeepEqual(existing.Spec.Selector, desired.Spec.Selector) { + return nil, fmt.Errorf("instance deployment %s/%s selector mismatch; delete and recreate the deployment to change immutable selector", desired.Namespace, desired.Name) + } + + updated := existing.DeepCopy() + if updated.Labels == nil { + updated.Labels = map[string]string{} + } + for key, value := range desired.Labels { + updated.Labels[key] = value + } + updated.Spec.Replicas = desired.Spec.Replicas + updated.Spec.Template = desired.Spec.Template + + result, updateErr := deployments.Update(ctx, updated, metav1.UpdateOptions{}) + if updateErr != nil { + return nil, fmt.Errorf("failed to update instance deployment %s/%s: %w", desired.Namespace, desired.Name, updateErr) + } + return result, nil +} + +func (s *InstanceDeploymentService) ScaleDeployment(ctx context.Context, userID, instanceID int, replicas int32) error { + if s.client == nil { + return fmt.Errorf("k8s client not initialized") + } + namespace := s.client.GetNamespace(userID) + name := InstanceDeploymentName(instanceID) + patch, err := json.Marshal(map[string]any{ + "spec": map[string]any{"replicas": replicas}, + }) + if err != nil { + return fmt.Errorf("failed to build instance deployment scale patch: %w", err) + } + if _, err := s.client.Clientset.AppsV1().Deployments(namespace).Patch(ctx, name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("failed to scale instance deployment %s/%s: %w", namespace, name, err) + } + return nil +} + +func (s *InstanceDeploymentService) RestartDeployment(ctx context.Context, userID, instanceID int) error { + if s.client == nil { + return fmt.Errorf("k8s client not initialized") + } + namespace := s.client.GetNamespace(userID) + name := InstanceDeploymentName(instanceID) + patch, err := json.Marshal(map[string]any{ + "spec": map[string]any{ + "template": map[string]any{ + "metadata": map[string]any{ + "annotations": map[string]string{ + instanceDeploymentRestartedAtAnnotation: time.Now().UTC().Format(time.RFC3339Nano), + }, + }, + }, + }, + }) + if err != nil { + return fmt.Errorf("failed to build instance deployment restart patch: %w", err) + } + if _, err := s.client.Clientset.AppsV1().Deployments(namespace).Patch(ctx, name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("failed to restart instance deployment %s/%s: %w", namespace, name, err) + } + return nil +} + +func (s *InstanceDeploymentService) DeleteDeployment(ctx context.Context, userID, instanceID int) error { + if s.client == nil { + return fmt.Errorf("k8s client not initialized") + } + namespace := s.client.GetNamespace(userID) + name := InstanceDeploymentName(instanceID) + err := s.client.Clientset.AppsV1().Deployments(namespace).Delete(ctx, name, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete instance deployment %s/%s: %w", namespace, name, err) + } + return nil +} + +func (s *InstanceDeploymentService) GetDeployment(ctx context.Context, userID, instanceID int) (*appsv1.Deployment, error) { + if s.client == nil { + return nil, fmt.Errorf("k8s client not initialized") + } + namespace := s.client.GetNamespace(userID) + name := InstanceDeploymentName(instanceID) + deployment, err := s.client.Clientset.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get instance deployment %s/%s: %w", namespace, name, err) + } + return deployment, nil +} + +func (s *InstanceDeploymentService) GetActivePod(ctx context.Context, userID, instanceID int) (*corev1.Pod, error) { + if s.client == nil { + return nil, fmt.Errorf("k8s client not initialized") + } + namespace := s.client.GetNamespace(userID) + selector := fmt.Sprintf("instance-id=%d,app=clawreef", instanceID) + pods, err := s.client.Clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return nil, fmt.Errorf("failed to list instance deployment pods: %w", err) + } + if len(pods.Items) == 0 { + return nil, fmt.Errorf("pod not found for instance %d", instanceID) + } + for _, pod := range pods.Items { + if pod.DeletionTimestamp == nil && pod.Status.Phase == corev1.PodRunning && podReady(&pod) { + selected := pod + return &selected, nil + } + } + selected := pods.Items[0] + return &selected, nil +} + +func BuildInstanceDeployment(client *Client, config PodConfig, replicas int32) *appsv1.Deployment { + namespace := client.GetNamespace(config.UserID) + name := InstanceDeploymentName(config.InstanceID) + runtimeType := normalizePodRuntimeType(config.RuntimeType) + labels := map[string]string{ + "app": "clawreef", + "instance-id": fmt.Sprintf("%d", config.InstanceID), + "instance-name": config.InstanceName, + "user-id": fmt.Sprintf("%d", config.UserID), + "instance-type": config.Type, + "runtime-type": runtimeType, + "managed-by": "clawreef", + } + + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: copyStringMap(labels), + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "clawreef", + "instance-id": fmt.Sprintf("%d", config.InstanceID), + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: instanceDeploymentAnnotations(config), + Labels: labels, + }, + Spec: buildInstanceDeploymentPodSpec(client, config, runtimeType), + }, + }, + } +} + +func instanceDeploymentAnnotations(config PodConfig) map[string]string { + if config.SecurityMode == PodSecurityChromiumCompat { + return map[string]string{"container.apparmor.security.beta.kubernetes.io/desktop": "unconfined"} + } + return nil +} + +func buildInstanceDeploymentPodSpec(client *Client, config PodConfig, runtimeType string) corev1.PodSpec { + pvcName := client.GetPVCName(config.InstanceID) + if config.ContainerPort == 0 { + config.ContainerPort = 3001 + } + pullPolicy := config.ImagePullPolicy + if pullPolicy == "" { + pullPolicy = corev1.PullIfNotPresent + } + + resources := corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(fmt.Sprintf("%g", config.CPUCores)), + corev1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", config.MemoryGB)), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(fmt.Sprintf("%g", config.CPUCores)), + corev1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", config.MemoryGB)), + }, + } + if config.GPUEnabled && config.GPUCount > 0 { + resources.Limits["nvidia.com/gpu"] = resource.MustParse(fmt.Sprintf("%d", config.GPUCount)) + resources.Requests["nvidia.com/gpu"] = resource.MustParse(fmt.Sprintf("%d", config.GPUCount)) + } + + container := corev1.Container{ + Name: "desktop", + Image: config.Image, + ImagePullPolicy: pullPolicy, + Ports: []corev1.ContainerPort{{ + ContainerPort: config.ContainerPort, + Name: "http", + }}, + StartupProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstrFromInt32(config.ContainerPort)}}, + FailureThreshold: 30, + PeriodSeconds: 5, + TimeoutSeconds: 2, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstrFromInt32(config.ContainerPort)}}, + InitialDelaySeconds: 3, + PeriodSeconds: 5, + TimeoutSeconds: 2, + FailureThreshold: 6, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstrFromInt32(config.ContainerPort)}}, + InitialDelaySeconds: 15, + PeriodSeconds: 10, + TimeoutSeconds: 2, + FailureThreshold: 3, + }, + SecurityContext: buildContainerSecurityContext(config.SecurityMode), + Resources: resources, + VolumeMounts: []corev1.VolumeMount{{ + Name: "data", + MountPath: config.MountPath, + }}, + Env: []corev1.EnvVar{ + {Name: "INSTANCE_ID", Value: fmt.Sprintf("%d", config.InstanceID)}, + {Name: "USER_ID", Value: fmt.Sprintf("%d", config.UserID)}, + }, + } + if runtimeType == "shell" { + container.Ports = nil + container.StartupProbe = nil + container.ReadinessProbe = nil + container.LivenessProbe = nil + } + for key, value := range config.ExtraEnv { + container.Env = append(container.Env, corev1.EnvVar{Name: key, Value: value}) + } + for _, secretName := range config.EnvFromSecretNames { + if secretName == "" { + continue + } + container.EnvFrom = append(container.EnvFrom, corev1.EnvFromSource{ + SecretRef: &corev1.SecretEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: secretName}}, + }) + } + + spec := corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyAlways, + SecurityContext: buildPodSecurityContext(config.FSGroup), + Containers: []corev1.Container{container}, + Volumes: []corev1.Volume{{ + Name: "data", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvcName}, + }, + }}, + } + + for _, mount := range config.ExtraPVCMounts { + if mount.Name == "" || mount.ClaimName == "" || mount.MountPath == "" { + continue + } + spec.Volumes = append(spec.Volumes, corev1.Volume{ + Name: mount.Name, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: mount.ClaimName, ReadOnly: mount.ReadOnly}, + }, + }) + spec.Containers[0].VolumeMounts = append(spec.Containers[0].VolumeMounts, corev1.VolumeMount{ + Name: mount.Name, + MountPath: mount.MountPath, + ReadOnly: mount.ReadOnly, + }) + } + + for index, fix := range config.VolumeOwnershipFixes { + if fix.Name == "" || fix.MountPath == "" || fix.UID < 0 || fix.GID < 0 { + continue + } + spec.InitContainers = append(spec.InitContainers, buildVolumeOwnershipInitContainer(index, config.Image, pullPolicy, fix)) + } + + for _, mount := range config.ConfigMapFileMounts { + if mount.Name == "" || mount.ConfigMapName == "" || mount.Key == "" || mount.MountPath == "" { + continue + } + spec.Volumes = append(spec.Volumes, corev1.Volume{ + Name: mount.Name, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: mount.ConfigMapName}, + Items: []corev1.KeyToPath{{Key: mount.Key, Path: mount.Key}}, + }, + }, + }) + volumeMount := corev1.VolumeMount{Name: mount.Name, MountPath: mount.MountPath, ReadOnly: true} + if !mount.AsDirectory { + volumeMount.SubPath = mount.Key + } + spec.Containers[0].VolumeMounts = append(spec.Containers[0].VolumeMounts, volumeMount) + } + + if config.SHMSizeGB > 0 { + shmLimit := resource.MustParse(fmt.Sprintf("%dGi", config.SHMSizeGB)) + spec.Volumes = append(spec.Volumes, corev1.Volume{ + Name: "shm", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory, SizeLimit: &shmLimit}, + }, + }) + spec.Containers[0].VolumeMounts = append(spec.Containers[0].VolumeMounts, corev1.VolumeMount{Name: "shm", MountPath: "/dev/shm"}) + } + + return spec +} + +func podReady(pod *corev1.Pod) bool { + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady { + return condition.Status == corev1.ConditionTrue + } + } + return false +} diff --git a/backend/internal/services/k8s/instance_deployment_service_test.go b/backend/internal/services/k8s/instance_deployment_service_test.go new file mode 100644 index 0000000..743b3d7 --- /dev/null +++ b/backend/internal/services/k8s/instance_deployment_service_test.go @@ -0,0 +1,99 @@ +package k8s + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +func TestBuildInstanceDeploymentUsesStableIdentityAndPVC(t *testing.T) { + client := &Client{Clientset: fake.NewSimpleClientset(), Namespace: "clawreef"} + deployment := BuildInstanceDeployment(client, PodConfig{ + InstanceID: 42, + InstanceName: "Pro Desktop", + UserID: 7, + Type: "openclaw", + RuntimeType: "desktop", + CPUCores: 2, + MemoryGB: 4, + Image: "registry/openclaw:pro", + MountPath: "/config", + ContainerPort: 3001, + ImagePullPolicy: corev1.PullIfNotPresent, + }, 1) + + if deployment.Name != "clawreef-42-deployment" { + t.Fatalf("deployment name = %q, want stable instance deployment name", deployment.Name) + } + if deployment.Namespace != "clawreef-user-7" { + t.Fatalf("namespace = %q, want user namespace", deployment.Namespace) + } + if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != 1 { + t.Fatalf("replicas = %#v, want 1", deployment.Spec.Replicas) + } + if got := deployment.Spec.Selector.MatchLabels["instance-id"]; got != "42" { + t.Fatalf("selector instance-id = %q, want 42", got) + } + template := deployment.Spec.Template + if got := template.Labels["runtime-type"]; got != "desktop" { + t.Fatalf("template runtime-type = %q, want desktop", got) + } + container := template.Spec.Containers[0] + if container.Name != "desktop" || container.Image != "registry/openclaw:pro" { + t.Fatalf("unexpected container: %#v", container) + } + if template.Spec.RestartPolicy != corev1.RestartPolicyAlways { + t.Fatalf("restart policy = %q, want Always", template.Spec.RestartPolicy) + } + if len(template.Spec.Volumes) == 0 || template.Spec.Volumes[0].PersistentVolumeClaim == nil { + t.Fatalf("expected PVC data volume, got %#v", template.Spec.Volumes) + } + if got := template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName; got != "clawreef-42-pvc" { + t.Fatalf("PVC name = %q, want clawreef-42-pvc", got) + } +} + +func TestInstanceDeploymentServiceEnsureAndScale(t *testing.T) { + client := &Client{Clientset: fake.NewSimpleClientset(), Namespace: "clawreef"} + service := &InstanceDeploymentService{ + client: client, + namespaceService: &NamespaceService{client: client}, + } + config := PodConfig{ + InstanceID: 43, + InstanceName: "Pro Desktop", + UserID: 8, + Type: "openclaw", + RuntimeType: "desktop", + CPUCores: 2, + MemoryGB: 4, + Image: "registry/openclaw:v1", + MountPath: "/config", + ContainerPort: 3001, + } + + if _, err := service.EnsureDeployment(context.Background(), config, 1); err != nil { + t.Fatalf("EnsureDeployment create returned error: %v", err) + } + deployment, err := client.Clientset.AppsV1().Deployments("clawreef-user-8").Get(context.Background(), "clawreef-43-deployment", metav1.GetOptions{}) + if err != nil { + t.Fatalf("get deployment: %v", err) + } + if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != 1 { + t.Fatalf("created replicas = %#v, want 1", deployment.Spec.Replicas) + } + + if err := service.ScaleDeployment(context.Background(), 8, 43, 0); err != nil { + t.Fatalf("ScaleDeployment returned error: %v", err) + } + scaled, err := client.Clientset.AppsV1().Deployments("clawreef-user-8").Get(context.Background(), "clawreef-43-deployment", metav1.GetOptions{}) + if err != nil { + t.Fatalf("get scaled deployment: %v", err) + } + if scaled.Spec.Replicas == nil || *scaled.Spec.Replicas != 0 { + t.Fatalf("scaled replicas = %#v, want 0", scaled.Spec.Replicas) + } +} diff --git a/backend/internal/services/k8s/pvc_service.go b/backend/internal/services/k8s/pvc_service.go index 9f8ec0e..ac115b7 100644 --- a/backend/internal/services/k8s/pvc_service.go +++ b/backend/internal/services/k8s/pvc_service.go @@ -3,6 +3,10 @@ package k8s import ( "context" "fmt" + "net" + "os" + "path" + "path/filepath" "sort" "strings" "time" @@ -136,6 +140,12 @@ func (s *PVCService) CreateTeamSharedPVC(ctx context.Context, userID, teamID, st storageClass = s.client.StorageClass } storageSize := resource.MustParse(fmt.Sprintf("%dGi", storageSizeGB)) + useWorkspaceNFS := strings.TrimSpace(s.client.WorkspaceNFSServer) != "" + if useWorkspaceNFS { + if err := s.ensureTeamSharedWorkspaceDirectory(userID, teamID); err != nil { + return nil, err + } + } pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ @@ -165,6 +175,11 @@ func (s *PVCService) CreateTeamSharedPVC(ctx context.Context, userID, teamID, st if errors.IsAlreadyExists(err) { existingPVC, getErr := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) if getErr == nil && existingPVC != nil && existingPVC.Labels["team-id"] == fmt.Sprintf("%d", teamID) { + if useWorkspaceNFS { + if err := s.ensureWorkspaceNFSPVForTeamSharedPVC(ctx, namespace, pvcName, existingPVC, userID, teamID, storageSizeGB, storageClass, fmt.Sprintf("clawreef-pv-user-%d-team-%d-shared", userID, teamID)); err != nil { + return nil, err + } + } if existingPVC.Status.Phase != corev1.ClaimBound { go s.monitorTeamSharedPVCBinding(context.Background(), namespace, pvcName, userID, teamID, storageSizeGB, storageClass, 15*time.Second) } @@ -174,6 +189,11 @@ func (s *PVCService) CreateTeamSharedPVC(ctx context.Context, userID, teamID, st return nil, fmt.Errorf("failed to create Team shared PVC %s: %w", pvcName, err) } + if useWorkspaceNFS { + if err := s.ensureWorkspaceNFSPVForTeamSharedPVC(ctx, namespace, pvcName, createdPVC, userID, teamID, storageSizeGB, storageClass, fmt.Sprintf("clawreef-pv-user-%d-team-%d-shared", userID, teamID)); err != nil { + return nil, err + } + } go s.monitorTeamSharedPVCBinding(context.Background(), namespace, pvcName, userID, teamID, storageSizeGB, storageClass, 15*time.Second) return createdPVC, nil } @@ -218,11 +238,6 @@ func (s *PVCService) waitForTeamSharedPVCBinding(ctx context.Context, namespace, func (s *PVCService) createPVForTeamSharedPVC(ctx context.Context, namespace, pvcName string, userID, teamID, storageSizeGB int, storageClass string) (*corev1.PersistentVolumeClaim, error) { pvName := fmt.Sprintf("clawreef-pv-user-%d-team-%d-shared", userID, teamID) - hostPathPrefix := "/data/clawreef" - if s.client != nil && s.client.HostPathPrefix != "" { - hostPathPrefix = s.client.HostPathPrefix - } - hostPath := fmt.Sprintf("%s/user-%d/team-%d-shared", hostPathPrefix, userID, teamID) existingPV, err := s.client.Clientset.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) if err == nil && existingPV != nil { @@ -245,6 +260,16 @@ func (s *PVCService) createPVForTeamSharedPVC(ctx context.Context, namespace, pv if err != nil { return nil, fmt.Errorf("failed to get Team shared PVC %s for UID: %w", pvcName, err) } + + if strings.TrimSpace(s.client.WorkspaceNFSServer) != "" { + return s.createWorkspaceNFSPVForTeamSharedPVC(ctx, namespace, pvcName, pvc, userID, teamID, storageSizeGB, storageClass, pvName) + } + + hostPathPrefix := "/data/clawreef" + if s.client != nil && s.client.HostPathPrefix != "" { + hostPathPrefix = s.client.HostPathPrefix + } + hostPath := fmt.Sprintf("%s/user-%d/team-%d-shared", hostPathPrefix, userID, teamID) storageSize := resource.MustParse(fmt.Sprintf("%dGi", storageSizeGB)) nodeAffinity, err := s.hostPathPVNodeAffinity(ctx) if err != nil { @@ -305,6 +330,183 @@ func (s *PVCService) createPVForTeamSharedPVC(ctx context.Context, namespace, pv return pvc, nil } +func (s *PVCService) createWorkspaceNFSPVForTeamSharedPVC(ctx context.Context, namespace, pvcName string, pvc *corev1.PersistentVolumeClaim, userID, teamID, storageSizeGB int, storageClass, pvName string) (*corev1.PersistentVolumeClaim, error) { + if err := s.ensureWorkspaceNFSPVForTeamSharedPVC(ctx, namespace, pvcName, pvc, userID, teamID, storageSizeGB, storageClass, pvName); err != nil { + return nil, err + } + + time.Sleep(3 * time.Second) + pvc, err := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get Team shared PVC after NFS PV creation: %w", err) + } + if pvc.Status.Phase != corev1.ClaimBound { + return nil, fmt.Errorf("Team shared PVC %s is still not bound after NFS PV creation, status: %s", pvcName, pvc.Status.Phase) + } + fmt.Printf("Team shared PVC %s successfully bound to NFS PV %s (%s:%s)\n", pvcName, pvName, s.client.WorkspaceNFSServer, teamSharedWorkspaceNFSPath(s.client.WorkspaceNFSPath, userID, teamID)) + return pvc, nil +} + +func (s *PVCService) ensureWorkspaceNFSPVForTeamSharedPVC(ctx context.Context, namespace, pvcName string, pvc *corev1.PersistentVolumeClaim, userID, teamID, storageSizeGB int, storageClass, pvName string) error { + if err := s.ensureTeamSharedWorkspaceDirectory(userID, teamID); err != nil { + return err + } + + storageSize := resource.MustParse(fmt.Sprintf("%dGi", storageSizeGB)) + nfsPath := teamSharedWorkspaceNFSPath(s.client.WorkspaceNFSPath, userID, teamID) + nfsServer, err := s.workspaceNFSServerForPV(ctx) + if err != nil { + return err + } + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvName, + Labels: map[string]string{ + "app": "clawreef", + "user-id": fmt.Sprintf("%d", userID), + "team-id": fmt.Sprintf("%d", teamID), + "managed-by": "clawreef", + }, + }, + Spec: corev1.PersistentVolumeSpec{ + Capacity: corev1.ResourceList{ + corev1.ResourceStorage: storageSize, + }, + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteMany, + }, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + StorageClassName: storageClass, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + NFS: &corev1.NFSVolumeSource{ + Server: nfsServer, + Path: nfsPath, + }, + }, + ClaimRef: &corev1.ObjectReference{ + Kind: "PersistentVolumeClaim", + APIVersion: "v1", + Namespace: namespace, + Name: pvcName, + UID: pvc.UID, + ResourceVersion: pvc.ResourceVersion, + }, + }, + } + if _, err := s.client.Clientset.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}); err != nil { + if errors.IsAlreadyExists(err) { + existing, getErr := s.client.Clientset.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) + if getErr != nil { + return fmt.Errorf("failed to get existing Team shared NFS PV %s: %w", pvName, getErr) + } + if existing.Spec.ClaimRef != nil && existing.Spec.ClaimRef.Namespace == namespace && existing.Spec.ClaimRef.Name == pvcName { + if existing.Spec.NFS != nil && + strings.TrimSpace(existing.Spec.NFS.Server) == nfsServer && + strings.TrimSpace(existing.Spec.NFS.Path) == nfsPath { + return nil + } + return fmt.Errorf("Team shared PV %s already exists for %s/%s but is not workspace NFS-backed", pvName, namespace, pvcName) + } + } + return fmt.Errorf("failed to create Team shared NFS PV %s: %w", pvName, err) + } + return nil +} + +func (s *PVCService) workspaceNFSServerForPV(ctx context.Context) (string, error) { + server := strings.TrimSpace(s.client.WorkspaceNFSServer) + if server == "" || s.client.Clientset == nil { + return server, nil + } + serviceName, namespace, ok := workspaceNFSServiceRef(server, s.client.Namespace) + if !ok { + return server, nil + } + svc, err := s.client.Clientset.CoreV1().Services(namespace).Get(ctx, serviceName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return server, nil + } + return "", fmt.Errorf("failed to resolve workspace NFS service %s/%s: %w", namespace, serviceName, err) + } + clusterIP := strings.TrimSpace(svc.Spec.ClusterIP) + if clusterIP == "" || strings.EqualFold(clusterIP, corev1.ClusterIPNone) { + return server, nil + } + return clusterIP, nil +} + +func workspaceNFSServiceRef(server, defaultNamespace string) (string, string, bool) { + host := strings.TrimSuffix(strings.TrimSpace(server), ".") + if host == "" || net.ParseIP(host) != nil { + return "", "", false + } + parts := strings.Split(strings.ToLower(host), ".") + if len(parts) == 1 { + namespace := strings.TrimSpace(defaultNamespace) + if namespace == "" { + namespace = "default" + } + return parts[0], namespace, true + } + if len(parts) >= 3 && parts[2] == "svc" && parts[0] != "" && parts[1] != "" { + return parts[0], parts[1], true + } + return "", "", false +} + +func (s *PVCService) ensureTeamSharedWorkspaceDirectory(userID, teamID int) error { + root := strings.TrimSpace(s.client.WorkspaceRoot) + if root == "" { + return nil + } + dir := filepath.Join(root, filepath.FromSlash(TeamSharedWorkspaceRelativePath(userID, teamID))) + dirs := append([]string{dir}, teamSharedRuntimeSubdirectories(dir)...) + for _, target := range dirs { + if err := os.MkdirAll(target, 0o2775); err != nil { + return fmt.Errorf("failed to create Team shared runtime workspace directory %s: %w", target, err) + } + _ = os.Chown(target, 1000, 1000) + if err := os.Chmod(target, 0o2775); err != nil { + return fmt.Errorf("failed to chmod Team shared runtime workspace directory %s: %w", target, err) + } + } + return nil +} + +func teamSharedRuntimeSubdirectories(root string) []string { + return []string{ + filepath.Join(root, "status"), + filepath.Join(root, "inbox"), + filepath.Join(root, "results"), + filepath.Join(root, "tasks"), + } +} + +func TeamSharedWorkspaceRelativePath(userID, teamID int) string { + return fmt.Sprintf("teams/user-%d/team-%d-shared", userID, teamID) +} + +func TeamSharedWorkspacePath(workspaceRoot string, userID, teamID int) string { + root := strings.TrimRight(strings.TrimSpace(workspaceRoot), "/") + if root == "" { + root = "/workspaces" + } + return root + "/" + TeamSharedWorkspaceRelativePath(userID, teamID) +} + +func teamSharedWorkspaceNFSPath(basePath string, userID, teamID int) string { + basePath = strings.TrimSpace(basePath) + if basePath == "" { + basePath = "/" + } + relativePath := TeamSharedWorkspaceRelativePath(userID, teamID) + if basePath == "/" { + return "/" + relativePath + } + return path.Join(basePath, relativePath) +} + func (s *PVCService) hostPathPVNodeAffinity(ctx context.Context) (*corev1.VolumeNodeAffinity, error) { if s == nil || s.client == nil { return nil, fmt.Errorf("k8s client not initialized") diff --git a/backend/internal/services/k8s/pvc_service_test.go b/backend/internal/services/k8s/pvc_service_test.go index 0965961..764dcb0 100644 --- a/backend/internal/services/k8s/pvc_service_test.go +++ b/backend/internal/services/k8s/pvc_service_test.go @@ -2,10 +2,14 @@ package k8s import ( "context" + "os" + "path/filepath" + "strings" "testing" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/fake" ) @@ -105,6 +109,222 @@ func TestHostPathPVNodeAffinitySkipsUnschedulableAndNotReadyNodes(t *testing.T) requireHostnameAffinity(t, affinity, "node-c-host") } +func TestCreatePVForTeamSharedPVCUsesWorkspaceNFSWhenConfigured(t *testing.T) { + ctx := context.Background() + workspaceRoot := t.TempDir() + namespace := "clawmanager-user-1" + pvcName := "clawreef-team-28-shared" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvcName, + Namespace: namespace, + UID: types.UID("pvc-uid"), + ResourceVersion: "1", + }, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + clientset := fake.NewSimpleClientset(pvc) + service := &PVCService{ + client: &Client{ + Clientset: clientset, + WorkspaceRoot: workspaceRoot, + WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local", + WorkspaceNFSPath: "/", + }, + } + + if _, err := service.createPVForTeamSharedPVC(ctx, namespace, pvcName, 1, 28, 10, "manual"); err != nil { + t.Fatalf("createPVForTeamSharedPVC returned error: %v", err) + } + + pv, err := clientset.CoreV1().PersistentVolumes().Get(ctx, "clawreef-pv-user-1-team-28-shared", metav1.GetOptions{}) + if err != nil { + t.Fatalf("expected Team shared PV: %v", err) + } + if pv.Spec.PersistentVolumeSource.NFS == nil { + t.Fatalf("expected NFS PV source, got %#v", pv.Spec.PersistentVolumeSource) + } + if pv.Spec.PersistentVolumeSource.NFS.Server != "workspace-store.clawmanager-system.svc.cluster.local" { + t.Fatalf("unexpected NFS server: %#v", pv.Spec.PersistentVolumeSource.NFS) + } + if pv.Spec.PersistentVolumeSource.NFS.Path != "/teams/user-1/team-28-shared" { + t.Fatalf("unexpected NFS path: %#v", pv.Spec.PersistentVolumeSource.NFS) + } + if pv.Spec.NodeAffinity != nil { + t.Fatalf("NFS-backed Team PV must not pin runtime sharing to one node: %#v", pv.Spec.NodeAffinity) + } + if _, err := os.Stat(filepath.Join(workspaceRoot, "teams", "user-1", "team-28-shared")); err != nil { + t.Fatalf("expected runtime workspace team directory to be created: %v", err) + } +} + +func TestCreatePVForTeamSharedPVCResolvesWorkspaceServiceDNS(t *testing.T) { + ctx := context.Background() + namespace := "clawmanager-user-1" + pvcName := "clawreef-team-28-shared" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvcName, + Namespace: namespace, + UID: types.UID("pvc-uid"), + ResourceVersion: "1", + }, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "workspace-store", + Namespace: "clawmanager-system", + }, + Spec: corev1.ServiceSpec{ClusterIP: "10.96.0.44"}, + } + clientset := fake.NewSimpleClientset(pvc, service) + pvcService := &PVCService{ + client: &Client{ + Clientset: clientset, + WorkspaceRoot: t.TempDir(), + WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local", + WorkspaceNFSPath: "/", + }, + } + + if _, err := pvcService.createPVForTeamSharedPVC(ctx, namespace, pvcName, 1, 28, 10, "manual"); err != nil { + t.Fatalf("createPVForTeamSharedPVC returned error: %v", err) + } + + pv, err := clientset.CoreV1().PersistentVolumes().Get(ctx, "clawreef-pv-user-1-team-28-shared", metav1.GetOptions{}) + if err != nil { + t.Fatalf("expected Team shared PV: %v", err) + } + if pv.Spec.NFS == nil || pv.Spec.NFS.Server != "10.96.0.44" { + t.Fatalf("expected resolved ClusterIP NFS server, got %#v", pv.Spec.PersistentVolumeSource) + } +} + +func TestCreateTeamSharedPVCCreatesWorkspaceDirectoryBeforeReturning(t *testing.T) { + ctx := context.Background() + workspaceRoot := t.TempDir() + client := &Client{ + Clientset: fake.NewSimpleClientset(), + Namespace: "clawmanager", + StorageClass: "manual", + WorkspaceRoot: workspaceRoot, + WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local", + WorkspaceNFSPath: "/", + } + service := &PVCService{ + client: client, + namespaceService: &NamespaceService{client: client}, + } + + if _, err := service.CreateTeamSharedPVC(ctx, 1, 28, 10, "manual"); err != nil { + t.Fatalf("CreateTeamSharedPVC returned error: %v", err) + } + + if _, err := os.Stat(filepath.Join(workspaceRoot, "teams", "user-1", "team-28-shared")); err != nil { + t.Fatalf("expected Team shared runtime workspace directory before Lite gateways start: %v", err) + } +} + +func TestCreateTeamSharedPVCCreatesWritableRuntimeSubdirectories(t *testing.T) { + ctx := context.Background() + workspaceRoot := t.TempDir() + client := &Client{ + Clientset: fake.NewSimpleClientset(), + Namespace: "clawmanager", + StorageClass: "manual", + WorkspaceRoot: workspaceRoot, + WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local", + WorkspaceNFSPath: "/", + } + service := &PVCService{ + client: client, + namespaceService: &NamespaceService{client: client}, + } + + if _, err := service.CreateTeamSharedPVC(ctx, 1, 28, 10, "manual"); err != nil { + t.Fatalf("CreateTeamSharedPVC returned error: %v", err) + } + + for _, name := range []string{"status", "inbox", "results", "tasks"} { + dir := filepath.Join(workspaceRoot, "teams", "user-1", "team-28-shared", name) + info, err := os.Stat(dir) + if err != nil { + t.Fatalf("expected Team shared runtime subdirectory %s before gateways start: %v", name, err) + } + if !info.IsDir() { + t.Fatalf("expected %s to be a directory", dir) + } + } +} + +func TestCreateTeamSharedPVCCreatesWorkspaceNFSPVBeforeReturning(t *testing.T) { + ctx := context.Background() + client := &Client{ + Clientset: fake.NewSimpleClientset(), + Namespace: "clawmanager", + StorageClass: "manual", + WorkspaceRoot: t.TempDir(), + WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local", + WorkspaceNFSPath: "/", + } + service := &PVCService{ + client: client, + namespaceService: &NamespaceService{client: client}, + } + + if _, err := service.CreateTeamSharedPVC(ctx, 1, 28, 10, "manual"); err != nil { + t.Fatalf("CreateTeamSharedPVC returned error: %v", err) + } + + pv, err := client.Clientset.CoreV1().PersistentVolumes().Get(ctx, "clawreef-pv-user-1-team-28-shared", metav1.GetOptions{}) + if err != nil { + t.Fatalf("expected Team shared NFS PV before Lite gateways start: %v", err) + } + if pv.Spec.NFS == nil || pv.Spec.NFS.Path != "/teams/user-1/team-28-shared" { + t.Fatalf("unexpected Team shared PV source: %#v", pv.Spec.PersistentVolumeSource) + } +} + +func TestCreateTeamSharedPVCRejectsExistingNonWorkspaceNFSPV(t *testing.T) { + ctx := context.Background() + namespace := "clawmanager-user-1" + pvcName := "clawreef-team-28-shared" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvcName, + Namespace: namespace, + Labels: map[string]string{"team-id": "28"}, + }, + } + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: "clawreef-pv-user-1-team-28-shared"}, + Spec: corev1.PersistentVolumeSpec{ + ClaimRef: &corev1.ObjectReference{Namespace: namespace, Name: pvcName}, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + HostPath: &corev1.HostPathVolumeSource{Path: "/data/clawreef/user-1/team-28-shared"}, + }, + }, + } + client := &Client{ + Clientset: fake.NewSimpleClientset(pvc, pv), + Namespace: "clawmanager", + StorageClass: "manual", + WorkspaceRoot: t.TempDir(), + WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local", + WorkspaceNFSPath: "/", + } + service := &PVCService{ + client: client, + namespaceService: &NamespaceService{client: client}, + } + + _, err := service.CreateTeamSharedPVC(ctx, 1, 28, 10, "manual") + if err == nil || !strings.Contains(err.Error(), "workspace NFS") { + t.Fatalf("expected existing hostPath Team PV to be rejected, got %v", err) + } +} + func TestHostPathPVNodeAffinitySkipsHardTaintedNodes(t *testing.T) { service := &PVCService{ client: &Client{ diff --git a/backend/internal/services/k8s/runtime_deployment_service.go b/backend/internal/services/k8s/runtime_deployment_service.go new file mode 100644 index 0000000..0e98c94 --- /dev/null +++ b/backend/internal/services/k8s/runtime_deployment_service.go @@ -0,0 +1,454 @@ +package k8s + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "strconv" + "strings" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/retry" +) + +const ( + runtimeAgentPort int32 = 19090 + runtimeWorkspaceVolume = "workspaces" + defaultWorkspaceMount = "/workspaces" + defaultGatewayPortStart = 20000 + defaultGatewayPortEnd = 20099 + hermesTUIDir = "/usr/local/lib/hermes-agent/ui-tui" +) + +type RuntimeDeploymentSpec struct { + Name string + Namespace string + RuntimeType string + Image string + Replicas int32 + WorkspaceNFSServer string + WorkspaceNFSPath string + WorkspaceMountPath string + GatewayPortStart int + GatewayPortEnd int + AgentControlToken string + AgentReportToken string + BackendURL string + TrustedProxyCIDRs string +} + +type RuntimeDeploymentPod struct { + RuntimeType string + Namespace string + DeploymentName string + PodName string + PodIP *string + NodeName *string + ImageRef string + State string +} + +type RuntimeDeploymentService interface { + Ensure(ctx context.Context, spec RuntimeDeploymentSpec) error + Scale(ctx context.Context, namespace, name string, replicas int32) error + RolloutImage(ctx context.Context, namespace, name, image string, maxUnavailable, maxSurge int) error + ListPods(ctx context.Context, namespace, runtimeType string) ([]RuntimeDeploymentPod, error) +} + +type runtimeDeploymentService struct { + client kubernetes.Interface +} + +func NewRuntimeDeploymentService(client kubernetes.Interface) RuntimeDeploymentService { + return &runtimeDeploymentService{client: client} +} + +func BuildRuntimeDeployment(spec RuntimeDeploymentSpec) *appsv1.Deployment { + labels := map[string]string{ + "app": spec.Name, + "clawmanager.io/runtime-type": spec.RuntimeType, + } + replicas := spec.Replicas + workspaceMountPath := runtimeWorkspaceMountPath(spec.WorkspaceMountPath) + gatewayPortStart := runtimeGatewayPortValue(spec.GatewayPortStart, defaultGatewayPortStart) + gatewayPortEnd := runtimeGatewayPortValue(spec.GatewayPortEnd, defaultGatewayPortEnd) + env := buildRuntimeAgentEnv(spec, workspaceMountPath, gatewayPortStart, gatewayPortEnd) + + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: spec.Name, + Namespace: spec.Namespace, + Labels: copyStringMap(labels), + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: copyStringMap(labels), + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: copyStringMap(labels), + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "runtime", + Image: spec.Image, + Ports: []corev1.ContainerPort{ + { + Name: "agent", + ContainerPort: runtimeAgentPort, + }, + }, + Env: env, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: runtimeWorkspaceVolume, + MountPath: workspaceMountPath, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: runtimeWorkspaceVolume, + VolumeSource: corev1.VolumeSource{ + NFS: &corev1.NFSVolumeSource{ + Server: spec.WorkspaceNFSServer, + Path: spec.WorkspaceNFSPath, + }, + }, + }, + }, + }, + }, + }, + } +} + +func buildRuntimeAgentEnv(spec RuntimeDeploymentSpec, workspaceRoot string, gatewayPortStart, gatewayPortEnd int) []corev1.EnvVar { + env := []corev1.EnvVar{ + {Name: "CLAWMANAGER_RUNTIME_TYPE", Value: spec.RuntimeType}, + {Name: "CLAWMANAGER_BACKEND_URL", Value: runtimeBackendURL(spec)}, + {Name: "CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME", Value: spec.Name}, + {Name: "CLAWMANAGER_RUNTIME_IMAGE_REF", Value: spec.Image}, + {Name: "CLAWMANAGER_AGENT_PORT", Value: "19090"}, + {Name: "CLAWMANAGER_GATEWAY_PORT_START", Value: strconv.Itoa(gatewayPortStart)}, + {Name: "CLAWMANAGER_GATEWAY_PORT_END", Value: strconv.Itoa(gatewayPortEnd)}, + {Name: "CLAWMANAGER_AGENT_CONTROL_TOKEN", Value: spec.AgentControlToken}, + {Name: "CLAWMANAGER_AGENT_REPORT_TOKEN", Value: spec.AgentReportToken}, + {Name: "RUNTIME_WORKSPACE_ROOT", Value: workspaceRoot}, + {Name: "RUNTIME_AGENT_LISTEN_ADDR", Value: "0.0.0.0:19090"}, + {Name: "RUNTIME_AGENT_PUBLIC_PORT", Value: "19090"}, + {Name: "RUNTIME_AGENT_CONTROL_TOKEN", Value: spec.AgentControlToken}, + {Name: "RUNTIME_AGENT_REPORT_TOKEN", Value: spec.AgentReportToken}, + {Name: "RUNTIME_GATEWAY_PORT_START", Value: strconv.Itoa(gatewayPortStart)}, + {Name: "RUNTIME_GATEWAY_PORT_END", Value: strconv.Itoa(gatewayPortEnd)}, + } + if strings.EqualFold(spec.RuntimeType, "hermes") { + env = append(env, corev1.EnvVar{Name: "HERMES_TUI_DIR", Value: hermesTUIDir}) + } + env = append(env, + fieldRefEnv("POD_NAME", "metadata.name"), + fieldRefEnv("POD_NAMESPACE", "metadata.namespace"), + fieldRefEnv("POD_IP", "status.podIP"), + fieldRefEnv("NODE_NAME", "spec.nodeName"), + ) + if spec.TrustedProxyCIDRs != "" { + env = append(env, corev1.EnvVar{Name: "CLAWMANAGER_TRUSTED_PROXY_CIDRS", Value: spec.TrustedProxyCIDRs}) + } + return env +} + +func runtimeBackendURL(spec RuntimeDeploymentSpec) string { + if spec.BackendURL != "" { + return spec.BackendURL + } + namespace := spec.Namespace + if namespace == "" { + namespace = "clawmanager-system" + } + return fmt.Sprintf("http://clawmanager-gateway.%s.svc.cluster.local:9001", namespace) +} + +func fieldRefEnv(name, fieldPath string) corev1.EnvVar { + return corev1.EnvVar{ + Name: name, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: fieldPath}, + }, + } +} + +func runtimeWorkspaceMountPath(value string) string { + if value == "" { + return defaultWorkspaceMount + } + return value +} + +func runtimeGatewayPortValue(value, defaultValue int) int { + if value == 0 { + return defaultValue + } + return value +} + +func (s *runtimeDeploymentService) Ensure(ctx context.Context, spec RuntimeDeploymentSpec) error { + if s == nil || s.client == nil { + return fmt.Errorf("k8s client not initialized") + } + + desired := BuildRuntimeDeployment(spec) + deployments := s.client.AppsV1().Deployments(spec.Namespace) + existing, err := deployments.Get(ctx, spec.Name, metav1.GetOptions{}) + if errors.IsNotFound(err) { + _, createErr := deployments.Create(ctx, desired, metav1.CreateOptions{}) + if createErr != nil { + return fmt.Errorf("failed to create runtime deployment %s/%s: %w", spec.Namespace, spec.Name, createErr) + } + return nil + } + if err != nil { + return fmt.Errorf("failed to get runtime deployment %s/%s: %w", spec.Namespace, spec.Name, err) + } + + if !reflect.DeepEqual(existing.Spec.Selector, desired.Spec.Selector) { + return fmt.Errorf("runtime deployment %s/%s selector mismatch; delete and recreate the deployment to change immutable selector", spec.Namespace, spec.Name) + } + + updated := existing.DeepCopy() + if updated.Labels == nil { + updated.Labels = map[string]string{} + } + for key, value := range desired.Labels { + updated.Labels[key] = value + } + updated.Spec.Replicas = desired.Spec.Replicas + if updated.Spec.Template.Labels == nil { + updated.Spec.Template.Labels = map[string]string{} + } + for key, value := range desired.Spec.Template.Labels { + updated.Spec.Template.Labels[key] = value + } + updated.Spec.Template.Spec = desired.Spec.Template.Spec + + _, err = deployments.Update(ctx, updated, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to update runtime deployment %s/%s: %w", spec.Namespace, spec.Name, err) + } + return nil +} + +func (s *runtimeDeploymentService) Scale(ctx context.Context, namespace, name string, replicas int32) error { + if s == nil || s.client == nil { + return fmt.Errorf("k8s client not initialized") + } + + deployments := s.client.AppsV1().Deployments(namespace) + patch, err := json.Marshal(map[string]interface{}{ + "spec": map[string]interface{}{ + "replicas": replicas, + }, + }) + if err != nil { + return fmt.Errorf("failed to build runtime deployment scale patch %s/%s: %w", namespace, name, err) + } + if _, err := deployments.Patch(ctx, name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("failed to scale runtime deployment %s/%s: %w", namespace, name, err) + } + return nil +} + +func (s *runtimeDeploymentService) RolloutImage(ctx context.Context, namespace, name, image string, maxUnavailable, maxSurge int) error { + if s == nil || s.client == nil { + return fmt.Errorf("k8s client not initialized") + } + image = strings.TrimSpace(image) + if image == "" { + return fmt.Errorf("runtime rollout image is required") + } + + deployments := s.client.AppsV1().Deployments(namespace) + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + existing, err := deployments.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return err + } + + updated := existing.DeepCopy() + containerIndex := -1 + for index, container := range updated.Spec.Template.Spec.Containers { + if container.Name == "runtime" { + containerIndex = index + break + } + } + if containerIndex < 0 { + return fmt.Errorf("runtime deployment %s/%s has no runtime container", namespace, name) + } + + container := &updated.Spec.Template.Spec.Containers[containerIndex] + container.Image = image + upsertEnvVar(container, "CLAWMANAGER_RUNTIME_IMAGE_REF", image) + + updated.Spec.Strategy.Type = appsv1.RollingUpdateDeploymentStrategyType + updated.Spec.Strategy.RollingUpdate = &appsv1.RollingUpdateDeployment{ + MaxUnavailable: intOrStringPtr(positiveRolloutInt(maxUnavailable)), + MaxSurge: intOrStringPtr(positiveRolloutInt(maxSurge)), + } + + _, err = deployments.Update(ctx, updated, metav1.UpdateOptions{}) + return err + }) + if err != nil { + return fmt.Errorf("failed to update runtime deployment %s/%s image: %w", namespace, name, err) + } + return nil +} + +func (s *runtimeDeploymentService) ListPods(ctx context.Context, namespace, runtimeType string) ([]RuntimeDeploymentPod, error) { + if s == nil || s.client == nil { + return nil, fmt.Errorf("k8s client not initialized") + } + namespace = strings.TrimSpace(namespace) + if namespace == "" { + return nil, fmt.Errorf("runtime namespace is required") + } + runtimeType = strings.ToLower(strings.TrimSpace(runtimeType)) + + listOptions := metav1.ListOptions{} + if runtimeType != "" { + listOptions.LabelSelector = labels.Set{"clawmanager.io/runtime-type": runtimeType}.String() + } + deploymentList, err := s.client.AppsV1().Deployments(namespace).List(ctx, listOptions) + if err != nil { + return nil, fmt.Errorf("failed to list runtime deployments in %s: %w", namespace, err) + } + + var pods []RuntimeDeploymentPod + for _, deployment := range deploymentList.Items { + deploymentRuntimeType := strings.ToLower(strings.TrimSpace(deployment.Labels["clawmanager.io/runtime-type"])) + if deploymentRuntimeType == "" { + continue + } + if runtimeType != "" && deploymentRuntimeType != runtimeType { + continue + } + if deployment.Spec.Selector == nil { + return nil, fmt.Errorf("runtime deployment %s/%s has no selector", deployment.Namespace, deployment.Name) + } + selector, err := metav1.LabelSelectorAsSelector(deployment.Spec.Selector) + if err != nil { + return nil, fmt.Errorf("runtime deployment %s/%s has invalid selector: %w", deployment.Namespace, deployment.Name, err) + } + podList, err := s.client.CoreV1().Pods(deployment.Namespace).List(ctx, metav1.ListOptions{LabelSelector: selector.String()}) + if err != nil { + return nil, fmt.Errorf("failed to list runtime deployment pods %s/%s: %w", deployment.Namespace, deployment.Name, err) + } + deploymentImage := runtimeContainerImage(deployment.Spec.Template.Spec.Containers) + for _, pod := range podList.Items { + image := runtimeContainerImage(pod.Spec.Containers) + if image == "" { + image = deploymentImage + } + pods = append(pods, RuntimeDeploymentPod{ + RuntimeType: deploymentRuntimeType, + Namespace: pod.Namespace, + DeploymentName: deployment.Name, + PodName: pod.Name, + PodIP: stringPtrIfNotEmpty(pod.Status.PodIP), + NodeName: stringPtrIfNotEmpty(pod.Spec.NodeName), + ImageRef: image, + State: runtimeK8sPodState(pod), + }) + } + } + return pods, nil +} + +func runtimeContainerImage(containers []corev1.Container) string { + for _, container := range containers { + if container.Name == "runtime" { + return strings.TrimSpace(container.Image) + } + } + if len(containers) == 1 { + return strings.TrimSpace(containers[0].Image) + } + return "" +} + +func runtimeK8sPodState(pod corev1.Pod) string { + if pod.DeletionTimestamp != nil { + return "deleted" + } + switch pod.Status.Phase { + case corev1.PodPending: + return "pending" + case corev1.PodRunning: + if runtimeK8sPodReady(pod) { + return "ready" + } + return "pending" + case corev1.PodSucceeded, corev1.PodFailed: + return "unhealthy" + default: + return "pending" + } +} + +func runtimeK8sPodReady(pod corev1.Pod) bool { + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue { + return true + } + } + return false +} + +func stringPtrIfNotEmpty(value string) *string { + value = strings.TrimSpace(value) + if value == "" { + return nil + } + return &value +} + +func upsertEnvVar(container *corev1.Container, name, value string) { + for index := range container.Env { + if container.Env[index].Name == name { + container.Env[index].Value = value + container.Env[index].ValueFrom = nil + return + } + } + container.Env = append(container.Env, corev1.EnvVar{Name: name, Value: value}) +} + +func positiveRolloutInt(value int) int { + if value <= 0 { + return 1 + } + return value +} + +func intOrStringPtr(value int) *intstr.IntOrString { + result := intstr.FromInt(value) + return &result +} diff --git a/backend/internal/services/k8s/runtime_deployment_service_test.go b/backend/internal/services/k8s/runtime_deployment_service_test.go new file mode 100644 index 0000000..0cced48 --- /dev/null +++ b/backend/internal/services/k8s/runtime_deployment_service_test.go @@ -0,0 +1,481 @@ +package k8s + +import ( + "context" + "fmt" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +func TestBuildRuntimeDeployment(t *testing.T) { + deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{ + Name: "runtime-openclaw", + Namespace: "runtime-system", + RuntimeType: "openclaw", + Image: "registry/openclaw:latest", + Replicas: 2, + WorkspaceNFSServer: "10.0.0.9", + WorkspaceNFSPath: "/exports/workspaces", + AgentControlToken: "control-secret", + AgentReportToken: "report-secret", + }) + + if deployment.Name != "runtime-openclaw" || deployment.Namespace != "runtime-system" { + t.Fatalf("unexpected deployment identity: %s/%s", deployment.Namespace, deployment.Name) + } + if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != 2 { + t.Fatalf("expected replicas 2, got %#v", deployment.Spec.Replicas) + } + if got := deployment.Labels["clawmanager.io/runtime-type"]; got != "openclaw" { + t.Fatalf("expected runtime-type label openclaw, got %q", got) + } + + if len(deployment.Spec.Template.Spec.Containers) != 1 { + t.Fatalf("expected one container, got %d", len(deployment.Spec.Template.Spec.Containers)) + } + container := deployment.Spec.Template.Spec.Containers[0] + if container.Name != "runtime" || container.Image != "registry/openclaw:latest" { + t.Fatalf("unexpected runtime container: %#v", container) + } + requireContainerPort(t, container, "agent", 19090) + requireVolumeMount(t, container, "workspaces", "/workspaces") + requireEnv(t, container, "CLAWMANAGER_RUNTIME_TYPE", "openclaw") + requireEnv(t, container, "CLAWMANAGER_AGENT_PORT", "19090") + requireEnv(t, container, "CLAWMANAGER_GATEWAY_PORT_START", "20000") + requireEnv(t, container, "CLAWMANAGER_GATEWAY_PORT_END", "20099") + requireEnv(t, container, "CLAWMANAGER_AGENT_CONTROL_TOKEN", "control-secret") + requireEnv(t, container, "CLAWMANAGER_AGENT_REPORT_TOKEN", "report-secret") + + if got := container.Resources.Requests[corev1.ResourceCPU]; got.Cmp(resource.MustParse("500m")) != 0 { + t.Fatalf("expected CPU request 500m, got %s", got.String()) + } + if got := container.Resources.Requests[corev1.ResourceMemory]; got.Cmp(resource.MustParse("1Gi")) != 0 { + t.Fatalf("expected memory request 1Gi, got %s", got.String()) + } + + if len(deployment.Spec.Template.Spec.Volumes) != 1 { + t.Fatalf("expected one volume, got %#v", deployment.Spec.Template.Spec.Volumes) + } + volume := deployment.Spec.Template.Spec.Volumes[0] + if volume.Name != "workspaces" || volume.NFS == nil { + t.Fatalf("expected workspaces NFS volume, got %#v", volume) + } + if volume.NFS.Server != "10.0.0.9" || volume.NFS.Path != "/exports/workspaces" { + t.Fatalf("unexpected NFS source: %#v", volume.NFS) + } +} + +func TestBuildRuntimeDeploymentKeepsCapacityLimitOutOfRuntimeEnv(t *testing.T) { + deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{ + Name: "runtime-openclaw", + Namespace: "runtime-system", + RuntimeType: "openclaw", + Image: "registry/openclaw:latest", + Replicas: 2, + WorkspaceNFSServer: "10.0.0.9", + WorkspaceNFSPath: "/exports/workspaces", + WorkspaceMountPath: "/runtime-workspaces", + GatewayPortStart: 21000, + GatewayPortEnd: 21049, + }) + + container := deployment.Spec.Template.Spec.Containers[0] + requireVolumeMount(t, container, "workspaces", "/runtime-workspaces") + requireEnv(t, container, "CLAWMANAGER_GATEWAY_PORT_START", "21000") + requireEnv(t, container, "CLAWMANAGER_GATEWAY_PORT_END", "21049") + requireEnvAbsent(t, container, "CLAWMANAGER_MAX_GATEWAYS_PER_POD") + requireEnvAbsent(t, container, "RUNTIME_MAX_GATEWAYS_PER_POD") +} + +func TestBuildRuntimeDeploymentInjectsAgentV2Environment(t *testing.T) { + deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{ + Name: "runtime-hermes", + Namespace: "runtime-system", + RuntimeType: "hermes", + Image: "registry/hermes:v2", + Replicas: 1, + WorkspaceNFSServer: "10.0.0.9", + WorkspaceNFSPath: "/exports/workspaces", + WorkspaceMountPath: "/workspaces", + GatewayPortStart: 21000, + GatewayPortEnd: 21099, + AgentControlToken: "control-secret", + AgentReportToken: "report-secret", + BackendURL: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001", + TrustedProxyCIDRs: "10.42.0.0/16,10.43.0.0/16", + }) + + container := deployment.Spec.Template.Spec.Containers[0] + requireEnv(t, container, "CLAWMANAGER_RUNTIME_TYPE", "hermes") + requireEnv(t, container, "CLAWMANAGER_BACKEND_URL", "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001") + requireEnv(t, container, "CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME", "runtime-hermes") + requireEnv(t, container, "CLAWMANAGER_RUNTIME_IMAGE_REF", "registry/hermes:v2") + requireEnv(t, container, "RUNTIME_WORKSPACE_ROOT", "/workspaces") + requireEnv(t, container, "RUNTIME_AGENT_LISTEN_ADDR", "0.0.0.0:19090") + requireEnv(t, container, "RUNTIME_AGENT_PUBLIC_PORT", "19090") + requireEnv(t, container, "RUNTIME_AGENT_CONTROL_TOKEN", "control-secret") + requireEnv(t, container, "RUNTIME_AGENT_REPORT_TOKEN", "report-secret") + requireEnv(t, container, "RUNTIME_GATEWAY_PORT_START", "21000") + requireEnv(t, container, "RUNTIME_GATEWAY_PORT_END", "21099") + requireEnv(t, container, "HERMES_TUI_DIR", "/usr/local/lib/hermes-agent/ui-tui") + requireEnv(t, container, "CLAWMANAGER_TRUSTED_PROXY_CIDRS", "10.42.0.0/16,10.43.0.0/16") + requireEnvFieldRef(t, container, "POD_NAME", "metadata.name") + requireEnvFieldRef(t, container, "POD_NAMESPACE", "metadata.namespace") + requireEnvFieldRef(t, container, "POD_IP", "status.podIP") + requireEnvFieldRef(t, container, "NODE_NAME", "spec.nodeName") +} + +func TestRuntimeDeploymentServiceEnsureCreatesAndUpdates(t *testing.T) { + client := fake.NewSimpleClientset() + service := NewRuntimeDeploymentService(client) + spec := RuntimeDeploymentSpec{ + Name: "runtime-hermes", + Namespace: "runtime-system", + RuntimeType: "hermes", + Image: "registry/hermes:v1", + Replicas: 1, + WorkspaceNFSServer: "nfs.local", + WorkspaceNFSPath: "/exports", + } + + if err := service.Ensure(context.Background(), spec); err != nil { + t.Fatalf("Ensure create returned error: %v", err) + } + created, err := client.AppsV1().Deployments(spec.Namespace).Get(context.Background(), spec.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("failed to get created deployment: %v", err) + } + if created.Spec.Replicas == nil || *created.Spec.Replicas != 1 { + t.Fatalf("expected created replicas 1, got %#v", created.Spec.Replicas) + } + + spec.Image = "registry/hermes:v2" + spec.Replicas = 3 + if err := service.Ensure(context.Background(), spec); err != nil { + t.Fatalf("Ensure update returned error: %v", err) + } + updated, err := client.AppsV1().Deployments(spec.Namespace).Get(context.Background(), spec.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("failed to get updated deployment: %v", err) + } + if updated.Spec.Template.Spec.Containers[0].Image != "registry/hermes:v2" { + t.Fatalf("expected updated image, got %q", updated.Spec.Template.Spec.Containers[0].Image) + } + if updated.Spec.Replicas == nil || *updated.Spec.Replicas != 3 { + t.Fatalf("expected updated replicas 3, got %#v", updated.Spec.Replicas) + } +} + +func TestRuntimeDeploymentServiceEnsurePreservesUnownedMetadata(t *testing.T) { + existing := BuildRuntimeDeployment(RuntimeDeploymentSpec{ + Name: "runtime-openclaw", + Namespace: "runtime-system", + RuntimeType: "openclaw", + Image: "registry/openclaw:v1", + Replicas: 1, + WorkspaceNFSServer: "nfs.local", + WorkspaceNFSPath: "/exports", + }) + existing.Labels["admission.example.com/injected"] = "keep" + existing.Annotations = map[string]string{"admission.example.com/sidecar": "enabled"} + existing.Finalizers = []string{"protect.example.com/runtime"} + existing.Spec.Template.Labels["admission.example.com/template-label"] = "keep" + existing.Spec.Template.Annotations = map[string]string{"admission.example.com/template-annotation": "enabled"} + client := fake.NewSimpleClientset(existing) + service := NewRuntimeDeploymentService(client) + + if err := service.Ensure(context.Background(), RuntimeDeploymentSpec{ + Name: "runtime-openclaw", + Namespace: "runtime-system", + RuntimeType: "openclaw", + Image: "registry/openclaw:v2", + Replicas: 3, + WorkspaceNFSServer: "nfs.local", + WorkspaceNFSPath: "/exports", + }); err != nil { + t.Fatalf("Ensure update returned error: %v", err) + } + + updated, err := client.AppsV1().Deployments("runtime-system").Get(context.Background(), "runtime-openclaw", metav1.GetOptions{}) + if err != nil { + t.Fatalf("failed to get updated deployment: %v", err) + } + if updated.Labels["admission.example.com/injected"] != "keep" { + t.Fatalf("expected unrelated label to survive, got %#v", updated.Labels) + } + if updated.Annotations["admission.example.com/sidecar"] != "enabled" { + t.Fatalf("expected annotation to survive, got %#v", updated.Annotations) + } + if len(updated.Finalizers) != 1 || updated.Finalizers[0] != "protect.example.com/runtime" { + t.Fatalf("expected finalizer to survive, got %#v", updated.Finalizers) + } + if updated.Labels["clawmanager.io/runtime-type"] != "openclaw" { + t.Fatalf("expected owned runtime label to be present, got %#v", updated.Labels) + } + if updated.Spec.Template.Labels["admission.example.com/template-label"] != "keep" { + t.Fatalf("expected unrelated template label to survive, got %#v", updated.Spec.Template.Labels) + } + if updated.Spec.Template.Annotations["admission.example.com/template-annotation"] != "enabled" { + t.Fatalf("expected template annotation to survive, got %#v", updated.Spec.Template.Annotations) + } + if updated.Spec.Template.Labels["clawmanager.io/runtime-type"] != "openclaw" { + t.Fatalf("expected owned template runtime label to be present, got %#v", updated.Spec.Template.Labels) + } + if updated.Spec.Template.Spec.Containers[0].Image != "registry/openclaw:v2" { + t.Fatalf("expected image update while preserving template metadata, got %q", updated.Spec.Template.Spec.Containers[0].Image) + } +} + +func TestRuntimeDeploymentServiceEnsureRejectsSelectorChange(t *testing.T) { + existing := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "runtime-openclaw", Namespace: "runtime-system"}, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "different"}}, + }, + } + client := fake.NewSimpleClientset(existing) + service := NewRuntimeDeploymentService(client) + + err := service.Ensure(context.Background(), RuntimeDeploymentSpec{ + Name: "runtime-openclaw", + Namespace: "runtime-system", + RuntimeType: "openclaw", + Image: "registry/openclaw:latest", + Replicas: 1, + }) + if err == nil { + t.Fatalf("expected selector mismatch error") + } +} + +func TestRuntimeDeploymentServiceScale(t *testing.T) { + deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{ + Name: "runtime-openclaw", + Namespace: "runtime-system", + RuntimeType: "openclaw", + Image: "registry/openclaw:latest", + Replicas: 1, + WorkspaceNFSServer: "nfs.local", + WorkspaceNFSPath: "/exports", + }) + deployment.Spec.Template.Annotations = map[string]string{"admission.example.com/template": "keep"} + client := fake.NewSimpleClientset(deployment) + service := NewRuntimeDeploymentService(client) + + if err := service.Scale(context.Background(), "runtime-system", "runtime-openclaw", 4); err != nil { + t.Fatalf("Scale returned error: %v", err) + } + scaled, err := client.AppsV1().Deployments("runtime-system").Get(context.Background(), "runtime-openclaw", metav1.GetOptions{}) + if err != nil { + t.Fatalf("failed to get scaled deployment: %v", err) + } + if scaled.Spec.Replicas == nil || *scaled.Spec.Replicas != 4 { + t.Fatalf("expected replicas 4, got %#v", scaled.Spec.Replicas) + } + if scaled.Spec.Template.Spec.Containers[0].Image != "registry/openclaw:latest" { + t.Fatalf("expected Scale to preserve image, got %q", scaled.Spec.Template.Spec.Containers[0].Image) + } + if scaled.Spec.Template.Annotations["admission.example.com/template"] != "keep" { + t.Fatalf("expected Scale to preserve template metadata, got %#v", scaled.Spec.Template.Annotations) + } +} + +func TestRuntimeDeploymentServiceRolloutImage(t *testing.T) { + deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{ + Name: "runtime-hermes", + Namespace: "runtime-system", + RuntimeType: "hermes", + Image: "registry/hermes:v1", + Replicas: 3, + WorkspaceNFSServer: "nfs.local", + WorkspaceNFSPath: "/exports", + }) + client := fake.NewSimpleClientset(deployment) + service := NewRuntimeDeploymentService(client) + + if err := service.RolloutImage(context.Background(), "runtime-system", "runtime-hermes", "registry/hermes:v2", 1, 2); err != nil { + t.Fatalf("RolloutImage returned error: %v", err) + } + + updated, err := client.AppsV1().Deployments("runtime-system").Get(context.Background(), "runtime-hermes", metav1.GetOptions{}) + if err != nil { + t.Fatalf("failed to get updated deployment: %v", err) + } + container := updated.Spec.Template.Spec.Containers[0] + if container.Image != "registry/hermes:v2" { + t.Fatalf("expected updated image, got %q", container.Image) + } + requireEnv(t, container, "CLAWMANAGER_RUNTIME_IMAGE_REF", "registry/hermes:v2") + if updated.Spec.Strategy.Type != appsv1.RollingUpdateDeploymentStrategyType { + t.Fatalf("expected RollingUpdate strategy, got %q", updated.Spec.Strategy.Type) + } + if updated.Spec.Strategy.RollingUpdate == nil { + t.Fatal("expected rolling update strategy") + } + if got := updated.Spec.Strategy.RollingUpdate.MaxUnavailable.IntValue(); got != 1 { + t.Fatalf("maxUnavailable = %d, want 1", got) + } + if got := updated.Spec.Strategy.RollingUpdate.MaxSurge.IntValue(); got != 2 { + t.Fatalf("maxSurge = %d, want 2", got) + } +} + +func TestRuntimeDeploymentServiceRolloutImageRetriesConflict(t *testing.T) { + deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{ + Name: "runtime-hermes", + Namespace: "runtime-system", + RuntimeType: "hermes", + Image: "registry/hermes:v1", + Replicas: 1, + WorkspaceNFSServer: "nfs.local", + WorkspaceNFSPath: "/exports", + }) + client := fake.NewSimpleClientset(deployment) + updateAttempts := 0 + client.Fake.PrependReactor("update", "deployments", func(action k8stesting.Action) (bool, runtime.Object, error) { + updateAttempts++ + if updateAttempts == 1 { + return true, nil, apierrors.NewConflict(schema.GroupResource{Group: "apps", Resource: "deployments"}, "runtime-hermes", fmt.Errorf("stale resource version")) + } + return false, nil, nil + }) + service := NewRuntimeDeploymentService(client) + + if err := service.RolloutImage(context.Background(), "runtime-system", "runtime-hermes", "registry/hermes:v2", 1, 1); err != nil { + t.Fatalf("RolloutImage returned error: %v", err) + } + if updateAttempts < 2 { + t.Fatalf("update attempts = %d, want retry after conflict", updateAttempts) + } + + updated, err := client.AppsV1().Deployments("runtime-system").Get(context.Background(), "runtime-hermes", metav1.GetOptions{}) + if err != nil { + t.Fatalf("failed to get updated deployment: %v", err) + } + if got := updated.Spec.Template.Spec.Containers[0].Image; got != "registry/hermes:v2" { + t.Fatalf("image = %q, want registry/hermes:v2", got) + } +} + +func TestRuntimeDeploymentServiceListPodsReturnsRuntimeDeploymentPods(t *testing.T) { + deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{ + Name: "openclaw-runtime", + Namespace: "runtime-system", + RuntimeType: "openclaw", + Image: "registry/openclaw-lite:final2", + Replicas: 1, + WorkspaceNFSServer: "nfs.local", + WorkspaceNFSPath: "/exports", + }) + podIP := "10.42.0.12" + nodeName := "node-a" + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "openclaw-runtime-abc", + Namespace: "runtime-system", + Labels: map[string]string{ + "app": "openclaw-runtime", + "clawmanager.io/runtime-type": "openclaw", + }, + }, + Spec: corev1.PodSpec{ + NodeName: nodeName, + Containers: []corev1.Container{{ + Name: "runtime", + Image: "registry/openclaw-lite:final2", + }}, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + PodIP: podIP, + Conditions: []corev1.PodCondition{{ + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + }}, + }, + } + client := fake.NewSimpleClientset(deployment, pod) + service := NewRuntimeDeploymentService(client) + + pods, err := service.ListPods(context.Background(), "runtime-system", "openclaw") + if err != nil { + t.Fatalf("ListPods returned error: %v", err) + } + if len(pods) != 1 { + t.Fatalf("pods length = %d, want 1: %#v", len(pods), pods) + } + got := pods[0] + if got.RuntimeType != "openclaw" || got.Namespace != "runtime-system" || got.DeploymentName != "openclaw-runtime" || got.PodName != "openclaw-runtime-abc" { + t.Fatalf("runtime pod identity = %+v, want openclaw runtime pod", got) + } + if got.ImageRef != "registry/openclaw-lite:final2" { + t.Fatalf("image = %q, want registry/openclaw-lite:final2", got.ImageRef) + } + if got.State != "ready" { + t.Fatalf("state = %q, want ready", got.State) + } + if got.PodIP == nil || *got.PodIP != podIP || got.NodeName == nil || *got.NodeName != nodeName { + t.Fatalf("pod network fields = ip:%v node:%v, want %s/%s", got.PodIP, got.NodeName, podIP, nodeName) + } +} + +func requireContainerPort(t *testing.T, container corev1.Container, name string, port int32) { + t.Helper() + for _, got := range container.Ports { + if got.Name == name && got.ContainerPort == port { + return + } + } + t.Fatalf("expected container port %s=%d, got %#v", name, port, container.Ports) +} + +func requireVolumeMount(t *testing.T, container corev1.Container, name, mountPath string) { + t.Helper() + for _, got := range container.VolumeMounts { + if got.Name == name && got.MountPath == mountPath { + return + } + } + t.Fatalf("expected volume mount %s at %s, got %#v", name, mountPath, container.VolumeMounts) +} + +func requireEnv(t *testing.T, container corev1.Container, name, value string) { + t.Helper() + for _, got := range container.Env { + if got.Name == name && got.Value == value { + return + } + } + t.Fatalf("expected env %s=%q, got %#v", name, value, container.Env) +} + +func requireEnvAbsent(t *testing.T, container corev1.Container, name string) { + t.Helper() + for _, got := range container.Env { + if got.Name == name { + t.Fatalf("expected env %s to be absent, got %#v", name, container.Env) + } + } +} + +func requireEnvFieldRef(t *testing.T, container corev1.Container, name, fieldPath string) { + t.Helper() + for _, got := range container.Env { + if got.Name != name || got.ValueFrom == nil || got.ValueFrom.FieldRef == nil { + continue + } + if got.ValueFrom.FieldRef.FieldPath == fieldPath { + return + } + } + t.Fatalf("expected env %s from fieldRef %q, got %#v", name, fieldPath, container.Env) +} diff --git a/backend/internal/services/openclaw_transfer_service.go b/backend/internal/services/openclaw_transfer_service.go index 41af7c4..22b8814 100644 --- a/backend/internal/services/openclaw_transfer_service.go +++ b/backend/internal/services/openclaw_transfer_service.go @@ -7,7 +7,10 @@ import ( "fmt" "io" "strings" + "sync" + "clawreef/internal/models" + "clawreef/internal/repository" "clawreef/internal/services/k8s" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes/scheme" @@ -39,9 +42,29 @@ type OpenClawTransferService interface { } type openClawTransferService struct { - podService *k8s.PodService + podService *k8s.PodService + instanceRepo repository.InstanceRepository + bindingRepo repository.InstanceRuntimeBindingRepository + runtimePodRepo repository.RuntimePodRepository } +type openClawTransferRuntimeRepositories struct { + instanceRepo repository.InstanceRepository + bindingRepo repository.InstanceRuntimeBindingRepository + runtimePodRepo repository.RuntimePodRepository +} + +type transferExecTarget struct { + namespace string + podName string + container string +} + +var ( + openClawTransferReposMu sync.RWMutex + openClawTransferRepos openClawTransferRuntimeRepositories +) + type workspaceTransferSpec struct { dirName string baseDirExpr string @@ -51,11 +74,27 @@ type workspaceTransferSpec struct { } func NewOpenClawTransferService() OpenClawTransferService { + openClawTransferReposMu.RLock() + repos := openClawTransferRepos + openClawTransferReposMu.RUnlock() return &openClawTransferService{ - podService: k8s.NewPodService(), + podService: k8s.NewPodService(), + instanceRepo: repos.instanceRepo, + bindingRepo: repos.bindingRepo, + runtimePodRepo: repos.runtimePodRepo, } } +func SetOpenClawTransferRuntimeRepositories(instanceRepo repository.InstanceRepository, bindingRepo repository.InstanceRuntimeBindingRepository, runtimePodRepo repository.RuntimePodRepository) { + openClawTransferReposMu.Lock() + openClawTransferRepos = openClawTransferRuntimeRepositories{ + instanceRepo: instanceRepo, + bindingRepo: bindingRepo, + runtimePodRepo: runtimePodRepo, + } + openClawTransferReposMu.Unlock() +} + // buildBaseDirExpr returns a POSIX shell expression that resolves the // OpenClaw persistent directory inside the desktop container. It honors the // CLAWMANAGER_AGENT_PERSISTENT_DIR env var (injected by ClawManager at pod @@ -150,21 +189,26 @@ func buildWorkspaceImportCommand(spec workspaceTransferSpec) []string { } func (s *openClawTransferService) Export(ctx context.Context, userID, instanceID int) ([]byte, error) { - return s.exportWorkspace(ctx, userID, instanceID, openClawWorkspaceSpec(), buildExportCommand()) + return s.exportWorkspace(ctx, userID, instanceID, openClawWorkspaceSpec()) } func (s *openClawTransferService) ExportHermes(ctx context.Context, userID, instanceID int) ([]byte, error) { - return s.exportWorkspace(ctx, userID, instanceID, hermesWorkspaceSpec(), buildHermesExportCommand()) + return s.exportWorkspace(ctx, userID, instanceID, hermesWorkspaceSpec()) } -func (s *openClawTransferService) exportWorkspace(ctx context.Context, userID, instanceID int, spec workspaceTransferSpec, command []string) ([]byte, error) { +func (s *openClawTransferService) exportWorkspace(ctx context.Context, userID, instanceID int, spec workspaceTransferSpec) ([]byte, error) { + resolvedSpec, err := s.workspaceSpecForInstance(ctx, userID, instanceID, spec) + if err != nil { + return nil, err + } + command := buildWorkspaceExportCommand(resolvedSpec) var stdout bytes.Buffer var stderr bytes.Buffer if err := s.exec(ctx, userID, instanceID, command, nil, &stdout, &stderr); err != nil { if isExportEmptyWorkspaceError(err) { - return nil, spec.missingErr + return nil, resolvedSpec.missingErr } - return nil, formatExecError("export "+spec.actionLabel, err, stderr.String()) + return nil, formatExecError("export "+resolvedSpec.actionLabel, err, stderr.String()) } return stdout.Bytes(), nil @@ -187,40 +231,76 @@ func isExportEmptyWorkspaceError(err error) bool { } func (s *openClawTransferService) Import(ctx context.Context, userID, instanceID int, archive io.Reader) error { - return s.importWorkspace(ctx, userID, instanceID, archive, openClawWorkspaceSpec(), buildImportCommand()) + return s.importWorkspace(ctx, userID, instanceID, archive, openClawWorkspaceSpec()) } func (s *openClawTransferService) ImportHermes(ctx context.Context, userID, instanceID int, archive io.Reader) error { - return s.importWorkspace(ctx, userID, instanceID, archive, hermesWorkspaceSpec(), buildHermesImportCommand()) + return s.importWorkspace(ctx, userID, instanceID, archive, hermesWorkspaceSpec()) } -func (s *openClawTransferService) importWorkspace(ctx context.Context, userID, instanceID int, archive io.Reader, spec workspaceTransferSpec, command []string) error { +func (s *openClawTransferService) importWorkspace(ctx context.Context, userID, instanceID int, archive io.Reader, spec workspaceTransferSpec) error { + resolvedSpec, err := s.workspaceSpecForInstance(ctx, userID, instanceID, spec) + if err != nil { + return err + } + command := buildWorkspaceImportCommand(resolvedSpec) var stderr bytes.Buffer if err := s.exec(ctx, userID, instanceID, command, archive, nil, &stderr); err != nil { - return formatExecError("import "+spec.actionLabel, err, stderr.String()) + return formatExecError("import "+resolvedSpec.actionLabel, err, stderr.String()) } return nil } +func (s *openClawTransferService) workspaceSpecForInstance(ctx context.Context, userID, instanceID int, spec workspaceTransferSpec) (workspaceTransferSpec, error) { + instance, err := s.runtimeManagedInstance(ctx, userID, instanceID) + if err != nil || instance == nil { + return spec, err + } + baseDir := "" + if instance.WorkspacePath != nil { + baseDir = strings.TrimSpace(*instance.WorkspacePath) + } + if baseDir == "" { + runtimeType, ok := NormalizeV2RuntimeType(instance.Type) + if !ok { + return spec, nil + } + baseDir = RuntimeWorkspacePath(runtimeType, instance.UserID, instance.ID) + } + spec.baseDirExpr = baseDir + return spec, nil +} + func (s *openClawTransferService) exec(ctx context.Context, userID, instanceID int, command []string, stdin io.Reader, stdout, stderr io.Writer) error { if s.podService == nil || s.podService.GetClient() == nil || s.podService.GetClient().Clientset == nil { return fmt.Errorf("k8s client not initialized") } - pod, err := s.podService.GetPod(ctx, userID, instanceID) + target, err := s.runtimeExecTarget(ctx, userID, instanceID) if err != nil { - return fmt.Errorf("failed to get pod: %w", err) + return err + } + if target == nil { + pod, podErr := s.podService.GetPod(ctx, userID, instanceID) + if podErr != nil { + return fmt.Errorf("failed to get pod: %w", podErr) + } + target = &transferExecTarget{ + namespace: pod.Namespace, + podName: pod.Name, + container: "desktop", + } } req := s.podService.GetClient().Clientset.CoreV1().RESTClient().Post(). Resource("pods"). - Name(pod.Name). - Namespace(pod.Namespace). + Name(target.podName). + Namespace(target.namespace). SubResource("exec") req.VersionedParams(&corev1.PodExecOptions{ - Container: "desktop", + Container: target.container, Command: command, Stdin: stdin != nil, Stdout: stdout != nil, @@ -241,6 +321,68 @@ func (s *openClawTransferService) exec(ctx context.Context, userID, instanceID i }) } +func (s *openClawTransferService) runtimeManagedInstance(ctx context.Context, userID, instanceID int) (*models.Instance, error) { + if s == nil || s.instanceRepo == nil { + return nil, nil + } + instance, err := s.instanceRepo.GetByID(instanceID) + if err != nil { + return nil, fmt.Errorf("failed to get instance: %w", err) + } + if instance == nil { + return nil, nil + } + if instance.UserID != userID { + return nil, fmt.Errorf("instance does not belong to user") + } + if !isLiteRuntimeInstance(instance) { + return nil, nil + } + return instance, nil +} + +func (s *openClawTransferService) runtimeExecTarget(ctx context.Context, userID, instanceID int) (*transferExecTarget, error) { + instance, err := s.runtimeManagedInstance(ctx, userID, instanceID) + if err != nil || instance == nil { + return nil, err + } + if s.bindingRepo == nil || s.runtimePodRepo == nil { + return nil, fmt.Errorf("runtime repositories are not configured for lite workspace transfer") + } + binding, err := s.bindingRepo.GetRunningByInstanceID(ctx, instanceID) + if err != nil { + return nil, fmt.Errorf("failed to get runtime binding: %w", err) + } + if binding == nil { + return nil, fmt.Errorf("runtime binding not found for instance %d", instanceID) + } + if binding.Generation != instance.RuntimeGeneration { + return nil, fmt.Errorf("runtime binding generation does not match instance") + } + runtimePod, err := s.runtimePodRepo.GetByID(ctx, binding.RuntimePodID) + if err != nil { + return nil, fmt.Errorf("failed to get runtime pod: %w", err) + } + if runtimePod == nil { + return nil, fmt.Errorf("runtime pod not found for instance %d", instanceID) + } + return &transferExecTarget{ + namespace: runtimePod.Namespace, + podName: runtimePod.PodName, + container: "runtime", + }, nil +} + +func isLiteRuntimeInstance(instance *models.Instance) bool { + if instance == nil { + return false + } + if strings.EqualFold(strings.TrimSpace(instance.InstanceMode), InstanceModeLite) { + return true + } + return strings.EqualFold(strings.TrimSpace(instance.RuntimeType), RuntimeBackendGateway) +} + func shellQuote(value string) string { return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'" } diff --git a/backend/internal/services/openclaw_transfer_service_test.go b/backend/internal/services/openclaw_transfer_service_test.go index b2445b0..8afe957 100644 --- a/backend/internal/services/openclaw_transfer_service_test.go +++ b/backend/internal/services/openclaw_transfer_service_test.go @@ -1,8 +1,11 @@ package services import ( + "context" "strings" "testing" + + "clawreef/internal/models" ) func TestBuildBaseDirExpr_UsesEnvVarWithFallback(t *testing.T) { @@ -110,6 +113,47 @@ func TestBuildHermesImportCommand_PreservesMountedHermesDirectory(t *testing.T) } } +func TestWorkspaceSpecForLiteUsesRuntimeWorkspacePath(t *testing.T) { + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.byID[123] = &models.Instance{ + ID: 123, + UserID: 45, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + RuntimeGeneration: 2, + } + service := &openClawTransferService{instanceRepo: instanceRepo} + + spec, err := service.workspaceSpecForInstance(context.Background(), 45, 123, openClawWorkspaceSpec()) + if err != nil { + t.Fatalf("workspaceSpecForInstance returned error: %v", err) + } + if got, want := spec.baseDirExpr, RuntimeWorkspacePath(RuntimeTypeOpenClaw, 45, 123); got != want { + t.Fatalf("base dir = %q, want %q", got, want) + } +} + +func TestWorkspaceSpecForProKeepsDesktopConfigPath(t *testing.T) { + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.byID[124] = &models.Instance{ + ID: 124, + UserID: 45, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendDesktop, + InstanceMode: InstanceModePro, + } + service := &openClawTransferService{instanceRepo: instanceRepo} + + spec, err := service.workspaceSpecForInstance(context.Background(), 45, 124, openClawWorkspaceSpec()) + if err != nil { + t.Fatalf("workspaceSpecForInstance returned error: %v", err) + } + if got, want := spec.baseDirExpr, buildBaseDirExpr(); got != want { + t.Fatalf("base dir = %q, want %q", got, want) + } +} + func TestShellQuote_HandlesSingleQuotes(t *testing.T) { cases := []struct { in, want string diff --git a/backend/internal/services/redis_client.go b/backend/internal/services/redis_client.go new file mode 100644 index 0000000..b10baae --- /dev/null +++ b/backend/internal/services/redis_client.go @@ -0,0 +1,17 @@ +package services + +import ( + "context" + "time" +) + +type PlatformRedisClient interface { + SetNX(ctx context.Context, key, value string, ttl time.Duration) (bool, error) + Del(ctx context.Context, key string) error + XAdd(ctx context.Context, key string, fields map[string]string) (string, error) + XRead(ctx context.Context, key, lastID string, block time.Duration) ([]redisStreamMessage, error) +} + +func NewPlatformRedisClient(rawURL string) (PlatformRedisClient, error) { + return newRedisBus(rawURL) +} diff --git a/backend/internal/services/risk_hit_service.go b/backend/internal/services/risk_hit_service.go index d198933..28df2d7 100644 --- a/backend/internal/services/risk_hit_service.go +++ b/backend/internal/services/risk_hit_service.go @@ -10,10 +10,17 @@ import ( // RiskHitService defines operations for persisted risk hits. type RiskHitService interface { - RecordHits(traceID string, sessionID, requestID *string, userID, instanceID, invocationID *int, action string, hits []RiskMatch) error + RecordHits(traceID string, sessionID, requestID *string, userID, instanceID, invocationID *int, attribution *RiskHitAttribution, action string, hits []RiskMatch) error ListHitsByTraceID(traceID string) ([]models.RiskHit, error) } +type RiskHitAttribution struct { + InstanceMode *string + RuntimeType *string + GatewayID *string + RuntimePodID *int64 +} + type riskHitService struct { repo repository.RiskHitRepository } @@ -23,7 +30,7 @@ func NewRiskHitService(repo repository.RiskHitRepository) RiskHitService { return &riskHitService{repo: repo} } -func (s *riskHitService) RecordHits(traceID string, sessionID, requestID *string, userID, instanceID, invocationID *int, action string, hits []RiskMatch) error { +func (s *riskHitService) RecordHits(traceID string, sessionID, requestID *string, userID, instanceID, invocationID *int, attribution *RiskHitAttribution, action string, hits []RiskMatch) error { traceID = strings.TrimSpace(traceID) if traceID == "" || len(hits) == 0 { return nil @@ -39,6 +46,10 @@ func (s *riskHitService) RecordHits(traceID string, sessionID, requestID *string RequestID: requestID, UserID: userID, InstanceID: instanceID, + InstanceMode: nil, + RuntimeType: nil, + GatewayID: nil, + RuntimePodID: nil, InvocationID: invocationID, RuleID: strings.TrimSpace(hit.RuleID), RuleName: strings.TrimSpace(hit.RuleName), @@ -46,6 +57,12 @@ func (s *riskHitService) RecordHits(traceID string, sessionID, requestID *string Action: action, MatchSummary: strings.TrimSpace(hit.MatchSummary), } + if attribution != nil { + record.InstanceMode = attribution.InstanceMode + record.RuntimeType = attribution.RuntimeType + record.GatewayID = attribution.GatewayID + record.RuntimePodID = attribution.RuntimePodID + } if record.RuleID == "" || record.RuleName == "" || record.Severity == "" || record.MatchSummary == "" { return fmt.Errorf("risk hit record is incomplete") } diff --git a/backend/internal/services/runtime_agent_client.go b/backend/internal/services/runtime_agent_client.go new file mode 100644 index 0000000..fcf5b0d --- /dev/null +++ b/backend/internal/services/runtime_agent_client.go @@ -0,0 +1,130 @@ +package services + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +type RuntimeAgentClient interface { + Health(ctx context.Context, endpoint string) error + CreateGateway(ctx context.Context, endpoint string, req RuntimeAgentCreateGatewayRequest) (*RuntimeAgentCreateGatewayResponse, error) + DeleteGateway(ctx context.Context, endpoint, gatewayID string) error + Drain(ctx context.Context, endpoint string) error +} + +type RuntimeAgentPortRange struct { + Start int `json:"start"` + End int `json:"end"` +} + +type RuntimeAgentCreateGatewayRequest struct { + InstanceID int `json:"instance_id"` + UserID int `json:"user_id"` + AgentType string `json:"agent_type"` + WorkspacePath string `json:"workspace_path"` + PortRange RuntimeAgentPortRange `json:"port_range"` + UID int `json:"uid"` + GID int `json:"gid"` + CPUCores float64 `json:"cpu_cores"` + MemoryMB int `json:"memory_mb"` + DiskQuotaMB int `json:"disk_quota_mb"` + Generation int `json:"generation"` + Environment map[string]string `json:"environment,omitempty"` +} + +type RuntimeAgentCreateGatewayResponse struct { + GatewayID string `json:"gateway_id"` + Port int `json:"port"` + PID *int `json:"pid,omitempty"` + Status string `json:"status"` +} + +type runtimeAgentHTTPClient struct { + controlToken string + httpClient *http.Client +} + +func NewRuntimeAgentClient(controlToken string) RuntimeAgentClient { + return NewRuntimeAgentClientWithHTTPClient(controlToken, nil) +} + +func NewRuntimeAgentClientWithHTTPClient(controlToken string, httpClient *http.Client) RuntimeAgentClient { + if httpClient == nil { + httpClient = &http.Client{ + Timeout: 30 * time.Second, + } + } + clientCopy := *httpClient + clientCopy.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + } + return &runtimeAgentHTTPClient{ + controlToken: controlToken, + httpClient: &clientCopy, + } +} + +func (c *runtimeAgentHTTPClient) Health(ctx context.Context, endpoint string) error { + return c.do(ctx, http.MethodGet, endpoint, "/v1/health", nil, nil) +} + +func (c *runtimeAgentHTTPClient) CreateGateway(ctx context.Context, endpoint string, req RuntimeAgentCreateGatewayRequest) (*RuntimeAgentCreateGatewayResponse, error) { + var resp RuntimeAgentCreateGatewayResponse + if err := c.do(ctx, http.MethodPost, endpoint, "/v1/gateways", req, &resp); err != nil { + return nil, err + } + return &resp, nil +} + +func (c *runtimeAgentHTTPClient) DeleteGateway(ctx context.Context, endpoint, gatewayID string) error { + return c.do(ctx, http.MethodDelete, endpoint, "/v1/gateways/"+url.PathEscape(gatewayID), nil, nil) +} + +func (c *runtimeAgentHTTPClient) Drain(ctx context.Context, endpoint string) error { + return c.do(ctx, http.MethodPost, endpoint, "/v1/drain", map[string]bool{"draining": true}, nil) +} + +func (c *runtimeAgentHTTPClient) do(ctx context.Context, method, endpoint, path string, body any, out any) error { + endpoint = strings.TrimRight(endpoint, "/") + var reader io.Reader + if body != nil { + payload, err := json.Marshal(body) + if err != nil { + return err + } + reader = bytes.NewReader(payload) + } + + req, err := http.NewRequestWithContext(ctx, method, endpoint+path, reader) + if err != nil { + return err + } + req.Header.Set("X-ClawManager-Control-Token", c.controlToken) + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + if resp.StatusCode == http.StatusConflict { + return fmt.Errorf("runtime agent conflict: %s", string(msg)) + } + return fmt.Errorf("runtime agent status %d: %s", resp.StatusCode, string(msg)) + } + if out == nil { + return nil + } + return json.NewDecoder(resp.Body).Decode(out) +} diff --git a/backend/internal/services/runtime_agent_client_test.go b/backend/internal/services/runtime_agent_client_test.go new file mode 100644 index 0000000..fc461bf --- /dev/null +++ b/backend/internal/services/runtime_agent_client_test.go @@ -0,0 +1,223 @@ +package services + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestRuntimeAgentClientCreateGateway(t *testing.T) { + var gotToken string + var gotReq RuntimeAgentCreateGatewayRequest + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/gateways" { + t.Fatalf("unexpected path %s", r.URL.Path) + } + gotToken = r.Header.Get("X-ClawManager-Control-Token") + if err := json.NewDecoder(r.Body).Decode(&gotReq); err != nil { + t.Fatalf("decode request: %v", err) + } + _ = json.NewEncoder(w).Encode(RuntimeAgentCreateGatewayResponse{ + GatewayID: "gw-7-3", + Port: 20017, + PID: runtimeAgentIntPtr(8842), + Status: "running", + }) + })) + defer server.Close() + + client := NewRuntimeAgentClient("secret") + resp, err := client.CreateGateway(context.Background(), server.URL, RuntimeAgentCreateGatewayRequest{ + InstanceID: 7, + UserID: 8, + AgentType: "openclaw", + WorkspacePath: "/workspaces/openclaw/user-8/instance-7", + PortRange: RuntimeAgentPortRange{Start: 20000, End: 20099}, + UID: 200007, + GID: 200007, + CPUCores: 2, + MemoryMB: 4096, + DiskQuotaMB: 20480, + Generation: 3, + }) + if err != nil { + t.Fatalf("CreateGateway returned error: %v", err) + } + if gotToken != "secret" { + t.Fatalf("unexpected token %q", gotToken) + } + if gotReq.InstanceID != 7 || gotReq.PortRange.Start != 20000 { + t.Fatalf("unexpected request %#v", gotReq) + } + if resp.GatewayID != "gw-7-3" || resp.Port != 20017 { + t.Fatalf("unexpected response %#v", resp) + } +} + +func TestRuntimeAgentClientCreateGatewayTrimsEndpointSlash(t *testing.T) { + var calls int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if r.URL.Path != "/v1/gateways" { + t.Fatalf("unexpected path %s", r.URL.Path) + } + _ = json.NewEncoder(w).Encode(RuntimeAgentCreateGatewayResponse{ + GatewayID: "gw-7-3", + Port: 20017, + Status: "running", + }) + })) + defer server.Close() + + client := NewRuntimeAgentClient("secret") + if _, err := client.CreateGateway(context.Background(), server.URL+"/", RuntimeAgentCreateGatewayRequest{}); err != nil { + t.Fatalf("CreateGateway returned error: %v", err) + } + if calls != 1 { + t.Fatalf("expected one gateway create call, got %d", calls) + } +} + +func TestRuntimeAgentClientDefaultTimeoutAllowsGatewayStartup(t *testing.T) { + client := NewRuntimeAgentClient("secret") + agentClient, ok := client.(*runtimeAgentHTTPClient) + if !ok { + t.Fatalf("unexpected client type %T", client) + } + if agentClient.httpClient.Timeout != 30*time.Second { + t.Fatalf("default timeout = %v, want 30s", agentClient.httpClient.Timeout) + } +} + +func TestRuntimeAgentClientDeleteGatewayEscapesGatewayID(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + t.Fatalf("unexpected method %s", r.Method) + } + if r.URL.EscapedPath() != "/v1/gateways/gw-7%2F3" { + t.Fatalf("unexpected escaped path %s", r.URL.EscapedPath()) + } + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + client := NewRuntimeAgentClient("secret") + if err := client.DeleteGateway(context.Background(), server.URL, "gw-7/3"); err != nil { + t.Fatalf("DeleteGateway returned error: %v", err) + } +} + +func TestRuntimeAgentClientHealthHandlesSuccessAndRedirect(t *testing.T) { + successServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/health" { + t.Fatalf("unexpected success path %s", r.URL.Path) + } + w.WriteHeader(http.StatusNoContent) + })) + defer successServer.Close() + redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v1/health": + w.Header().Set("Location", "/ok") + w.WriteHeader(http.StatusFound) + _, _ = w.Write([]byte("redirect not allowed")) + case "/ok": + w.WriteHeader(http.StatusOK) + default: + t.Fatalf("unexpected redirect path %s", r.URL.Path) + } + })) + defer redirectServer.Close() + + client := NewRuntimeAgentClient("secret") + if err := client.Health(context.Background(), successServer.URL); err != nil { + t.Fatalf("Health returned error for 204: %v", err) + } + err := client.Health(context.Background(), redirectServer.URL) + if err == nil || !strings.Contains(err.Error(), "runtime agent status 302") || !strings.Contains(err.Error(), "redirect not allowed") { + t.Fatalf("unexpected redirect error %v", err) + } +} + +func TestRuntimeAgentClientWithHTTPClientDoesNotMutateRedirectPolicy(t *testing.T) { + supplied := &http.Client{Timeout: 10 * time.Second} + client := NewRuntimeAgentClientWithHTTPClient("secret", supplied) + + if supplied.CheckRedirect != nil { + t.Fatalf("constructor mutated supplied CheckRedirect") + } + + agentClient, ok := client.(*runtimeAgentHTTPClient) + if !ok { + t.Fatalf("unexpected client type %T", client) + } + if agentClient.httpClient == supplied { + t.Fatalf("constructor reused supplied client pointer") + } + if agentClient.httpClient.Timeout != supplied.Timeout { + t.Fatalf("timeout was not preserved: got %v want %v", agentClient.httpClient.Timeout, supplied.Timeout) + } + if agentClient.httpClient.CheckRedirect == nil { + t.Fatalf("copied client has no redirect policy") + } +} + +func TestRuntimeAgentClientDrainSendsJSONBody(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/drain" { + t.Fatalf("unexpected path %s", r.URL.Path) + } + if r.Header.Get("Content-Type") != "application/json" { + t.Fatalf("unexpected content type %q", r.Header.Get("Content-Type")) + } + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("read body: %v", err) + } + if strings.TrimSpace(string(body)) != `{"draining":true}` { + t.Fatalf("unexpected body %s", body) + } + w.WriteHeader(http.StatusAccepted) + })) + defer server.Close() + + client := NewRuntimeAgentClient("secret") + if err := client.Drain(context.Background(), server.URL); err != nil { + t.Fatalf("Drain returned error: %v", err) + } +} + +func TestRuntimeAgentClientConflict(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "no free port", http.StatusConflict) + })) + defer server.Close() + + client := NewRuntimeAgentClient("secret") + _, err := client.CreateGateway(context.Background(), server.URL, RuntimeAgentCreateGatewayRequest{}) + if err == nil || !strings.Contains(err.Error(), "runtime agent conflict") || !strings.Contains(err.Error(), "no free port") { + t.Fatalf("unexpected error %v", err) + } +} + +func TestRuntimeAgentClientNonConflictErrorIncludesStatusAndBody(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "agent exploded", http.StatusInternalServerError) + })) + defer server.Close() + + client := NewRuntimeAgentClient("secret") + _, err := client.CreateGateway(context.Background(), server.URL, RuntimeAgentCreateGatewayRequest{}) + if err == nil || !strings.Contains(err.Error(), "runtime agent status 500") || !strings.Contains(err.Error(), "agent exploded") { + t.Fatalf("unexpected error %v", err) + } +} + +func runtimeAgentIntPtr(v int) *int { + return &v +} diff --git a/backend/internal/services/runtime_capacity.go b/backend/internal/services/runtime_capacity.go new file mode 100644 index 0000000..e5a59b7 --- /dev/null +++ b/backend/internal/services/runtime_capacity.go @@ -0,0 +1,80 @@ +package services + +import ( + "fmt" + "path" + "strings" +) + +const ( + RuntimeTypeOpenClaw = "openclaw" + RuntimeTypeHermes = "hermes" + + InstanceModeLite = "lite" + InstanceModePro = "pro" + + RuntimeBackendGateway = "gateway" + RuntimeBackendDesktop = "desktop" + RuntimeBackendShell = "shell" + + RuntimeGatewayPortStart = 20000 + RuntimeGatewayPortEnd = 20099 + RuntimePodCapacity = 100 + RuntimeLinuxIDBase = 200000 +) + +func NormalizeV2RuntimeType(instanceType string) (string, bool) { + switch strings.ToLower(strings.TrimSpace(instanceType)) { + case RuntimeTypeOpenClaw: + return RuntimeTypeOpenClaw, true + case RuntimeTypeHermes: + return RuntimeTypeHermes, true + default: + return "", false + } +} + +func NormalizeInstanceMode(mode string) (string, bool) { + switch strings.ToLower(strings.TrimSpace(mode)) { + case InstanceModeLite: + return InstanceModeLite, true + case InstanceModePro: + return InstanceModePro, true + default: + return "", false + } +} + +func RuntimeTypeForInstanceMode(mode string) (string, bool) { + normalized, ok := NormalizeInstanceMode(mode) + if !ok { + return "", false + } + if normalized == InstanceModeLite { + return RuntimeBackendGateway, true + } + return RuntimeBackendDesktop, true +} + +func InstanceModeForRuntimeType(runtimeType string) string { + if strings.EqualFold(strings.TrimSpace(runtimeType), RuntimeBackendGateway) { + return InstanceModeLite + } + return InstanceModePro +} + +func RuntimeWorkspacePath(runtimeType string, userID int, instanceID int) string { + return RuntimeWorkspacePathWithRoot("/workspaces", runtimeType, userID, instanceID) +} + +func RuntimeWorkspacePathWithRoot(root, runtimeType string, userID int, instanceID int) string { + root = strings.TrimSpace(root) + if root == "" { + root = "/workspaces" + } + return path.Join(root, fmt.Sprintf("%s/user-%d/instance-%d", runtimeType, userID, instanceID)) +} + +func RuntimeLinuxID(instanceID int) int { + return RuntimeLinuxIDBase + instanceID +} diff --git a/backend/internal/services/runtime_capacity_test.go b/backend/internal/services/runtime_capacity_test.go new file mode 100644 index 0000000..3435615 --- /dev/null +++ b/backend/internal/services/runtime_capacity_test.go @@ -0,0 +1,52 @@ +package services + +import "testing" + +func TestNormalizeV2RuntimeTypeAcceptsManagedRuntimeTypes(t *testing.T) { + for _, input := range []string{"openclaw", " OpenClaw ", "hermes", "Hermes"} { + got, ok := NormalizeV2RuntimeType(input) + if !ok { + t.Fatalf("expected %q to be accepted", input) + } + if got != RuntimeTypeOpenClaw && got != RuntimeTypeHermes { + t.Fatalf("expected normalized managed runtime type, got %q", got) + } + } +} + +func TestNormalizeV2RuntimeTypeRejectsLegacyRuntimeTypes(t *testing.T) { + for _, input := range []string{"webtop", "ubuntu", "", "desktop", "shell"} { + if got, ok := NormalizeV2RuntimeType(input); ok { + t.Fatalf("expected %q to be rejected, got %q", input, got) + } + } +} + +func TestRuntimeWorkspacePath(t *testing.T) { + got := RuntimeWorkspacePath("openclaw", 45, 123) + want := "/workspaces/openclaw/user-45/instance-123" + if got != want { + t.Fatalf("expected workspace path %q, got %q", want, got) + } +} + +func TestRuntimeLinuxID(t *testing.T) { + if got, want := RuntimeLinuxID(123), 200123; got != want { + t.Fatalf("expected linux id %d, got %d", want, got) + } +} + +func TestInstanceModeRuntimeTypeMapping(t *testing.T) { + if got, ok := RuntimeTypeForInstanceMode(" lite "); !ok || got != RuntimeBackendGateway { + t.Fatalf("lite runtime type = %q/%v, want gateway/true", got, ok) + } + if got, ok := RuntimeTypeForInstanceMode("Pro"); !ok || got != RuntimeBackendDesktop { + t.Fatalf("pro runtime type = %q/%v, want desktop/true", got, ok) + } + if got := InstanceModeForRuntimeType(RuntimeBackendGateway); got != InstanceModeLite { + t.Fatalf("gateway mode = %q, want lite", got) + } + if got := InstanceModeForRuntimeType(RuntimeBackendDesktop); got != InstanceModePro { + t.Fatalf("desktop mode = %q, want pro", got) + } +} diff --git a/backend/internal/services/runtime_events.go b/backend/internal/services/runtime_events.go new file mode 100644 index 0000000..d0f1f5b --- /dev/null +++ b/backend/internal/services/runtime_events.go @@ -0,0 +1,68 @@ +package services + +import ( + "context" + "encoding/json" + "time" +) + +const runtimeEventStreamKey = "clawmanager:runtime-events" + +type RuntimeEvent struct { + Type string `json:"type"` + Payload json.RawMessage `json:"payload"` + CreatedAt time.Time `json:"created_at"` +} + +type RuntimeEventService interface { + Publish(ctx context.Context, eventType string, payload any) error + Read(ctx context.Context, lastID string, block time.Duration) ([]redisStreamMessage, error) +} + +func NewRuntimeEventService(redis PlatformRedisClient) RuntimeEventService { + if redis == nil { + return noopRuntimeEventService{} + } + return &redisRuntimeEventService{redis: redis} +} + +type redisRuntimeEventService struct { + redis PlatformRedisClient +} + +func (s *redisRuntimeEventService) Publish(ctx context.Context, eventType string, payload any) error { + payloadJSON, err := json.Marshal(payload) + if err != nil { + return err + } + event := RuntimeEvent{ + Type: eventType, + Payload: payloadJSON, + CreatedAt: time.Now().UTC(), + } + eventJSON, err := json.Marshal(event) + if err != nil { + return err + } + _, err = s.redis.XAdd(ctx, runtimeEventStreamKey, map[string]string{ + "type": event.Type, + "payload": string(event.Payload), + "created_at": event.CreatedAt.Format(time.RFC3339Nano), + "event": string(eventJSON), + }) + return err +} + +func (s *redisRuntimeEventService) Read(ctx context.Context, lastID string, block time.Duration) ([]redisStreamMessage, error) { + return s.redis.XRead(ctx, runtimeEventStreamKey, lastID, block) +} + +type noopRuntimeEventService struct{} + +func (noopRuntimeEventService) Publish(ctx context.Context, eventType string, payload any) error { + return nil +} + +func (noopRuntimeEventService) Read(ctx context.Context, lastID string, block time.Duration) ([]redisStreamMessage, error) { + return nil, nil +} diff --git a/backend/internal/services/runtime_events_test.go b/backend/internal/services/runtime_events_test.go new file mode 100644 index 0000000..51b1dd4 --- /dev/null +++ b/backend/internal/services/runtime_events_test.go @@ -0,0 +1,75 @@ +package services + +import ( + "context" + "encoding/json" + "testing" + "time" +) + +func TestRuntimeEventServiceNoopWhenRedisNil(t *testing.T) { + service := NewRuntimeEventService(nil) + + if err := service.Publish(context.Background(), "runtime.test", map[string]string{"ok": "true"}); err != nil { + t.Fatalf("Publish returned error: %v", err) + } + messages, err := service.Read(context.Background(), "0", time.Millisecond) + if err != nil { + t.Fatalf("Read returned error: %v", err) + } + if messages != nil { + t.Fatalf("messages = %#v, want nil", messages) + } +} + +func TestRuntimeEventServicePublishesJSONFields(t *testing.T) { + redis := &fakePlatformRedisClient{} + service := NewRuntimeEventService(redis) + + if err := service.Publish(context.Background(), "runtime.instance.running", map[string]int{"instance_id": 17}); err != nil { + t.Fatalf("Publish returned error: %v", err) + } + if redis.xaddKey != runtimeEventStreamKey { + t.Fatalf("XAdd key = %q", redis.xaddKey) + } + if redis.xaddFields["type"] != "runtime.instance.running" { + t.Fatalf("type field = %q", redis.xaddFields["type"]) + } + var payload map[string]int + if err := json.Unmarshal([]byte(redis.xaddFields["payload"]), &payload); err != nil { + t.Fatalf("payload is not json: %v", err) + } + if payload["instance_id"] != 17 { + t.Fatalf("payload instance_id = %d", payload["instance_id"]) + } + var event RuntimeEvent + if err := json.Unmarshal([]byte(redis.xaddFields["event"]), &event); err != nil { + t.Fatalf("event is not json: %v", err) + } + if event.Type != "runtime.instance.running" { + t.Fatalf("event type = %q", event.Type) + } +} + +type fakePlatformRedisClient struct { + xaddKey string + xaddFields map[string]string +} + +func (c *fakePlatformRedisClient) SetNX(ctx context.Context, key, value string, ttl time.Duration) (bool, error) { + return true, nil +} + +func (c *fakePlatformRedisClient) Del(ctx context.Context, key string) error { + return nil +} + +func (c *fakePlatformRedisClient) XAdd(ctx context.Context, key string, fields map[string]string) (string, error) { + c.xaddKey = key + c.xaddFields = fields + return "1-0", nil +} + +func (c *fakePlatformRedisClient) XRead(ctx context.Context, key, lastID string, block time.Duration) ([]redisStreamMessage, error) { + return []redisStreamMessage{{ID: "1-0", Fields: map[string]string{"type": "runtime.test"}}}, nil +} diff --git a/backend/internal/services/runtime_leader.go b/backend/internal/services/runtime_leader.go new file mode 100644 index 0000000..26d994e --- /dev/null +++ b/backend/internal/services/runtime_leader.go @@ -0,0 +1,99 @@ +package services + +import ( + "context" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + runtimeLeaderLeaseName = "clawmanager-runtime-scheduler" + runtimeLeaderLeaseTTL = 15 * time.Second +) + +type RuntimeLeaderService interface { + IsLeader(ctx context.Context) bool +} + +type runtimeLeaderService struct { + client kubernetes.Interface + namespace string + holder string +} + +func NewRuntimeLeaderService(client kubernetes.Interface, namespace, holder string) RuntimeLeaderService { + return &runtimeLeaderService{ + client: client, + namespace: namespace, + holder: holder, + } +} + +func (s *runtimeLeaderService) IsLeader(ctx context.Context) bool { + if s == nil || s.client == nil || s.namespace == "" || s.holder == "" { + return false + } + + leases := s.client.CoordinationV1().Leases(s.namespace) + now := metav1.NewMicroTime(time.Now()) + lease, err := leases.Get(ctx, runtimeLeaderLeaseName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + _, createErr := leases.Create(ctx, s.newLease(now), metav1.CreateOptions{}) + if createErr == nil { + return true + } + if !errors.IsAlreadyExists(createErr) { + return false + } + lease, err = leases.Get(ctx, runtimeLeaderLeaseName, metav1.GetOptions{}) + } + if err != nil { + return false + } + + if !s.canAcquire(lease, now.Time) { + return false + } + + lease.Spec.HolderIdentity = &s.holder + lease.Spec.RenewTime = &now + ttlSeconds := int32(runtimeLeaderLeaseTTL / time.Second) + lease.Spec.LeaseDurationSeconds = &ttlSeconds + if _, err := leases.Update(ctx, lease, metav1.UpdateOptions{}); err != nil { + return false + } + return true +} + +func (s *runtimeLeaderService) newLease(now metav1.MicroTime) *coordinationv1.Lease { + ttlSeconds := int32(runtimeLeaderLeaseTTL / time.Second) + return &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: runtimeLeaderLeaseName, + Namespace: s.namespace, + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: &s.holder, + AcquireTime: &now, + RenewTime: &now, + LeaseDurationSeconds: &ttlSeconds, + }, + } +} + +func (s *runtimeLeaderService) canAcquire(lease *coordinationv1.Lease, now time.Time) bool { + if lease == nil { + return true + } + if lease.Spec.HolderIdentity != nil && *lease.Spec.HolderIdentity == s.holder { + return true + } + if lease.Spec.RenewTime == nil { + return true + } + return lease.Spec.RenewTime.Add(runtimeLeaderLeaseTTL).Before(now) +} diff --git a/backend/internal/services/runtime_leader_test.go b/backend/internal/services/runtime_leader_test.go new file mode 100644 index 0000000..fe8e1e4 --- /dev/null +++ b/backend/internal/services/runtime_leader_test.go @@ -0,0 +1,67 @@ +package services + +import ( + "context" + "testing" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +func TestRuntimeLeaderServiceFirstCallerBecomesLeader(t *testing.T) { + service := NewRuntimeLeaderService(fake.NewSimpleClientset(), "runtime-system", "backend-a") + + if !service.IsLeader(context.Background()) { + t.Fatalf("expected first caller to become leader") + } +} + +func TestRuntimeLeaderServiceDifferentCallerCannotStealUnexpiredLease(t *testing.T) { + now := metav1.NewMicroTime(time.Now()) + holder := "backend-a" + client := fake.NewSimpleClientset(&coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{Name: "clawmanager-runtime-scheduler", Namespace: "runtime-system"}, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: &holder, + RenewTime: &now, + LeaseDurationSeconds: int32Ptr(15), + }, + }) + service := NewRuntimeLeaderService(client, "runtime-system", "backend-b") + + if service.IsLeader(context.Background()) { + t.Fatalf("expected different caller not to steal unexpired lease") + } +} + +func TestRuntimeLeaderServiceDifferentCallerCanAcquireExpiredLease(t *testing.T) { + renewTime := metav1.NewMicroTime(time.Now().Add(-30 * time.Second)) + holder := "backend-a" + client := fake.NewSimpleClientset(&coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{Name: "clawmanager-runtime-scheduler", Namespace: "runtime-system"}, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: &holder, + RenewTime: &renewTime, + LeaseDurationSeconds: int32Ptr(15), + }, + }) + service := NewRuntimeLeaderService(client, "runtime-system", "backend-b") + + if !service.IsLeader(context.Background()) { + t.Fatalf("expected different caller to acquire expired lease") + } + + lease, err := client.CoordinationV1().Leases("runtime-system").Get(context.Background(), "clawmanager-runtime-scheduler", metav1.GetOptions{}) + if err != nil { + t.Fatalf("failed to load lease: %v", err) + } + if lease.Spec.HolderIdentity == nil || *lease.Spec.HolderIdentity != "backend-b" { + t.Fatalf("expected lease holder backend-b, got %#v", lease.Spec.HolderIdentity) + } +} + +func int32Ptr(value int32) *int32 { + return &value +} diff --git a/backend/internal/services/runtime_scheduler.go b/backend/internal/services/runtime_scheduler.go new file mode 100644 index 0000000..a4dbf6c --- /dev/null +++ b/backend/internal/services/runtime_scheduler.go @@ -0,0 +1,880 @@ +package services + +import ( + "context" + "errors" + "fmt" + "log" + "strconv" + "strings" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" + "clawreef/internal/services/k8s" +) + +type RuntimeScheduler struct { + instanceRepo repository.InstanceRepository + podRepo repository.RuntimePodRepository + bindingRepo repository.InstanceRuntimeBindingRepository + rolloutRepo repository.RuntimeRolloutRepository + agentClient RuntimeAgentClient + events RuntimeEventService + leader RuntimeLeaderService + deployments k8s.RuntimeDeploymentService + envBuilder RuntimeGatewayEnvBuilder + tick time.Duration + + workspaceRoot string + runtimeNamespace string + gatewayPortStart int + gatewayPortEnd int + heartbeatTimeout time.Duration + maxGatewaysPerPod int +} + +var errRuntimeScaleOutPending = errors.New("runtime scale-out pending") + +const runtimeRolloutStaleWindowMultiplier = 3 + +type RuntimeGatewayEnvBuilder func(*models.Instance) (map[string]string, error) + +type RuntimeSchedulerOption func(*RuntimeScheduler) + +func WithRuntimeSchedulerWorkspaceRoot(root string) RuntimeSchedulerOption { + return func(s *RuntimeScheduler) { + if strings.TrimSpace(root) != "" { + s.workspaceRoot = strings.TrimSpace(root) + } + } +} + +func WithRuntimeSchedulerGatewayPortRange(start, end int) RuntimeSchedulerOption { + return func(s *RuntimeScheduler) { + if start > 0 { + s.gatewayPortStart = start + } + if end > 0 { + s.gatewayPortEnd = end + } + if s.gatewayPortEnd < s.gatewayPortStart { + s.gatewayPortEnd = s.gatewayPortStart + } + } +} + +func WithRuntimeSchedulerHeartbeatTimeout(timeout time.Duration) RuntimeSchedulerOption { + return func(s *RuntimeScheduler) { + s.heartbeatTimeout = timeout + } +} + +func WithRuntimeSchedulerNamespace(namespace string) RuntimeSchedulerOption { + return func(s *RuntimeScheduler) { + if strings.TrimSpace(namespace) != "" { + s.runtimeNamespace = strings.TrimSpace(namespace) + } + } +} + +func WithRuntimeSchedulerMaxGatewaysPerPod(capacity int) RuntimeSchedulerOption { + return func(s *RuntimeScheduler) { + if capacity > 0 { + s.maxGatewaysPerPod = capacity + } + } +} + +func WithRuntimeSchedulerGatewayEnvBuilder(builder RuntimeGatewayEnvBuilder) RuntimeSchedulerOption { + return func(s *RuntimeScheduler) { + s.envBuilder = builder + } +} + +func NewRuntimeScheduler( + instanceRepo repository.InstanceRepository, + podRepo repository.RuntimePodRepository, + bindingRepo repository.InstanceRuntimeBindingRepository, + rolloutRepo repository.RuntimeRolloutRepository, + agentClient RuntimeAgentClient, + events RuntimeEventService, + leader RuntimeLeaderService, + deployments k8s.RuntimeDeploymentService, + tick time.Duration, + opts ...RuntimeSchedulerOption, +) *RuntimeScheduler { + if tick <= 0 { + tick = 2 * time.Second + } + s := &RuntimeScheduler{ + instanceRepo: instanceRepo, + podRepo: podRepo, + bindingRepo: bindingRepo, + rolloutRepo: rolloutRepo, + agentClient: agentClient, + events: events, + leader: leader, + deployments: deployments, + tick: tick, + workspaceRoot: "/workspaces", + runtimeNamespace: "clawmanager-system", + gatewayPortStart: RuntimeGatewayPortStart, + gatewayPortEnd: RuntimeGatewayPortEnd, + heartbeatTimeout: 10 * time.Second, + maxGatewaysPerPod: RuntimePodCapacity, + } + for _, opt := range opts { + opt(s) + } + return s +} + +func (s *RuntimeScheduler) Start(ctx context.Context) { + if s == nil { + return + } + go func() { + ticker := time.NewTicker(s.tick) + defer ticker.Stop() + + s.reconcileIfLeader(ctx) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s.reconcileIfLeader(ctx) + } + } + }() +} + +func (s *RuntimeScheduler) HeartbeatTimeout() time.Duration { + if s == nil { + return 0 + } + return s.heartbeatTimeout +} + +func (s *RuntimeScheduler) DrainPod(ctx context.Context, podID int64) error { + if s == nil || s.podRepo == nil { + return fmt.Errorf("runtime scheduler pod repository is not configured") + } + pod, err := s.podRepo.GetByID(ctx, podID) + if err != nil { + return err + } + if pod == nil { + return fmt.Errorf("runtime pod %d not found", podID) + } + if pod.AgentEndpoint != nil && strings.TrimSpace(*pod.AgentEndpoint) != "" && s.agentClient != nil { + if err := s.agentClient.Drain(ctx, strings.TrimSpace(*pod.AgentEndpoint)); err != nil { + return err + } + } + return s.podRepo.MarkState(ctx, podID, "draining", true) +} + +func (s *RuntimeScheduler) FailoverPod(ctx context.Context, podID int64, reason string) error { + if s == nil || s.podRepo == nil || s.bindingRepo == nil || s.instanceRepo == nil { + return fmt.Errorf("runtime scheduler dependencies are not configured") + } + var errs []error + if err := s.podRepo.MarkState(ctx, podID, "unhealthy", false); err != nil { + errs = append(errs, err) + } + + bindings, err := s.bindingRepo.ListByRuntimePodID(ctx, podID) + if err != nil { + errs = append(errs, err) + return errors.Join(errs...) + } + for _, binding := range bindings { + nextGeneration := binding.Generation + 1 + instance, err := s.instanceRepo.GetByID(binding.InstanceID) + if err != nil { + errs = append(errs, err) + continue + } + if instance != nil { + nextGeneration = instance.RuntimeGeneration + 1 + } + message := reason + if err := s.instanceRepo.UpdateRuntimeState(ctx, binding.InstanceID, "creating", nextGeneration, &message); err != nil { + errs = append(errs, err) + continue + } + if err := s.bindingRepo.DeleteByInstanceID(ctx, binding.InstanceID); err != nil { + errs = append(errs, err) + continue + } + if err := s.podRepo.ReleaseSlot(ctx, podID); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +func (s *RuntimeScheduler) StartRollout(ctx context.Context, rolloutID int64) error { + if s == nil || s.rolloutRepo == nil || s.podRepo == nil { + return fmt.Errorf("runtime scheduler rollout dependencies are not configured") + } + rollout, err := s.rolloutRepo.GetByID(ctx, rolloutID) + if err != nil { + return err + } + if rollout == nil { + return fmt.Errorf("runtime rollout %d not found", rolloutID) + } + + startedAt := time.Now().UTC() + if err := s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "running", &startedAt, nil, nil); err != nil { + return err + } + + allPods, err := s.podRepo.List(ctx, rollout.RuntimeType) + if err != nil { + message := err.Error() + _ = s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "error", &startedAt, nil, &message) + return err + } + currentPods := s.currentRuntimePods(allPods, time.Now().UTC()) + if runtimePodsAlreadyAtImage(currentPods, rollout.RuntimeType, rollout.TargetImageRef) { + finishedAt := time.Now().UTC() + return s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "finished", &startedAt, &finishedAt, nil) + } + batchSize := rollout.BatchSize + if batchSize <= 0 { + batchSize = 1 + } + maxUnavailable := rollout.MaxUnavailable + if maxUnavailable <= 0 { + maxUnavailable = 1 + } + if err := s.rolloutRuntimeDeployments(ctx, rollout, allPods, maxUnavailable, batchSize); err != nil { + message := err.Error() + _ = s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "error", &startedAt, nil, &message) + return err + } + unavailable := 0 + readyCandidates := 0 + for _, pod := range currentPods { + if pod.Draining || pod.State != "ready" { + unavailable++ + continue + } + readyCandidates++ + } + remaining := maxUnavailable - unavailable + if remaining <= 0 { + return nil + } + drainLimit := minInt(batchSize, remaining) + var errs []error + drained := 0 + for _, pod := range currentPods { + if drained >= drainLimit { + break + } + if pod.State != "ready" || pod.Draining { + continue + } + if err := s.DrainPod(ctx, pod.ID); err != nil { + errs = append(errs, err) + continue + } + drained++ + } + if len(errs) > 0 { + joined := errors.Join(errs...) + message := joined.Error() + _ = s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "error", &startedAt, nil, &message) + return joined + } + if drained == 0 && readyCandidates == 0 && unavailable == 0 && len(currentPods) > 0 { + finishedAt := time.Now().UTC() + return s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "finished", &startedAt, &finishedAt, nil) + } + return nil +} + +func (s *RuntimeScheduler) reconcileRollouts(ctx context.Context) error { + if s == nil || s.rolloutRepo == nil { + return nil + } + rollouts, err := s.rolloutRepo.ListActive(ctx, "") + if err != nil { + return fmt.Errorf("list active runtime rollouts: %w", err) + } + var errs []error + for _, rollout := range rollouts { + switch rollout.Status { + case "pending": + if err := s.StartRollout(ctx, rollout.ID); err != nil { + errs = append(errs, fmt.Errorf("start runtime rollout %d: %w", rollout.ID, err)) + } + case "running": + if err := s.finishRolloutIfReady(ctx, rollout); err != nil { + errs = append(errs, fmt.Errorf("finish runtime rollout %d: %w", rollout.ID, err)) + } + } + } + return errors.Join(errs...) +} + +func (s *RuntimeScheduler) rolloutRuntimeDeployments(ctx context.Context, rollout *models.RuntimeRollout, pods []models.RuntimePod, maxUnavailable, maxSurge int) error { + if s == nil || s.deployments == nil || rollout == nil { + return nil + } + targetImage := strings.TrimSpace(rollout.TargetImageRef) + if targetImage == "" { + return nil + } + type deploymentRef struct { + namespace string + name string + } + refs := map[deploymentRef]struct{}{} + for _, pod := range pods { + if pod.RuntimeType != rollout.RuntimeType { + continue + } + namespace := strings.TrimSpace(pod.Namespace) + name := strings.TrimSpace(pod.DeploymentName) + if namespace == "" || name == "" { + continue + } + refs[deploymentRef{namespace: namespace, name: name}] = struct{}{} + } + if len(refs) == 0 { + name := defaultRuntimeDeploymentName(rollout.RuntimeType) + if name == "" { + return fmt.Errorf("no runtime deployment found for %s rollout", rollout.RuntimeType) + } + refs[deploymentRef{namespace: s.runtimeNamespace, name: name}] = struct{}{} + } + var errs []error + for ref := range refs { + if err := s.deployments.RolloutImage(ctx, ref.namespace, ref.name, targetImage, maxUnavailable, maxSurge); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +func (s *RuntimeScheduler) RuntimeDeploymentPods(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) { + if s == nil || s.deployments == nil { + return nil, nil + } + runtimeType = strings.TrimSpace(runtimeType) + pods, err := s.deployments.ListPods(ctx, s.runtimeNamespace, runtimeType) + if err != nil { + return nil, err + } + result := make([]models.RuntimePod, 0, len(pods)) + for index, pod := range pods { + result = append(result, models.RuntimePod{ + ID: -int64(index + 1), + RuntimeType: pod.RuntimeType, + Namespace: pod.Namespace, + PodName: pod.PodName, + PodIP: pod.PodIP, + NodeName: pod.NodeName, + DeploymentName: pod.DeploymentName, + ImageRef: pod.ImageRef, + State: runtimeDeploymentFallbackState(pod.State), + Capacity: s.maxGatewaysPerPod, + UsedSlots: 0, + Draining: false, + }) + } + return result, nil +} + +func defaultRuntimeDeploymentName(runtimeType string) string { + switch strings.ToLower(strings.TrimSpace(runtimeType)) { + case RuntimeTypeOpenClaw: + return "openclaw-runtime" + case RuntimeTypeHermes: + return "hermes-runtime" + default: + return "" + } +} + +func runtimeDeploymentFallbackState(k8sState string) string { + switch strings.TrimSpace(k8sState) { + case "pending", "deleted": + return strings.TrimSpace(k8sState) + default: + return "unhealthy" + } +} + +func (s *RuntimeScheduler) finishRolloutIfReady(ctx context.Context, rollout models.RuntimeRollout) error { + if s == nil || s.podRepo == nil || s.rolloutRepo == nil { + return nil + } + targetImage := strings.TrimSpace(rollout.TargetImageRef) + if targetImage == "" { + return nil + } + pods, err := s.podRepo.List(ctx, rollout.RuntimeType) + if err != nil { + return err + } + pods = s.currentRuntimePods(pods, time.Now().UTC()) + if len(pods) == 0 { + return nil + } + for _, pod := range pods { + if pod.RuntimeType != rollout.RuntimeType { + continue + } + if pod.State != "ready" || pod.Draining || strings.TrimSpace(pod.ImageRef) != targetImage { + return nil + } + } + finishedAt := time.Now().UTC() + return s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "finished", rollout.StartedAt, &finishedAt, nil) +} + +func (s *RuntimeScheduler) currentRuntimePods(pods []models.RuntimePod, now time.Time) []models.RuntimePod { + if s == nil || s.heartbeatTimeout <= 0 { + return pods + } + cutoff := now.UTC().Add(-runtimeRolloutStaleWindowMultiplier * s.heartbeatTimeout) + current := pods[:0] + for _, pod := range pods { + if pod.LastSeenAt != nil && pod.LastSeenAt.UTC().Before(cutoff) { + continue + } + current = append(current, pod) + } + return current +} + +func runtimePodsAlreadyAtImage(pods []models.RuntimePod, runtimeType, targetImage string) bool { + targetImage = strings.TrimSpace(targetImage) + if targetImage == "" { + return false + } + matched := 0 + for _, pod := range pods { + if pod.RuntimeType != runtimeType { + continue + } + matched++ + if pod.State != "ready" || pod.Draining || strings.TrimSpace(pod.ImageRef) != targetImage { + return false + } + } + return matched > 0 +} + +func (s *RuntimeScheduler) reconcileIfLeader(ctx context.Context) { + if s.leader != nil && !s.leader.IsLeader(ctx) { + return + } + if err := s.reconcile(ctx); err != nil { + log.Printf("runtime scheduler reconcile failed: %v", err) + } +} + +func (s *RuntimeScheduler) reconcile(ctx context.Context) error { + if s == nil || s.instanceRepo == nil || s.bindingRepo == nil || s.podRepo == nil { + return fmt.Errorf("runtime scheduler dependencies are not configured") + } + var errs []error + if err := s.failoverStalePods(ctx); err != nil { + errs = append(errs, err) + } + if err := s.reconcileRollouts(ctx); err != nil { + errs = append(errs, err) + } + creating, err := s.instanceRepo.GetV2Creating(ctx, 100) + if err != nil { + errs = append(errs, err) + } else { + for _, instance := range creating { + if !isSchedulerManagedV2Instance(instance) { + continue + } + binding, err := s.bindingRepo.GetByInstanceID(ctx, instance.ID) + if err != nil { + errs = append(errs, fmt.Errorf("get binding for creating instance %d: %w", instance.ID, err)) + continue + } + if binding != nil { + continue + } + if err := s.assignInstance(ctx, instance); err != nil { + if errors.Is(err, errRuntimeScaleOutPending) { + continue + } + errs = append(errs, fmt.Errorf("assign creating instance %d: %w", instance.ID, err)) + s.markInstanceError(ctx, instance, err, &errs) + } + } + } + + desired, err := s.instanceRepo.GetV2DesiredRunning(ctx, 100) + if err != nil { + errs = append(errs, err) + } else { + for _, instance := range desired { + if !isSchedulerManagedV2Instance(instance) { + continue + } + if isRuntimeErrorInstance(instance) && !isRecoverableRuntimeSchedulingError(instance) { + continue + } + binding, err := s.bindingRepo.GetRunningByInstanceID(ctx, instance.ID) + if err != nil { + errs = append(errs, fmt.Errorf("get running binding for instance %d: %w", instance.ID, err)) + continue + } + if binding != nil { + continue + } + binding, err = s.bindingRepo.GetByInstanceID(ctx, instance.ID) + if err != nil { + errs = append(errs, fmt.Errorf("get binding for desired instance %d: %w", instance.ID, err)) + continue + } + if binding != nil { + continue + } + if assignErr := s.assignInstance(ctx, instance); assignErr != nil { + if errors.Is(assignErr, errRuntimeScaleOutPending) { + continue + } + errs = append(errs, fmt.Errorf("assign desired instance %d: %w", instance.ID, assignErr)) + s.markInstanceError(ctx, instance, assignErr, &errs) + } + } + } + return errors.Join(errs...) +} + +func (s *RuntimeScheduler) failoverStalePods(ctx context.Context) error { + if s.heartbeatTimeout <= 0 { + return nil + } + pods, err := s.podRepo.List(ctx, "") + if err != nil { + return fmt.Errorf("list runtime pods for heartbeat failover: %w", err) + } + cutoff := time.Now().UTC().Add(-s.heartbeatTimeout) + var errs []error + for _, pod := range pods { + if pod.State == "unhealthy" || pod.State == "pending" { + continue + } + if pod.LastSeenAt == nil || !pod.LastSeenAt.Before(cutoff) { + continue + } + reason := fmt.Sprintf("runtime pod heartbeat lost since %s", pod.LastSeenAt.UTC().Format(time.RFC3339)) + if err := s.FailoverPod(ctx, pod.ID, reason); err != nil { + errs = append(errs, fmt.Errorf("failover stale runtime pod %d: %w", pod.ID, err)) + } + } + return errors.Join(errs...) +} + +func (s *RuntimeScheduler) assignInstance(ctx context.Context, instance models.Instance) error { + if s == nil || s.podRepo == nil { + return fmt.Errorf("runtime scheduler pod repository is not configured") + } + if !isSchedulerManagedV2Instance(instance) { + return fmt.Errorf("instance %d is not scheduler-managed v2", instance.ID) + } + if s.bindingRepo != nil { + binding, err := s.bindingRepo.GetByInstanceID(ctx, instance.ID) + if err != nil { + return fmt.Errorf("failed to check existing binding: %w", err) + } + if binding != nil { + return nil + } + } + runtimeType, ok := schedulerRuntimeType(instance) + if !ok { + return fmt.Errorf("unsupported runtime type %q", instance.Type) + } + pods, err := s.podRepo.ListSchedulable(ctx, runtimeType) + if err != nil { + return err + } + if len(pods) == 0 { + scaled, err := s.scaleOutIfAtCapacity(ctx, runtimeType) + if err != nil { + return fmt.Errorf("scale out %s runtime deployment: %w", runtimeType, err) + } + if scaled { + return errRuntimeScaleOutPending + } + } + var lastErr error + for _, pod := range pods { + if pod.AgentEndpoint == nil || strings.TrimSpace(*pod.AgentEndpoint) == "" { + continue + } + claimed, err := s.podRepo.TryClaimSlot(ctx, pod.ID) + if err != nil { + lastErr = err + continue + } + if !claimed { + continue + } + if err := s.createGatewayOnPod(ctx, instance, runtimeType, pod); err != nil { + if releaseErr := s.podRepo.ReleaseSlot(ctx, pod.ID); releaseErr != nil { + return errors.Join(err, releaseErr) + } + lastErr = err + continue + } + return nil + } + if lastErr != nil { + return fmt.Errorf("no schedulable %s runtime pod: %w", runtimeType, lastErr) + } + return fmt.Errorf("no schedulable %s runtime pod", runtimeType) +} + +type runtimeDeploymentCapacity struct { + namespace string + name string + active int + full int +} + +func (s *RuntimeScheduler) scaleOutIfAtCapacity(ctx context.Context, runtimeType string) (bool, error) { + if s == nil || s.deployments == nil || s.podRepo == nil { + return false, nil + } + pods, err := s.podRepo.List(ctx, runtimeType) + if err != nil { + return false, fmt.Errorf("list %s runtime pods for scale-out: %w", runtimeType, err) + } + groups := map[string]*runtimeDeploymentCapacity{} + for _, pod := range pods { + if pod.RuntimeType != runtimeType || pod.State != "ready" || pod.Draining || pod.Capacity <= 0 { + continue + } + namespace := strings.TrimSpace(pod.Namespace) + name := strings.TrimSpace(pod.DeploymentName) + if namespace == "" || name == "" { + continue + } + key := namespace + "/" + name + group := groups[key] + if group == nil { + group = &runtimeDeploymentCapacity{namespace: namespace, name: name} + groups[key] = group + } + group.active++ + if pod.UsedSlots >= pod.Capacity { + group.full++ + } + } + var target *runtimeDeploymentCapacity + for _, group := range groups { + if group.active == 0 || group.active != group.full { + continue + } + if target == nil || group.active > target.active { + target = group + } + } + if target == nil { + return false, nil + } + replicas := int32(target.active + 1) + if err := s.deployments.Scale(ctx, target.namespace, target.name, replicas); err != nil { + return false, err + } + if s.events != nil { + if err := s.events.Publish(ctx, "runtime.pool.scaleout", map[string]any{ + "runtime_type": runtimeType, + "namespace": target.namespace, + "deployment": target.name, + "replicas": replicas, + }); err != nil { + log.Printf("runtime scheduler publish scale-out event failed: %v", err) + } + } + return true, nil +} + +func (s *RuntimeScheduler) createGatewayOnPod(ctx context.Context, instance models.Instance, runtimeType string, pod models.RuntimePod) error { + if s.agentClient == nil || s.bindingRepo == nil || s.instanceRepo == nil { + return fmt.Errorf("runtime scheduler gateway dependencies are not configured") + } + endpoint := strings.TrimSpace(*pod.AgentEndpoint) + workspacePath := RuntimeWorkspacePathWithRoot(s.workspaceRoot, runtimeType, instance.UserID, instance.ID) + environment, err := s.gatewayEnvironment(&instance) + if err != nil { + return fmt.Errorf("build runtime gateway environment: %w", err) + } + uid, gid := runtimeGatewayLinuxIDs(instance.ID, environment) + resp, err := s.agentClient.CreateGateway(ctx, endpoint, RuntimeAgentCreateGatewayRequest{ + InstanceID: instance.ID, + UserID: instance.UserID, + AgentType: runtimeType, + WorkspacePath: workspacePath, + PortRange: RuntimeAgentPortRange{ + Start: s.gatewayPortStart, + End: s.gatewayPortEnd, + }, + UID: uid, + GID: gid, + CPUCores: instance.CPUCores, + MemoryMB: instance.MemoryGB * 1024, + DiskQuotaMB: instance.DiskGB * 1024, + Generation: instance.RuntimeGeneration, + Environment: environment, + }) + if err != nil { + return err + } + if resp == nil { + return fmt.Errorf("runtime agent returned empty gateway response") + } + + now := time.Now().UTC() + binding := &models.InstanceRuntimeBinding{ + InstanceID: instance.ID, + RuntimePodID: pod.ID, + RuntimeType: runtimeType, + GatewayID: resp.GatewayID, + GatewayPort: resp.Port, + GatewayPID: resp.PID, + WorkspacePath: workspacePath, + State: "running", + Generation: instance.RuntimeGeneration, + LastHealthAt: &now, + } + if err := s.bindingRepo.Create(ctx, binding); err != nil { + return s.cleanupGatewayAfterAssignFailure(ctx, endpoint, instance.ID, resp.GatewayID, false, err) + } + if err := s.instanceRepo.SetWorkspacePath(ctx, instance.ID, workspacePath); err != nil { + return s.cleanupGatewayAfterAssignFailure(ctx, endpoint, instance.ID, resp.GatewayID, true, err) + } + if err := s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "running", instance.RuntimeGeneration, nil); err != nil { + return s.cleanupGatewayAfterAssignFailure(ctx, endpoint, instance.ID, resp.GatewayID, true, err) + } + if s.events != nil { + if err := s.events.Publish(ctx, "runtime.instance.running", map[string]any{ + "instance_id": instance.ID, + "runtime_type": runtimeType, + "runtime_pod_id": pod.ID, + "gateway_id": resp.GatewayID, + "gateway_port": resp.Port, + "workspace_path": workspacePath, + "generation": instance.RuntimeGeneration, + }); err != nil { + log.Printf("runtime scheduler publish event failed: %v", err) + } + } + return nil +} + +func runtimeGatewayLinuxIDs(instanceID int, environment map[string]string) (int, int) { + linuxID := RuntimeLinuxID(instanceID) + if !strings.EqualFold(strings.TrimSpace(environment["CLAWMANAGER_TEAM_ENABLED"]), "true") { + return linuxID, linuxID + } + sharedGID, err := strconv.Atoi(strings.TrimSpace(environment["CLAWMANAGER_TEAM_SHARED_GID"])) + if err != nil || sharedGID <= 0 { + return linuxID, linuxID + } + return linuxID, sharedGID +} + +func (s *RuntimeScheduler) gatewayEnvironment(instance *models.Instance) (map[string]string, error) { + if s == nil || s.envBuilder == nil { + return nil, nil + } + env, err := s.envBuilder(instance) + if err != nil { + return nil, err + } + if len(env) == 0 { + return nil, nil + } + copied := make(map[string]string, len(env)) + for key, value := range env { + if strings.TrimSpace(key) != "" { + copied[key] = value + } + } + if len(copied) == 0 { + return nil, nil + } + return copied, nil +} + +func (s *RuntimeScheduler) cleanupGatewayAfterAssignFailure(ctx context.Context, endpoint string, instanceID int, gatewayID string, bindingCreated bool, cause error) error { + errs := []error{cause} + if gatewayID != "" && s.agentClient != nil { + if err := s.agentClient.DeleteGateway(ctx, endpoint, gatewayID); err != nil { + errs = append(errs, fmt.Errorf("delete gateway %s: %w", gatewayID, err)) + } + } + if bindingCreated && s.bindingRepo != nil { + if err := s.bindingRepo.DeleteByInstanceID(ctx, instanceID); err != nil { + errs = append(errs, fmt.Errorf("delete binding for instance %d: %w", instanceID, err)) + } + } + return errors.Join(errs...) +} + +func (s *RuntimeScheduler) markInstanceError(ctx context.Context, instance models.Instance, cause error, errs *[]error) { + message := cause.Error() + if err := s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "error", instance.RuntimeGeneration, &message); err != nil { + *errs = append(*errs, fmt.Errorf("mark instance %d error: %w", instance.ID, err)) + } +} + +func schedulerRuntimeType(instance models.Instance) (string, bool) { + return NormalizeV2RuntimeType(instance.Type) +} + +func isSchedulerManagedV2Instance(instance models.Instance) bool { + if _, ok := schedulerRuntimeType(instance); !ok { + return false + } + if normalizeInstanceRuntimeType(instance.RuntimeType) != RuntimeBackendGateway { + return false + } + if mode, ok := NormalizeInstanceMode(instance.InstanceMode); ok && mode != InstanceModeLite { + return false + } + return instance.WorkspacePath != nil && strings.TrimSpace(*instance.WorkspacePath) != "" +} + +func isRuntimeErrorInstance(instance models.Instance) bool { + return strings.EqualFold(strings.TrimSpace(instance.Status), "error") +} + +func isRecoverableRuntimeSchedulingError(instance models.Instance) bool { + if !isRuntimeErrorInstance(instance) || instance.RuntimeErrorMessage == nil { + return false + } + runtimeType, ok := schedulerRuntimeType(instance) + if !ok { + return false + } + message := strings.TrimSpace(*instance.RuntimeErrorMessage) + return message == fmt.Sprintf("no schedulable %s runtime pod", runtimeType) +} + +func minInt(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/backend/internal/services/runtime_scheduler_test.go b/backend/internal/services/runtime_scheduler_test.go new file mode 100644 index 0000000..89c02ff --- /dev/null +++ b/backend/internal/services/runtime_scheduler_test.go @@ -0,0 +1,1840 @@ +package services + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" + "clawreef/internal/services/k8s" +) + +func TestRuntimeSchedulerAssignsCreatingInstanceToReadyPod(t *testing.T) { + ctx := context.Background() + endpoint := "http://agent.runtime" + workspacePath := "/workspaces/openclaw/user-45/instance-17" + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.creating = []models.Instance{{ + ID: 17, + UserID: 45, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + Status: "creating", + CPUCores: 1.5, + MemoryGB: 2, + DiskGB: 8, + WorkspacePath: &workspacePath, + RuntimeGeneration: 3, + }} + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: { + ID: 9, + RuntimeType: RuntimeTypeOpenClaw, + AgentEndpoint: &endpoint, + State: "ready", + Capacity: 1, + }, + }, + schedulable: []models.RuntimePod{{ + ID: 9, + RuntimeType: RuntimeTypeOpenClaw, + AgentEndpoint: &endpoint, + State: "ready", + Capacity: 1, + }}, + } + bindingRepo := newFakeRuntimeBindingRepo() + agent := &fakeRuntimeAgentClient{ + createResponse: &RuntimeAgentCreateGatewayResponse{ + GatewayID: "gw-17", + Port: 20017, + Status: "running", + }, + } + events := &fakeRuntimeEventService{} + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + bindingRepo, + &fakeRuntimeRolloutRepo{}, + agent, + events, + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + + if got := len(agent.createRequests); got != 1 { + t.Fatalf("CreateGateway calls = %d, want 1", got) + } + req := agent.createRequests[0] + if req.endpoint != endpoint { + t.Fatalf("CreateGateway endpoint = %q, want %q", req.endpoint, endpoint) + } + if req.req.WorkspacePath != "/workspaces/openclaw/user-45/instance-17" { + t.Fatalf("workspace path = %q", req.req.WorkspacePath) + } + if req.req.PortRange.Start != 20000 || req.req.PortRange.End != 20099 { + t.Fatalf("port range = %+v", req.req.PortRange) + } + if req.req.UID != RuntimeLinuxID(17) || req.req.GID != RuntimeLinuxID(17) { + t.Fatalf("linux IDs = %d/%d", req.req.UID, req.req.GID) + } + if req.req.MemoryMB != 2048 || req.req.DiskQuotaMB != 8192 { + t.Fatalf("resource MB = %d/%d", req.req.MemoryMB, req.req.DiskQuotaMB) + } + + binding := bindingRepo.bindings[17] + if binding == nil { + t.Fatal("binding was not created") + } + if binding.RuntimePodID != 9 || binding.GatewayPort != 20017 { + t.Fatalf("binding pod/port = %d/%d", binding.RuntimePodID, binding.GatewayPort) + } + if instanceRepo.workspacePaths[17] != "/workspaces/openclaw/user-45/instance-17" { + t.Fatalf("instance workspace path = %q", instanceRepo.workspacePaths[17]) + } + state := instanceRepo.runtimeStates[17] + if state.status != "running" || state.generation != 3 { + t.Fatalf("runtime state = %+v", state) + } + if podRepo.claims[9] != 1 { + t.Fatalf("pod claims = %d, want 1", podRepo.claims[9]) + } + if got := len(events.published); got != 1 { + t.Fatalf("published events = %d, want 1", got) + } +} + +func TestRuntimeSchedulerPassesGatewayEnvironmentToRuntimeAgent(t *testing.T) { + ctx := context.Background() + endpoint := "http://agent.runtime" + token := "igt_instance_68" + workspacePath := "/workspaces/openclaw/user-1/instance-68" + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.creating = []models.Instance{{ + ID: 68, + UserID: 1, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + Status: "creating", + AccessToken: &token, + WorkspacePath: &workspacePath, + RuntimeGeneration: 4, + }} + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 10: { + ID: 10, + RuntimeType: RuntimeTypeOpenClaw, + AgentEndpoint: &endpoint, + State: "ready", + Capacity: 10, + }, + }, + schedulable: []models.RuntimePod{{ + ID: 10, + RuntimeType: RuntimeTypeOpenClaw, + AgentEndpoint: &endpoint, + State: "ready", + Capacity: 10, + }}, + } + agent := &fakeRuntimeAgentClient{ + createResponse: &RuntimeAgentCreateGatewayResponse{ + GatewayID: "gw-68-4", + Port: 20068, + Status: "starting", + }, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + newFakeRuntimeBindingRepo(), + &fakeRuntimeRolloutRepo{}, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + WithRuntimeSchedulerGatewayEnvBuilder(func(instance *models.Instance) (map[string]string, error) { + if instance == nil || instance.ID != 68 { + t.Fatalf("gateway env builder received instance %#v, want 68", instance) + } + return map[string]string{ + "CLAWMANAGER_LLM_BASE_URL": "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001/api/v1/gateway/llm", + "CLAWMANAGER_LLM_API_KEY": token, + "CLAWMANAGER_LLM_MODEL": `["auto","gpt-5.5"]`, + "OPENAI_API_KEY": token, + }, nil + }), + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + + if got := len(agent.createRequests); got != 1 { + t.Fatalf("CreateGateway calls = %d, want 1", got) + } + env := agent.createRequests[0].req.Environment + if env["CLAWMANAGER_LLM_API_KEY"] != token || env["OPENAI_API_KEY"] != token { + t.Fatalf("gateway environment missing instance token aliases: %#v", env) + } + if env["CLAWMANAGER_LLM_BASE_URL"] != "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001/api/v1/gateway/llm" { + t.Fatalf("CLAWMANAGER_LLM_BASE_URL = %q", env["CLAWMANAGER_LLM_BASE_URL"]) + } + if env["CLAWMANAGER_LLM_MODEL"] != `["auto","gpt-5.5"]` { + t.Fatalf("CLAWMANAGER_LLM_MODEL = %q", env["CLAWMANAGER_LLM_MODEL"]) + } +} + +func TestRuntimeSchedulerUsesTeamSharedGIDForTeamLiteGateway(t *testing.T) { + ctx := context.Background() + endpoint := "http://agent.runtime" + workspacePath := "/workspaces/openclaw/user-1/instance-130" + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.creating = []models.Instance{{ + ID: 130, + UserID: 1, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + Status: "creating", + WorkspacePath: &workspacePath, + RuntimeGeneration: 1, + MemoryGB: 1, + DiskGB: 1, + }} + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 11: { + ID: 11, + RuntimeType: RuntimeTypeOpenClaw, + AgentEndpoint: &endpoint, + State: "ready", + Capacity: 10, + }, + }, + schedulable: []models.RuntimePod{{ + ID: 11, + RuntimeType: RuntimeTypeOpenClaw, + AgentEndpoint: &endpoint, + State: "ready", + Capacity: 10, + }}, + } + agent := &fakeRuntimeAgentClient{ + createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-130-1", Port: 20030, Status: "running"}, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + newFakeRuntimeBindingRepo(), + &fakeRuntimeRolloutRepo{}, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + WithRuntimeSchedulerGatewayEnvBuilder(func(instance *models.Instance) (map[string]string, error) { + return map[string]string{ + "CLAWMANAGER_TEAM_ENABLED": "true", + "CLAWMANAGER_TEAM_SHARED_GID": "1000", + }, nil + }), + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + if got := len(agent.createRequests); got != 1 { + t.Fatalf("CreateGateway calls = %d, want 1", got) + } + req := agent.createRequests[0].req + if req.UID != RuntimeLinuxID(130) { + t.Fatalf("UID = %d, want isolated instance UID %d", req.UID, RuntimeLinuxID(130)) + } + if req.GID != 1000 { + t.Fatalf("GID = %d, want Team shared GID 1000", req.GID) + } +} + +func TestRuntimeSchedulerNoSchedulablePodReturnsErrorAndReleasesNothing(t *testing.T) { + ctx := context.Background() + instanceRepo := newFakeRuntimeInstanceRepo() + podRepo := &fakeRuntimePodRepo{} + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + newFakeRuntimeBindingRepo(), + &fakeRuntimeRolloutRepo{}, + &fakeRuntimeAgentClient{}, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + err := scheduler.assignInstance(ctx, models.Instance{ + ID: 1, + UserID: 2, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + WorkspacePath: ptrString("/workspaces/openclaw/user-2/instance-1"), + }) + if err == nil { + t.Fatal("assignInstance returned nil error") + } + if podRepo.releaseCount != 0 { + t.Fatalf("release count = %d, want 0", podRepo.releaseCount) + } +} + +func TestRuntimeSchedulerScalesOutWhenAllReadyPodsAtCapacity(t *testing.T) { + ctx := context.Background() + endpoint := "http://pod-1.runtime" + workspacePath := "/workspaces/openclaw/user-12/instance-27" + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.creating = []models.Instance{{ + ID: 27, + UserID: 12, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + Status: "creating", + WorkspacePath: &workspacePath, + RuntimeGeneration: 2, + }} + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 5: { + ID: 5, + RuntimeType: RuntimeTypeOpenClaw, + Namespace: "clawmanager-system", + PodName: "openclaw-runtime-abc", + DeploymentName: "openclaw-runtime", + AgentEndpoint: &endpoint, + State: "ready", + Capacity: 100, + UsedSlots: 100, + }, + }, + } + deployments := &fakeRuntimeDeploymentService{} + agent := &fakeRuntimeAgentClient{} + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + newFakeRuntimeBindingRepo(), + &fakeRuntimeRolloutRepo{}, + agent, + NewRuntimeEventService(nil), + nil, + deployments, + time.Second, + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + if got := len(deployments.scales); got != 1 { + t.Fatalf("deployment Scale calls = %d, want 1", got) + } + scale := deployments.scales[0] + if scale.namespace != "clawmanager-system" || scale.name != "openclaw-runtime" || scale.replicas != 2 { + t.Fatalf("deployment Scale call = %+v, want clawmanager-system/openclaw-runtime replicas 2", scale) + } + if got := len(agent.createRequests); got != 0 { + t.Fatalf("CreateGateway calls = %d, want 0 while waiting for scale-out pod", got) + } + if state, ok := instanceRepo.runtimeStates[27]; ok { + t.Fatalf("instance runtime state = %+v, want unchanged while waiting for scale-out pod", state) + } +} + +func TestRuntimeSchedulerDoesNotScaleOutWhenGatewayCreateFailsWithFreeSlot(t *testing.T) { + ctx := context.Background() + endpoint := "http://pod-1.runtime" + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 5: { + ID: 5, + RuntimeType: RuntimeTypeOpenClaw, + Namespace: "clawmanager-system", + DeploymentName: "openclaw-runtime", + AgentEndpoint: &endpoint, + State: "ready", + Capacity: 100, + UsedSlots: 1, + }, + }, + schedulable: []models.RuntimePod{{ + ID: 5, + RuntimeType: RuntimeTypeOpenClaw, + Namespace: "clawmanager-system", + DeploymentName: "openclaw-runtime", + AgentEndpoint: &endpoint, + State: "ready", + Capacity: 100, + UsedSlots: 1, + }}, + } + deployments := &fakeRuntimeDeploymentService{} + scheduler := NewRuntimeScheduler( + newFakeRuntimeInstanceRepo(), + podRepo, + newFakeRuntimeBindingRepo(), + &fakeRuntimeRolloutRepo{}, + &fakeRuntimeAgentClient{createErr: errors.New("agent timeout")}, + NewRuntimeEventService(nil), + nil, + deployments, + time.Second, + ) + + err := scheduler.assignInstance(ctx, models.Instance{ + ID: 28, + UserID: 12, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + WorkspacePath: ptrString("/workspaces/openclaw/user-12/instance-28"), + RuntimeGeneration: 1, + }) + if err == nil { + t.Fatal("assignInstance returned nil error") + } + if got := len(deployments.scales); got != 0 { + t.Fatalf("deployment Scale calls = %d, want 0", got) + } +} + +func TestRuntimeSchedulerReconcileAssignsDesiredRunningInstanceWithMissingBinding(t *testing.T) { + ctx := context.Background() + endpoint := "http://agent.runtime" + workspacePath := "/workspaces/openclaw/user-46/instance-18" + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.desiredRunning = []models.Instance{{ + ID: 18, + UserID: 46, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + Status: "running", + MemoryGB: 1, + DiskGB: 1, + WorkspacePath: &workspacePath, + RuntimeGeneration: 4, + }} + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + schedulable: []models.RuntimePod{ + {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + } + agent := &fakeRuntimeAgentClient{ + createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-18", Port: 20018, Status: "running"}, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + newFakeRuntimeBindingRepo(), + &fakeRuntimeRolloutRepo{}, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + if got := len(agent.createRequests); got != 1 { + t.Fatalf("CreateGateway calls = %d, want 1", got) + } +} + +func TestRuntimeSchedulerReconcileRetriesRecoverableNoSchedulableError(t *testing.T) { + ctx := context.Background() + endpoint := "http://agent.runtime" + workspacePath := "/workspaces/openclaw/user-46/instance-68" + errorMessage := "no schedulable openclaw runtime pod" + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.desiredRunning = []models.Instance{{ + ID: 68, + UserID: 46, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + Status: "error", + RuntimeErrorMessage: &errorMessage, + MemoryGB: 1, + DiskGB: 1, + WorkspacePath: &workspacePath, + RuntimeGeneration: 29, + }} + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 50: {ID: 50, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 100, UsedSlots: 0}, + }, + schedulable: []models.RuntimePod{ + {ID: 50, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 100, UsedSlots: 0}, + }, + } + agent := &fakeRuntimeAgentClient{ + createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-68", Port: 20068, Status: "running"}, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + newFakeRuntimeBindingRepo(), + &fakeRuntimeRolloutRepo{}, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + if got := len(agent.createRequests); got != 1 { + t.Fatalf("CreateGateway calls = %d, want 1", got) + } + state := instanceRepo.runtimeStates[68] + if state.status != "running" || state.generation != 29 || state.message != nil { + t.Fatalf("runtime state = %+v, want running generation 29 without error message", state) + } +} + +func TestRuntimeSchedulerReconcileSkipsNonRecoverableErrorInstance(t *testing.T) { + ctx := context.Background() + endpoint := "http://agent.runtime" + workspacePath := "/workspaces/openclaw/user-46/instance-69" + errorMessage := "create gateway failed: invalid runtime response" + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.desiredRunning = []models.Instance{{ + ID: 69, + UserID: 46, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + Status: "error", + RuntimeErrorMessage: &errorMessage, + MemoryGB: 1, + DiskGB: 1, + WorkspacePath: &workspacePath, + RuntimeGeneration: 24, + }} + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 50: {ID: 50, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 100, UsedSlots: 0}, + }, + schedulable: []models.RuntimePod{ + {ID: 50, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 100, UsedSlots: 0}, + }, + } + agent := &fakeRuntimeAgentClient{ + createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-69", Port: 20069, Status: "running"}, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + newFakeRuntimeBindingRepo(), + &fakeRuntimeRolloutRepo{}, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + if got := len(agent.createRequests); got != 0 { + t.Fatalf("CreateGateway calls = %d, want 0 for non-recoverable error", got) + } + if _, ok := instanceRepo.runtimeStates[69]; ok { + t.Fatalf("runtime state was changed for non-recoverable error: %+v", instanceRepo.runtimeStates[69]) + } +} + +func TestRuntimeSchedulerReconcileSkipsAssignmentWhenBindingLookupErrors(t *testing.T) { + ctx := context.Background() + endpoint := "http://agent.runtime" + lookupErr := errors.New("binding db unavailable") + workspacePath := "/workspaces/openclaw/user-47/instance-19" + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.desiredRunning = []models.Instance{{ + ID: 19, + UserID: 47, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + Status: "running", + MemoryGB: 1, + DiskGB: 1, + WorkspacePath: &workspacePath, + RuntimeGeneration: 5, + }} + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + schedulable: []models.RuntimePod{ + {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.runningErr = lookupErr + agent := &fakeRuntimeAgentClient{ + createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-19", Port: 20019, Status: "running"}, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + bindingRepo, + &fakeRuntimeRolloutRepo{}, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + err := scheduler.reconcile(ctx) + if err == nil { + t.Fatal("reconcile returned nil error") + } + if !errors.Is(err, lookupErr) { + t.Fatalf("reconcile error = %v, want lookup error", err) + } + if got := len(agent.createRequests); got != 0 { + t.Fatalf("CreateGateway calls = %d, want 0", got) + } + if bindingRepo.bindings[19] != nil { + t.Fatal("binding was created despite lookup error") + } + if podRepo.claims[9] != 0 { + t.Fatalf("pod claims = %d, want 0", podRepo.claims[9]) + } +} + +func TestRuntimeSchedulerReconcileSkipsLegacyInstanceWithoutWorkspacePath(t *testing.T) { + ctx := context.Background() + endpoint := "http://agent.runtime" + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.creating = []models.Instance{{ + ID: 20, + UserID: 48, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + Status: "creating", + MemoryGB: 1, + DiskGB: 1, + RuntimeGeneration: 1, + }} + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + schedulable: []models.RuntimePod{ + {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + } + agent := &fakeRuntimeAgentClient{ + createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-20", Port: 20020, Status: "running"}, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + newFakeRuntimeBindingRepo(), + &fakeRuntimeRolloutRepo{}, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + if got := len(agent.createRequests); got != 0 { + t.Fatalf("CreateGateway calls = %d, want 0", got) + } + if _, ok := instanceRepo.runtimeStates[20]; ok { + t.Fatalf("legacy instance was marked with runtime state: %+v", instanceRepo.runtimeStates[20]) + } +} + +func TestRuntimeSchedulerSkipsCreatingInstanceWithExistingBinding(t *testing.T) { + ctx := context.Background() + endpoint := "http://agent.runtime" + workspacePath := "/workspaces/openclaw/user-49/instance-21" + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.creating = []models.Instance{{ + ID: 21, + UserID: 49, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + Status: "creating", + MemoryGB: 1, + DiskGB: 1, + WorkspacePath: &workspacePath, + RuntimeGeneration: 1, + }} + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[21] = &models.InstanceRuntimeBinding{InstanceID: 21, RuntimePodID: 9, State: "creating", Generation: 1} + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + schedulable: []models.RuntimePod{ + {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + } + agent := &fakeRuntimeAgentClient{ + createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-21", Port: 20021, Status: "running"}, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + bindingRepo, + &fakeRuntimeRolloutRepo{}, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + if got := len(agent.createRequests); got != 0 { + t.Fatalf("CreateGateway calls = %d, want 0", got) + } + if podRepo.claims[9] != 0 { + t.Fatalf("pod claims = %d, want 0", podRepo.claims[9]) + } +} + +func TestRuntimeSchedulerCleansUpGatewayAndReleasesSlotWhenBindingCreateFails(t *testing.T) { + ctx := context.Background() + endpoint := "http://agent.runtime" + createErr := errors.New("binding insert failed") + instanceRepo := newFakeRuntimeInstanceRepo() + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + schedulable: []models.RuntimePod{ + {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + } + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.createErr = createErr + agent := &fakeRuntimeAgentClient{ + createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-22", Port: 20022, Status: "running"}, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + bindingRepo, + &fakeRuntimeRolloutRepo{}, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + err := scheduler.assignInstance(ctx, models.Instance{ + ID: 22, + UserID: 50, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + WorkspacePath: ptrString("/workspaces/openclaw/user-50/instance-22"), + MemoryGB: 1, + DiskGB: 1, + RuntimeGeneration: 1, + }) + if err == nil { + t.Fatal("assignInstance returned nil error") + } + if !errors.Is(err, createErr) { + t.Fatalf("assignInstance error = %v, want binding create error", err) + } + if got := len(agent.deleteRequests); got != 1 { + t.Fatalf("DeleteGateway calls = %d, want 1", got) + } + if agent.deleteRequests[0].gatewayID != "gw-22" { + t.Fatalf("deleted gateway = %q, want gw-22", agent.deleteRequests[0].gatewayID) + } + if podRepo.releases[9] != 1 { + t.Fatalf("pod releases = %d, want 1", podRepo.releases[9]) + } + if bindingRepo.bindings[22] != nil { + t.Fatal("binding remains after failed create") + } +} + +func TestRuntimeSchedulerCleansUpGatewayBindingAndSlotWhenWorkspacePathUpdateFails(t *testing.T) { + ctx := context.Background() + endpoint := "http://agent.runtime" + workspaceErr := errors.New("workspace path update failed") + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.setWorkspacePathErr = workspaceErr + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + schedulable: []models.RuntimePod{ + {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + } + bindingRepo := newFakeRuntimeBindingRepo() + agent := &fakeRuntimeAgentClient{ + createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-23", Port: 20023, Status: "running"}, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + bindingRepo, + &fakeRuntimeRolloutRepo{}, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + err := scheduler.assignInstance(ctx, models.Instance{ + ID: 23, + UserID: 51, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + WorkspacePath: ptrString("/workspaces/openclaw/user-51/instance-23"), + MemoryGB: 1, + DiskGB: 1, + RuntimeGeneration: 1, + }) + if err == nil { + t.Fatal("assignInstance returned nil error") + } + if !errors.Is(err, workspaceErr) { + t.Fatalf("assignInstance error = %v, want workspace update error", err) + } + if got := len(agent.deleteRequests); got != 1 { + t.Fatalf("DeleteGateway calls = %d, want 1", got) + } + if agent.deleteRequests[0].gatewayID != "gw-23" { + t.Fatalf("deleted gateway = %q, want gw-23", agent.deleteRequests[0].gatewayID) + } + if bindingRepo.deleteCalls[23] != 1 { + t.Fatalf("binding delete calls = %d, want 1", bindingRepo.deleteCalls[23]) + } + if bindingRepo.bindings[23] != nil { + t.Fatal("binding remains after workspace update failure") + } + if podRepo.releases[9] != 1 { + t.Fatalf("pod releases = %d, want 1", podRepo.releases[9]) + } +} + +func TestRuntimeSchedulerDesiredRunningSkipsNonRunningExistingBinding(t *testing.T) { + ctx := context.Background() + endpoint := "http://agent.runtime" + workspacePath := "/workspaces/openclaw/user-52/instance-24" + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.desiredRunning = []models.Instance{{ + ID: 24, + UserID: 52, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendGateway, + InstanceMode: InstanceModeLite, + Status: "running", + MemoryGB: 1, + DiskGB: 1, + WorkspacePath: &workspacePath, + RuntimeGeneration: 1, + }} + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[24] = &models.InstanceRuntimeBinding{ + InstanceID: 24, + RuntimePodID: 9, + State: "creating", + Generation: 1, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + schedulable: []models.RuntimePod{ + {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1}, + }, + } + agent := &fakeRuntimeAgentClient{ + createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-24", Port: 20024, Status: "running"}, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + bindingRepo, + &fakeRuntimeRolloutRepo{}, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + if got := len(agent.createRequests); got != 0 { + t.Fatalf("CreateGateway calls = %d, want 0", got) + } + if podRepo.claims[9] != 0 { + t.Fatalf("pod claims = %d, want 0", podRepo.claims[9]) + } + if bindingRepo.bindings[24].GatewayID != "" { + t.Fatalf("binding was overwritten: %+v", bindingRepo.bindings[24]) + } +} + +func TestRuntimeSchedulerFailoverPodMarksUnhealthyDeletesBindingsReleasesSlotsAndRecreatesInstances(t *testing.T) { + ctx := context.Background() + reason := "agent heartbeat lost" + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.byID[17] = &models.Instance{ID: 17, RuntimeGeneration: 3} + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[17] = &models.InstanceRuntimeBinding{ + InstanceID: 17, + RuntimePodID: 9, + Generation: 3, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, State: "ready"}, + }, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + bindingRepo, + &fakeRuntimeRolloutRepo{}, + &fakeRuntimeAgentClient{}, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + if err := scheduler.FailoverPod(ctx, 9, reason); err != nil { + t.Fatalf("FailoverPod returned error: %v", err) + } + + if got := podRepo.marked[9]; got.state != "unhealthy" || got.draining { + t.Fatalf("marked pod state = %+v", got) + } + if bindingRepo.bindings[17] != nil { + t.Fatal("binding was not deleted") + } + if podRepo.releases[9] != 1 { + t.Fatalf("pod releases = %d, want 1", podRepo.releases[9]) + } + state := instanceRepo.runtimeStates[17] + if state.status != "creating" || state.generation != 4 { + t.Fatalf("runtime state = %+v, want creating generation 4", state) + } + if state.message == nil || *state.message != reason { + t.Fatalf("runtime state message = %v, want %q", state.message, reason) + } +} + +func TestRuntimeSchedulerFailoverLeavesBindingAndSlotWhenInstanceUpdateFails(t *testing.T) { + ctx := context.Background() + reason := "agent heartbeat lost" + updateErr := errors.New("instance update failed") + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.byID[17] = &models.Instance{ID: 17, RuntimeGeneration: 3} + instanceRepo.updateRuntimeStateErr = updateErr + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[17] = &models.InstanceRuntimeBinding{ + InstanceID: 17, + RuntimePodID: 9, + Generation: 3, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, State: "ready"}, + }, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + bindingRepo, + &fakeRuntimeRolloutRepo{}, + &fakeRuntimeAgentClient{}, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + err := scheduler.FailoverPod(ctx, 9, reason) + if err == nil { + t.Fatal("FailoverPod returned nil error") + } + if !errors.Is(err, updateErr) { + t.Fatalf("FailoverPod error = %v, want update error", err) + } + if bindingRepo.bindings[17] == nil { + t.Fatal("binding was deleted despite instance update failure") + } + if podRepo.releases[9] != 0 { + t.Fatalf("pod releases = %d, want 0", podRepo.releases[9]) + } +} + +func TestRuntimeSchedulerReconcileFailoversStalePod(t *testing.T) { + ctx := context.Background() + staleSeen := time.Now().UTC().Add(-30 * time.Second) + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.byID[17] = &models.Instance{ID: 17, RuntimeGeneration: 3} + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[17] = &models.InstanceRuntimeBinding{ + InstanceID: 17, + RuntimePodID: 9, + Generation: 3, + State: "running", + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, State: "ready", LastSeenAt: &staleSeen}, + }, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + bindingRepo, + &fakeRuntimeRolloutRepo{}, + &fakeRuntimeAgentClient{}, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + WithRuntimeSchedulerHeartbeatTimeout(10*time.Second), + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + + if got := podRepo.marked[9]; got.state != "unhealthy" || got.draining { + t.Fatalf("marked pod state = %+v", got) + } + if bindingRepo.bindings[17] != nil { + t.Fatal("binding was not deleted") + } + if podRepo.releases[9] != 1 { + t.Fatalf("pod releases = %d, want 1", podRepo.releases[9]) + } + state := instanceRepo.runtimeStates[17] + if state.status != "creating" || state.generation != 4 { + t.Fatalf("runtime state = %+v, want creating generation 4", state) + } + if state.message == nil || !strings.Contains(*state.message, "runtime pod heartbeat lost") { + t.Fatalf("runtime state message = %v, want heartbeat lost reason", state.message) + } +} + +func TestRuntimeSchedulerReconcileKeepsRecentPodBinding(t *testing.T) { + ctx := context.Background() + recentSeen := time.Now().UTC() + instanceRepo := newFakeRuntimeInstanceRepo() + instanceRepo.byID[17] = &models.Instance{ID: 17, RuntimeGeneration: 3} + bindingRepo := newFakeRuntimeBindingRepo() + bindingRepo.bindings[17] = &models.InstanceRuntimeBinding{ + InstanceID: 17, + RuntimePodID: 9, + Generation: 3, + State: "running", + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, State: "ready", LastSeenAt: &recentSeen}, + }, + } + scheduler := NewRuntimeScheduler( + instanceRepo, + podRepo, + bindingRepo, + &fakeRuntimeRolloutRepo{}, + &fakeRuntimeAgentClient{}, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + WithRuntimeSchedulerHeartbeatTimeout(10*time.Second), + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + + if got := podRepo.marked[9]; got.state != "" || got.draining { + t.Fatalf("pod was unexpectedly marked: %+v", got) + } + if bindingRepo.bindings[17] == nil { + t.Fatal("binding was deleted") + } + if podRepo.releases[9] != 0 { + t.Fatalf("pod releases = %d, want 0", podRepo.releases[9]) + } +} + +func TestRuntimeSchedulerRolloutExistingDrainingPodsPreventAdditionalDrains(t *testing.T) { + ctx := context.Background() + readyEndpoint := "http://ready.runtime" + drainingEndpoint := "http://draining.runtime" + rolloutRepo := &fakeRuntimeRolloutRepo{ + rollouts: map[int64]*models.RuntimeRollout{ + 7: {ID: 7, RuntimeType: RuntimeTypeOpenClaw, Status: "pending", BatchSize: 3, MaxUnavailable: 1}, + }, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 1: {ID: 1, RuntimeType: RuntimeTypeOpenClaw, State: "ready", Draining: true, AgentEndpoint: &drainingEndpoint}, + 2: {ID: 2, RuntimeType: RuntimeTypeOpenClaw, State: "ready", AgentEndpoint: &readyEndpoint}, + }, + } + agent := &fakeRuntimeAgentClient{} + scheduler := NewRuntimeScheduler( + newFakeRuntimeInstanceRepo(), + podRepo, + newFakeRuntimeBindingRepo(), + rolloutRepo, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + if err := scheduler.StartRollout(ctx, 7); err != nil { + t.Fatalf("StartRollout returned error: %v", err) + } + if got := len(agent.drainEndpoints); got != 0 { + t.Fatalf("Drain calls = %d, want 0", got) + } + if len(rolloutRepo.statuses) != 1 || rolloutRepo.statuses[0].status != "running" { + t.Fatalf("rollout statuses = %+v, want only running", rolloutRepo.statuses) + } +} + +func TestRuntimeSchedulerRolloutBatchLimitedByMaxUnavailable(t *testing.T) { + ctx := context.Background() + endpoint1 := "http://pod-1.runtime" + endpoint2 := "http://pod-2.runtime" + endpoint3 := "http://pod-3.runtime" + rolloutRepo := &fakeRuntimeRolloutRepo{ + rollouts: map[int64]*models.RuntimeRollout{ + 8: {ID: 8, RuntimeType: RuntimeTypeOpenClaw, Status: "pending", BatchSize: 3, MaxUnavailable: 2}, + }, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 1: {ID: 1, RuntimeType: RuntimeTypeOpenClaw, State: "ready", AgentEndpoint: &endpoint1}, + 2: {ID: 2, RuntimeType: RuntimeTypeOpenClaw, State: "ready", AgentEndpoint: &endpoint2}, + 3: {ID: 3, RuntimeType: RuntimeTypeOpenClaw, State: "ready", AgentEndpoint: &endpoint3}, + }, + } + agent := &fakeRuntimeAgentClient{} + scheduler := NewRuntimeScheduler( + newFakeRuntimeInstanceRepo(), + podRepo, + newFakeRuntimeBindingRepo(), + rolloutRepo, + agent, + NewRuntimeEventService(nil), + nil, + &fakeRuntimeDeploymentService{}, + time.Second, + ) + + if err := scheduler.StartRollout(ctx, 8); err != nil { + t.Fatalf("StartRollout returned error: %v", err) + } + if got := len(agent.drainEndpoints); got != 2 { + t.Fatalf("Drain calls = %d, want 2", got) + } +} + +func TestRuntimeSchedulerRolloutUpdatesDeployments(t *testing.T) { + ctx := context.Background() + endpoint := "http://pod-1.runtime" + rolloutRepo := &fakeRuntimeRolloutRepo{ + rollouts: map[int64]*models.RuntimeRollout{ + 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, TargetImageRef: "new-image", Status: "pending", BatchSize: 1, MaxUnavailable: 1}, + }, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 1: { + ID: 1, + RuntimeType: RuntimeTypeOpenClaw, + State: "ready", + Namespace: "runtime-system", + DeploymentName: "openclaw-runtime", + AgentEndpoint: &endpoint, + }, + }, + } + deployments := &fakeRuntimeDeploymentService{} + scheduler := NewRuntimeScheduler( + newFakeRuntimeInstanceRepo(), + podRepo, + newFakeRuntimeBindingRepo(), + rolloutRepo, + &fakeRuntimeAgentClient{}, + NewRuntimeEventService(nil), + nil, + deployments, + time.Second, + ) + + if err := scheduler.StartRollout(ctx, 9); err != nil { + t.Fatalf("StartRollout returned error: %v", err) + } + if got := len(deployments.rolloutImageCalls); got != 1 { + t.Fatalf("deployment RolloutImage calls = %d, want 1", got) + } + call := deployments.rolloutImageCalls[0] + if call.namespace != "runtime-system" || call.name != "openclaw-runtime" || call.image != "new-image" { + t.Fatalf("RolloutImage call = %+v, want runtime-system/openclaw-runtime new-image", call) + } + if call.maxUnavailable != 1 || call.maxSurge != 1 { + t.Fatalf("RolloutImage strategy = %+v, want maxUnavailable=1 maxSurge=1", call) + } +} + +func TestRuntimeSchedulerRolloutUsesStalePodDeploymentRefWhenNoCurrentPods(t *testing.T) { + ctx := context.Background() + staleSeen := time.Now().UTC().Add(-5 * time.Minute) + rolloutRepo := &fakeRuntimeRolloutRepo{ + rollouts: map[int64]*models.RuntimeRollout{ + 12: {ID: 12, RuntimeType: RuntimeTypeOpenClaw, TargetImageRef: "registry/openclaw:v2", Status: "pending", BatchSize: 1, MaxUnavailable: 1}, + }, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 41: { + ID: 41, + RuntimeType: RuntimeTypeOpenClaw, + State: "unhealthy", + Namespace: "runtime-system", + DeploymentName: "openclaw-runtime", + ImageRef: "registry/openclaw:v1", + LastSeenAt: &staleSeen, + }, + }, + } + agent := &fakeRuntimeAgentClient{} + deployments := &fakeRuntimeDeploymentService{} + scheduler := NewRuntimeScheduler( + newFakeRuntimeInstanceRepo(), + podRepo, + newFakeRuntimeBindingRepo(), + rolloutRepo, + agent, + NewRuntimeEventService(nil), + nil, + deployments, + time.Second, + WithRuntimeSchedulerHeartbeatTimeout(10*time.Second), + ) + + if err := scheduler.StartRollout(ctx, 12); err != nil { + t.Fatalf("StartRollout returned error: %v", err) + } + if got := len(deployments.rolloutImageCalls); got != 1 { + t.Fatalf("deployment RolloutImage calls = %d, want 1", got) + } + call := deployments.rolloutImageCalls[0] + if call.namespace != "runtime-system" || call.name != "openclaw-runtime" || call.image != "registry/openclaw:v2" { + t.Fatalf("RolloutImage call = %+v, want runtime-system/openclaw-runtime registry/openclaw:v2", call) + } + if got := len(agent.drainEndpoints); got != 0 { + t.Fatalf("Drain calls = %d, want 0 with no current runtime pods", got) + } + if len(rolloutRepo.statuses) != 1 || rolloutRepo.statuses[0].status != "running" { + t.Fatalf("rollout statuses = %+v, want only running", rolloutRepo.statuses) + } +} + +func TestRuntimeSchedulerRolloutFallsBackToDefaultDeploymentWhenNoPodRows(t *testing.T) { + ctx := context.Background() + rolloutRepo := &fakeRuntimeRolloutRepo{ + rollouts: map[int64]*models.RuntimeRollout{ + 13: {ID: 13, RuntimeType: RuntimeTypeOpenClaw, TargetImageRef: "registry/openclaw:v3", Status: "pending", BatchSize: 2, MaxUnavailable: 1}, + }, + } + deployments := &fakeRuntimeDeploymentService{} + scheduler := NewRuntimeScheduler( + newFakeRuntimeInstanceRepo(), + &fakeRuntimePodRepo{pods: map[int64]*models.RuntimePod{}}, + newFakeRuntimeBindingRepo(), + rolloutRepo, + &fakeRuntimeAgentClient{}, + NewRuntimeEventService(nil), + nil, + deployments, + time.Second, + WithRuntimeSchedulerNamespace("runtime-system"), + ) + + if err := scheduler.StartRollout(ctx, 13); err != nil { + t.Fatalf("StartRollout returned error: %v", err) + } + if got := len(deployments.rolloutImageCalls); got != 1 { + t.Fatalf("deployment RolloutImage calls = %d, want 1", got) + } + call := deployments.rolloutImageCalls[0] + if call.namespace != "runtime-system" || call.name != "openclaw-runtime" || call.image != "registry/openclaw:v3" { + t.Fatalf("RolloutImage call = %+v, want runtime-system/openclaw-runtime registry/openclaw:v3", call) + } +} + +func TestRuntimeSchedulerRolloutSameImageFinishesWithoutDrain(t *testing.T) { + ctx := context.Background() + recentSeen := time.Now().UTC() + endpoint := "http://pod-1.runtime" + rolloutRepo := &fakeRuntimeRolloutRepo{ + rollouts: map[int64]*models.RuntimeRollout{ + 11: {ID: 11, RuntimeType: RuntimeTypeHermes, TargetImageRef: "registry/hermes:v2", Status: "pending", BatchSize: 1, MaxUnavailable: 1}, + }, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 4: { + ID: 4, + RuntimeType: RuntimeTypeHermes, + State: "ready", + Namespace: "runtime-system", + DeploymentName: "hermes-runtime", + ImageRef: "registry/hermes:v2", + AgentEndpoint: &endpoint, + LastSeenAt: &recentSeen, + }, + }, + } + agent := &fakeRuntimeAgentClient{} + deployments := &fakeRuntimeDeploymentService{} + scheduler := NewRuntimeScheduler( + newFakeRuntimeInstanceRepo(), + podRepo, + newFakeRuntimeBindingRepo(), + rolloutRepo, + agent, + NewRuntimeEventService(nil), + nil, + deployments, + time.Second, + ) + + if err := scheduler.StartRollout(ctx, 11); err != nil { + t.Fatalf("StartRollout returned error: %v", err) + } + if got := len(agent.drainEndpoints); got != 0 { + t.Fatalf("Drain calls = %d, want 0", got) + } + if got := len(deployments.rolloutImageCalls); got != 0 { + t.Fatalf("RolloutImage calls = %d, want 0", got) + } + if len(rolloutRepo.statuses) != 2 || rolloutRepo.statuses[0].status != "running" || rolloutRepo.statuses[1].status != "finished" { + t.Fatalf("rollout statuses = %+v, want running then finished", rolloutRepo.statuses) + } +} + +func TestRuntimeSchedulerReconcileStartsPendingRollouts(t *testing.T) { + ctx := context.Background() + recentSeen := time.Now().UTC() + endpoint := "http://pod-1.runtime" + rolloutRepo := &fakeRuntimeRolloutRepo{ + rollouts: map[int64]*models.RuntimeRollout{ + 10: { + ID: 10, + RuntimeType: RuntimeTypeHermes, + TargetImageRef: "registry/hermes:v2", + Status: "pending", + BatchSize: 2, + MaxUnavailable: 1, + }, + }, + } + podRepo := &fakeRuntimePodRepo{ + pods: map[int64]*models.RuntimePod{ + 3: { + ID: 3, + RuntimeType: RuntimeTypeHermes, + State: "ready", + Namespace: "runtime-system", + DeploymentName: "hermes-runtime", + AgentEndpoint: &endpoint, + LastSeenAt: &recentSeen, + }, + }, + } + deployments := &fakeRuntimeDeploymentService{} + scheduler := NewRuntimeScheduler( + newFakeRuntimeInstanceRepo(), + podRepo, + newFakeRuntimeBindingRepo(), + rolloutRepo, + &fakeRuntimeAgentClient{}, + NewRuntimeEventService(nil), + nil, + deployments, + time.Second, + ) + + if err := scheduler.reconcile(ctx); err != nil { + t.Fatalf("reconcile returned error: %v", err) + } + if got := len(deployments.rolloutImageCalls); got != 1 { + t.Fatalf("deployment RolloutImage calls = %d, want 1", got) + } + call := deployments.rolloutImageCalls[0] + if call.namespace != "runtime-system" || call.name != "hermes-runtime" || call.image != "registry/hermes:v2" { + t.Fatalf("RolloutImage call = %+v, want runtime-system/hermes-runtime registry/hermes:v2", call) + } + if len(rolloutRepo.statuses) == 0 || rolloutRepo.statuses[0].status != "running" { + t.Fatalf("rollout statuses = %+v, want running", rolloutRepo.statuses) + } +} + +type fakeRuntimeInstanceRepo struct { + creating []models.Instance + desiredRunning []models.Instance + byID map[int]*models.Instance + workspacePaths map[int]string + runtimeStates map[int]fakeRuntimeState + updateRuntimeStateErr error + setWorkspacePathErr error +} + +type fakeRuntimeState struct { + status string + generation int + message *string +} + +func newFakeRuntimeInstanceRepo() *fakeRuntimeInstanceRepo { + return &fakeRuntimeInstanceRepo{ + byID: map[int]*models.Instance{}, + workspacePaths: map[int]string{}, + runtimeStates: map[int]fakeRuntimeState{}, + } +} + +func (r *fakeRuntimeInstanceRepo) Create(instance *models.Instance) error { return nil } +func (r *fakeRuntimeInstanceRepo) GetByID(id int) (*models.Instance, error) { + return r.byID[id], nil +} +func (r *fakeRuntimeInstanceRepo) GetByAccessToken(accessToken string) (*models.Instance, error) { + return nil, nil +} +func (r *fakeRuntimeInstanceRepo) GetByAgentBootstrapToken(bootstrapToken string) (*models.Instance, error) { + return nil, nil +} +func (r *fakeRuntimeInstanceRepo) GetAll(offset, limit int) ([]models.Instance, error) { + return nil, nil +} +func (r *fakeRuntimeInstanceRepo) CountAll() (int, error) { return 0, nil } +func (r *fakeRuntimeInstanceRepo) GetByUserID(userID int, offset, limit int) ([]models.Instance, error) { + return nil, nil +} +func (r *fakeRuntimeInstanceRepo) CountByUserID(userID int) (int, error) { return 0, nil } +func (r *fakeRuntimeInstanceRepo) CountActiveByMode(ctx context.Context, mode string) (int, error) { + return 0, nil +} +func (r *fakeRuntimeInstanceRepo) ExistsByUserIDAndName(userID int, name string) (bool, error) { + return false, nil +} +func (r *fakeRuntimeInstanceRepo) GetAllRunning() ([]models.Instance, error) { return nil, nil } +func (r *fakeRuntimeInstanceRepo) GetV2DesiredRunning(ctx context.Context, limit int) ([]models.Instance, error) { + return r.desiredRunning, nil +} +func (r *fakeRuntimeInstanceRepo) GetV2Creating(ctx context.Context, limit int) ([]models.Instance, error) { + return r.creating, nil +} +func (r *fakeRuntimeInstanceRepo) UpdateRuntimeState(ctx context.Context, id int, status string, generation int, message *string) error { + if r.updateRuntimeStateErr != nil { + return r.updateRuntimeStateErr + } + r.runtimeStates[id] = fakeRuntimeState{status: status, generation: generation, message: message} + return nil +} +func (r *fakeRuntimeInstanceRepo) SetWorkspacePath(ctx context.Context, id int, workspacePath string) error { + if r.setWorkspacePathErr != nil { + return r.setWorkspacePathErr + } + r.workspacePaths[id] = workspacePath + return nil +} +func (r *fakeRuntimeInstanceRepo) UpdateWorkspaceUsage(ctx context.Context, id int, usageBytes int64) error { + return nil +} +func (r *fakeRuntimeInstanceRepo) Update(instance *models.Instance) error { return nil } +func (r *fakeRuntimeInstanceRepo) Delete(id int) error { return nil } + +type fakeRuntimePodRepo struct { + pods map[int64]*models.RuntimePod + schedulable []models.RuntimePod + claims map[int64]int + releases map[int64]int + marked map[int64]fakePodMark + releaseCount int + getErr error +} + +type fakePodMark struct { + state string + draining bool +} + +func (r *fakeRuntimePodRepo) UpsertFromAgent(ctx context.Context, pod *models.RuntimePod) error { + return nil +} +func (r *fakeRuntimePodRepo) GetByID(ctx context.Context, id int64) (*models.RuntimePod, error) { + if r.getErr != nil { + return nil, r.getErr + } + return r.pods[id], nil +} +func (r *fakeRuntimePodRepo) GetByNamespaceName(ctx context.Context, namespace, podName string) (*models.RuntimePod, error) { + return nil, nil +} +func (r *fakeRuntimePodRepo) List(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) { + var pods []models.RuntimePod + for _, pod := range r.pods { + if runtimeType == "" || pod.RuntimeType == runtimeType { + pods = append(pods, *pod) + } + } + return pods, nil +} +func (r *fakeRuntimePodRepo) ListSchedulable(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) { + return r.schedulable, nil +} +func (r *fakeRuntimePodRepo) TryClaimSlot(ctx context.Context, podID int64) (bool, error) { + if r.claims == nil { + r.claims = map[int64]int{} + } + r.claims[podID]++ + return true, nil +} +func (r *fakeRuntimePodRepo) ReleaseSlot(ctx context.Context, podID int64) error { + if r.releases == nil { + r.releases = map[int64]int{} + } + r.releases[podID]++ + r.releaseCount++ + return nil +} +func (r *fakeRuntimePodRepo) MarkState(ctx context.Context, podID int64, state string, draining bool) error { + if r.marked == nil { + r.marked = map[int64]fakePodMark{} + } + r.marked[podID] = fakePodMark{state: state, draining: draining} + return nil +} +func (r *fakeRuntimePodRepo) UpdateHeartbeat(ctx context.Context, podID int64, state string, usedSlots int, capacity int, draining bool, lastSeenAt time.Time) error { + if r.pods != nil { + if pod := r.pods[podID]; pod != nil { + pod.State = state + pod.UsedSlots = usedSlots + pod.Capacity = capacity + pod.Draining = draining + pod.LastSeenAt = &lastSeenAt + } + } + return nil +} +func (r *fakeRuntimePodRepo) MarkUnseenUnhealthy(ctx context.Context, cutoff time.Time) error { + return nil +} +func (r *fakeRuntimePodRepo) UpdateMetrics(ctx context.Context, podID int64, metrics repository.RuntimePodMetricsUpdate) error { + return nil +} + +type fakeRuntimeBindingRepo struct { + bindings map[int]*models.InstanceRuntimeBinding + deleteCalls map[int]int + deleteAndReleaseCalls map[int]int + runningErr error + getErr error + createErr error +} + +func newFakeRuntimeBindingRepo() *fakeRuntimeBindingRepo { + return &fakeRuntimeBindingRepo{ + bindings: map[int]*models.InstanceRuntimeBinding{}, + deleteCalls: map[int]int{}, + deleteAndReleaseCalls: map[int]int{}, + } +} + +func (r *fakeRuntimeBindingRepo) Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error { + if r.createErr != nil { + return r.createErr + } + copy := *binding + r.bindings[binding.InstanceID] = © + return nil +} +func (r *fakeRuntimeBindingRepo) GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) { + if r.getErr != nil { + return nil, r.getErr + } + return r.bindings[instanceID], nil +} +func (r *fakeRuntimeBindingRepo) GetRunningByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) { + if r.runningErr != nil { + return nil, r.runningErr + } + binding := r.bindings[instanceID] + if binding == nil || binding.State != "running" { + return nil, nil + } + return binding, nil +} +func (r *fakeRuntimeBindingRepo) ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error) { + var bindings []models.InstanceRuntimeBinding + for _, binding := range r.bindings { + if binding.RuntimePodID == runtimePodID { + bindings = append(bindings, *binding) + } + } + return bindings, nil +} +func (r *fakeRuntimeBindingRepo) ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error) { + return nil, nil +} +func (r *fakeRuntimeBindingRepo) UpdateRunning(ctx context.Context, instanceID int, generation int, gatewayID string, port int, pid *int) error { + return nil +} +func (r *fakeRuntimeBindingRepo) UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error { + return nil +} +func (r *fakeRuntimeBindingRepo) DeleteByInstanceID(ctx context.Context, instanceID int) error { + r.deleteCalls[instanceID]++ + delete(r.bindings, instanceID) + return nil +} +func (r *fakeRuntimeBindingRepo) DeleteByInstanceIDAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64) error { + r.deleteAndReleaseCalls[instanceID]++ + delete(r.bindings, instanceID) + return nil +} + +type fakeRuntimeRolloutRepo struct { + rollouts map[int64]*models.RuntimeRollout + statuses []fakeRolloutStatus +} + +type fakeRolloutStatus struct { + id int64 + status string + startedAt *time.Time + finishedAt *time.Time + message *string +} + +func (r *fakeRuntimeRolloutRepo) Create(ctx context.Context, rollout *models.RuntimeRollout) error { + return nil +} +func (r *fakeRuntimeRolloutRepo) GetByID(ctx context.Context, id int64) (*models.RuntimeRollout, error) { + if r.rollouts == nil { + return nil, nil + } + return r.rollouts[id], nil +} +func (r *fakeRuntimeRolloutRepo) ListActive(ctx context.Context, runtimeType string) ([]models.RuntimeRollout, error) { + var active []models.RuntimeRollout + for _, rollout := range r.rollouts { + if rollout == nil { + continue + } + if runtimeType != "" && rollout.RuntimeType != runtimeType { + continue + } + if rollout.Status == "pending" || rollout.Status == "running" { + active = append(active, *rollout) + } + } + return active, nil +} +func (r *fakeRuntimeRolloutRepo) UpdateStatus(ctx context.Context, id int64, status string, startedAt *time.Time, finishedAt *time.Time, message *string) error { + r.statuses = append(r.statuses, fakeRolloutStatus{id: id, status: status, startedAt: startedAt, finishedAt: finishedAt, message: message}) + return nil +} + +type fakeRuntimeAgentClient struct { + createResponse *RuntimeAgentCreateGatewayResponse + createErr error + deleteErr error + createRequests []fakeCreateGatewayRequest + deleteRequests []fakeDeleteGatewayRequest + drainEndpoints []string +} + +type fakeCreateGatewayRequest struct { + endpoint string + req RuntimeAgentCreateGatewayRequest +} + +type fakeDeleteGatewayRequest struct { + endpoint string + gatewayID string +} + +func (c *fakeRuntimeAgentClient) Health(ctx context.Context, endpoint string) error { return nil } +func (c *fakeRuntimeAgentClient) CreateGateway(ctx context.Context, endpoint string, req RuntimeAgentCreateGatewayRequest) (*RuntimeAgentCreateGatewayResponse, error) { + c.createRequests = append(c.createRequests, fakeCreateGatewayRequest{endpoint: endpoint, req: req}) + if c.createErr != nil { + return nil, c.createErr + } + if c.createResponse == nil { + return nil, errors.New("missing create response") + } + return c.createResponse, nil +} +func (c *fakeRuntimeAgentClient) DeleteGateway(ctx context.Context, endpoint, gatewayID string) error { + c.deleteRequests = append(c.deleteRequests, fakeDeleteGatewayRequest{endpoint: endpoint, gatewayID: gatewayID}) + return c.deleteErr +} +func (c *fakeRuntimeAgentClient) Drain(ctx context.Context, endpoint string) error { + c.drainEndpoints = append(c.drainEndpoints, endpoint) + return nil +} + +type fakeRuntimeEventService struct { + published []fakeRuntimeEvent +} + +type fakeRuntimeEvent struct { + eventType string + payload any +} + +func (s *fakeRuntimeEventService) Publish(ctx context.Context, eventType string, payload any) error { + s.published = append(s.published, fakeRuntimeEvent{eventType: eventType, payload: payload}) + return nil +} +func (s *fakeRuntimeEventService) Read(ctx context.Context, lastID string, block time.Duration) ([]redisStreamMessage, error) { + return nil, nil +} + +type fakeRuntimeDeploymentService struct { + ensureCalls int + scaleCalls int + scales []fakeScaleCall + rolloutImageCalls []fakeRolloutImageCall + pods []k8s.RuntimeDeploymentPod +} + +type fakeScaleCall struct { + namespace string + name string + replicas int32 +} + +type fakeRolloutImageCall struct { + namespace string + name string + image string + maxUnavailable int + maxSurge int +} + +func (s *fakeRuntimeDeploymentService) Ensure(ctx context.Context, spec k8s.RuntimeDeploymentSpec) error { + s.ensureCalls++ + return nil +} +func (s *fakeRuntimeDeploymentService) Scale(ctx context.Context, namespace, name string, replicas int32) error { + s.scaleCalls++ + s.scales = append(s.scales, fakeScaleCall{namespace: namespace, name: name, replicas: replicas}) + return nil +} +func (s *fakeRuntimeDeploymentService) RolloutImage(ctx context.Context, namespace, name, image string, maxUnavailable, maxSurge int) error { + s.rolloutImageCalls = append(s.rolloutImageCalls, fakeRolloutImageCall{ + namespace: namespace, + name: name, + image: image, + maxUnavailable: maxUnavailable, + maxSurge: maxSurge, + }) + return nil +} +func (s *fakeRuntimeDeploymentService) ListPods(ctx context.Context, namespace, runtimeType string) ([]k8s.RuntimeDeploymentPod, error) { + var pods []k8s.RuntimeDeploymentPod + for _, pod := range s.pods { + if namespace != "" && pod.Namespace != namespace { + continue + } + if runtimeType != "" && pod.RuntimeType != runtimeType { + continue + } + pods = append(pods, pod) + } + return pods, nil +} + +func TestIsSchedulerManagedV2InstanceRejectsProDesktop(t *testing.T) { + workspacePath := "/workspaces/openclaw/user-7/instance-42" + instance := models.Instance{ + ID: 42, + Type: RuntimeTypeOpenClaw, + RuntimeType: RuntimeBackendDesktop, + InstanceMode: InstanceModePro, + WorkspacePath: &workspacePath, + } + if isSchedulerManagedV2Instance(instance) { + t.Fatal("pro desktop instance must not be scheduler-managed") + } + + instance.RuntimeType = RuntimeBackendGateway + instance.InstanceMode = InstanceModeLite + if !isSchedulerManagedV2Instance(instance) { + t.Fatal("lite gateway instance should be scheduler-managed") + } +} + +func ptrString(value string) *string { + return &value +} diff --git a/backend/internal/services/runtime_workspace_file_service.go b/backend/internal/services/runtime_workspace_file_service.go new file mode 100644 index 0000000..7af41fe --- /dev/null +++ b/backend/internal/services/runtime_workspace_file_service.go @@ -0,0 +1,660 @@ +package services + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" + "clawreef/internal/services/k8s" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/remotecommand" +) + +const ( + runtimeWorkspaceErrPrefix = "CLAW_WORKSPACE_ERR:" + runtimeWorkspaceBaseDir = "/config" +) + +type runtimeWorkspaceExecutor interface { + exec(ctx context.Context, userID, instanceID int, command []string, stdin io.Reader, stdout, stderr io.Writer) error +} + +type runtimeWorkspaceFileService struct { + auditRepo repository.WorkspaceFileAuditRepository + executor runtimeWorkspaceExecutor +} + +type k8sRuntimeWorkspaceExecutor struct { + deploymentService *k8s.InstanceDeploymentService +} + +type runtimeWorkspaceEntryPayload struct { + Name string `json:"name"` + Path string `json:"path"` + IsDir bool `json:"is_dir"` + Size int64 `json:"size"` + ModifiedAt time.Time `json:"modified_at"` +} + +func NewRuntimeWorkspaceFileService(auditRepo repository.WorkspaceFileAuditRepository) WorkspaceFileService { + return &runtimeWorkspaceFileService{ + auditRepo: auditRepo, + executor: &k8sRuntimeWorkspaceExecutor{ + deploymentService: k8s.NewInstanceDeploymentService(), + }, + } +} + +func (s *runtimeWorkspaceFileService) List(ctx context.Context, scope WorkspaceFileScope, relativePath string) ([]WorkspaceEntry, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + relative, err := cleanWorkspaceRelativePath(relativePath) + if err != nil { + return nil, err + } + var response struct { + Entries []runtimeWorkspaceEntryPayload `json:"entries"` + } + if err := s.runJSON(ctx, scope, []string{"python3", "-c", runtimeWorkspaceListScript, runtimeWorkspaceBase(scope), relative}, nil, &response); err != nil { + return nil, err + } + entries := make([]WorkspaceEntry, 0, len(response.Entries)) + for _, payload := range response.Entries { + entry := runtimeWorkspaceEntryFromPayload(payload) + entries = append(entries, entry) + } + sort.Slice(entries, func(i, j int) bool { + if entries[i].IsDir != entries[j].IsDir { + return entries[i].IsDir + } + left := strings.ToLower(entries[i].Name) + right := strings.ToLower(entries[j].Name) + if left == right { + return entries[i].Name < entries[j].Name + } + return left < right + }) + return entries, nil +} + +func (s *runtimeWorkspaceFileService) Preview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspacePreview, error) { + relative, entry, err := s.statFile(ctx, scope, relativePath) + if err != nil { + return nil, err + } + kind, contentType, previewable, maxBytes := workspacePreviewKind(entry.Name, entry.Size) + downloadURL := workspaceDownloadURL(scope.InstanceID, relative) + if !previewable { + return &WorkspacePreview{ + Kind: "binary", + ContentType: "application/octet-stream", + DownloadURL: downloadURL, + }, nil + } + if maxBytes > 0 && entry.Size > maxBytes { + return nil, ErrWorkspacePreviewTooLarge + } + if kind == "text" { + file, _, _, err := s.openRemoteFile(ctx, scope, relative, false) + if err != nil { + return nil, err + } + defer file.Close() + data, err := io.ReadAll(io.LimitReader(file, textPreviewMaxBytes+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > textPreviewMaxBytes { + return nil, ErrWorkspacePreviewTooLarge + } + return &WorkspacePreview{ + Kind: "text", + ContentType: contentType, + Text: string(data), + DownloadURL: downloadURL, + }, nil + } + + return &WorkspacePreview{ + Kind: kind, + ContentType: contentType, + PreviewURL: workspacePreviewURL(scope.InstanceID, relative), + DownloadURL: downloadURL, + }, nil +} + +func (s *runtimeWorkspaceFileService) OpenPreview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) { + _, entry, err := s.statFile(ctx, scope, relativePath) + if err != nil { + return nil, "", 0, err + } + _, contentType, previewable, maxBytes := workspacePreviewKind(entry.Name, entry.Size) + if !previewable { + return nil, "", 0, ErrWorkspaceFileExpected + } + if maxBytes > 0 && entry.Size > maxBytes { + return nil, "", 0, ErrWorkspacePreviewTooLarge + } + file, _, size, err := s.openRemoteFile(ctx, scope, relativePath, false) + if err != nil { + return nil, "", 0, err + } + return file, contentType, size, nil +} + +func (s *runtimeWorkspaceFileService) OpenDownload(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) { + relative, entry, err := s.statFile(ctx, scope, relativePath) + if err != nil { + return nil, "", 0, err + } + file, filename, size, err := s.openRemoteFile(ctx, scope, relative, true) + if err != nil { + return nil, "", 0, err + } + if filename == "" { + filename = entry.Name + } + return file, filename, size, nil +} + +func (s *runtimeWorkspaceFileService) Upload(ctx context.Context, scope WorkspaceFileScope, relativeDir string, filename string, reader io.Reader, size int64) (*WorkspaceEntry, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + maxBytes := WorkspaceUploadMaxBytes() + if size > maxBytes { + return nil, ErrWorkspaceUploadTooLarge + } + cleanName, err := sanitizeWorkspaceFilename(filename) + if err != nil { + return nil, err + } + dir, err := cleanWorkspaceRelativePath(relativeDir) + if err != nil { + return nil, err + } + targetRelative := joinWorkspaceRelative(dir, cleanName) + limited := io.LimitReader(reader, maxBytes+1) + var payload runtimeWorkspaceEntryPayload + if err := s.runJSON(ctx, scope, []string{ + "python3", + "-c", + runtimeWorkspaceUploadScript, + runtimeWorkspaceBase(scope), + dir, + cleanName, + fmt.Sprintf("%d", maxBytes), + }, limited, &payload); err != nil { + return nil, err + } + if err := s.recordAudit(ctx, scope, "upload", targetRelative, payload.Size); err != nil { + return nil, err + } + entry := runtimeWorkspaceEntryFromPayload(payload) + return &entry, nil +} + +func (s *runtimeWorkspaceFileService) Mkdir(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspaceEntry, error) { + if isWorkspaceRootRelative(relativePath) { + return nil, ErrWorkspaceRootOperation + } + relative, err := cleanWorkspaceRelativePath(relativePath) + if err != nil { + return nil, err + } + var payload runtimeWorkspaceEntryPayload + if err := s.runJSON(ctx, scope, []string{"python3", "-c", runtimeWorkspaceMkdirScript, runtimeWorkspaceBase(scope), relative}, nil, &payload); err != nil { + return nil, err + } + if err := s.recordAudit(ctx, scope, "mkdir", relative, 0); err != nil { + return nil, err + } + entry := runtimeWorkspaceEntryFromPayload(payload) + return &entry, nil +} + +func (s *runtimeWorkspaceFileService) Rename(ctx context.Context, scope WorkspaceFileScope, oldPath, newPath string) (*WorkspaceEntry, error) { + if isWorkspaceRootRelative(oldPath) || isWorkspaceRootRelative(newPath) { + return nil, ErrWorkspaceRootOperation + } + oldRelative, err := cleanWorkspaceRelativePath(oldPath) + if err != nil { + return nil, err + } + newRelative, err := cleanWorkspaceRelativePath(newPath) + if err != nil { + return nil, err + } + var payload runtimeWorkspaceEntryPayload + if err := s.runJSON(ctx, scope, []string{"python3", "-c", runtimeWorkspaceRenameScript, runtimeWorkspaceBase(scope), oldRelative, newRelative}, nil, &payload); err != nil { + return nil, err + } + if err := s.recordAudit(ctx, scope, "rename", oldRelative+" -> "+newRelative, 0); err != nil { + return nil, err + } + entry := runtimeWorkspaceEntryFromPayload(payload) + return &entry, nil +} + +func (s *runtimeWorkspaceFileService) Delete(ctx context.Context, scope WorkspaceFileScope, relativePath string) error { + if isWorkspaceRootRelative(relativePath) { + return ErrWorkspaceRootOperation + } + relative, err := cleanWorkspaceRelativePath(relativePath) + if err != nil { + return err + } + if err := s.run(ctx, scope, []string{"python3", "-c", runtimeWorkspaceDeleteScript, runtimeWorkspaceBase(scope), relative}, nil, io.Discard); err != nil { + return err + } + return s.recordAudit(ctx, scope, "delete", relative, 0) +} + +func (s *runtimeWorkspaceFileService) statFile(ctx context.Context, scope WorkspaceFileScope, relativePath string) (string, WorkspaceEntry, error) { + relative, err := cleanWorkspaceRelativePath(relativePath) + if err != nil { + return "", WorkspaceEntry{}, err + } + var payload runtimeWorkspaceEntryPayload + if err := s.runJSON(ctx, scope, []string{"python3", "-c", runtimeWorkspaceStatFileScript, runtimeWorkspaceBase(scope), relative}, nil, &payload); err != nil { + return "", WorkspaceEntry{}, err + } + entry := runtimeWorkspaceEntryFromPayload(payload) + return relative, entry, nil +} + +func (s *runtimeWorkspaceFileService) openRemoteFile(ctx context.Context, scope WorkspaceFileScope, relativePath string, audit bool) (*os.File, string, int64, error) { + relative, entry, err := s.statFile(ctx, scope, relativePath) + if err != nil { + return nil, "", 0, err + } + file, err := os.CreateTemp("", "clawmanager-runtime-workspace-*") + if err != nil { + return nil, "", 0, err + } + ok := false + defer func() { + if !ok { + name := file.Name() + _ = file.Close() + _ = os.Remove(name) + } + }() + if err := s.run(ctx, scope, []string{"python3", "-c", runtimeWorkspaceStreamFileScript, runtimeWorkspaceBase(scope), relative}, nil, file); err != nil { + return nil, "", 0, err + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return nil, "", 0, err + } + if audit { + if err := s.recordAudit(ctx, scope, "download", relative, entry.Size); err != nil { + return nil, "", 0, err + } + } + ok = true + return file, entry.Name, entry.Size, nil +} + +func (s *runtimeWorkspaceFileService) runJSON(ctx context.Context, scope WorkspaceFileScope, command []string, stdin io.Reader, target any) error { + var stdout bytes.Buffer + if err := s.run(ctx, scope, command, stdin, &stdout); err != nil { + return err + } + if err := json.Unmarshal(stdout.Bytes(), target); err != nil { + return fmt.Errorf("failed to parse runtime workspace response: %w", err) + } + return nil +} + +func (s *runtimeWorkspaceFileService) run(ctx context.Context, scope WorkspaceFileScope, command []string, stdin io.Reader, stdout io.Writer) error { + if s == nil || s.executor == nil { + return fmt.Errorf("runtime workspace executor is not configured") + } + var stderr bytes.Buffer + if err := s.executor.exec(ctx, scope.UserID, scope.InstanceID, command, stdin, stdout, &stderr); err != nil { + return mapRuntimeWorkspaceError(err, stderr.String()) + } + return nil +} + +func (s *runtimeWorkspaceFileService) recordAudit(ctx context.Context, scope WorkspaceFileScope, action, relativePath string, bytes int64) error { + if s.auditRepo == nil { + return nil + } + if err := ctx.Err(); err != nil { + return err + } + return s.auditRepo.Create(ctx, &models.WorkspaceFileAudit{ + InstanceID: scope.InstanceID, + UserID: scope.UserID, + Action: action, + RelativePath: relativePath, + Bytes: bytes, + CreatedAt: time.Now().UTC(), + }) +} + +func (e *k8sRuntimeWorkspaceExecutor) exec(ctx context.Context, userID, instanceID int, command []string, stdin io.Reader, stdout, stderr io.Writer) error { + client := k8s.GetClient() + if client == nil || client.Clientset == nil || client.Config == nil { + return fmt.Errorf("k8s client not initialized") + } + deploymentService := e.deploymentService + if deploymentService == nil { + deploymentService = k8s.NewInstanceDeploymentService() + } + pod, err := deploymentService.GetActivePod(ctx, userID, instanceID) + if err != nil { + return fmt.Errorf("failed to get active instance pod: %w", err) + } + req := client.Clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(pod.Name). + Namespace(pod.Namespace). + SubResource("exec") + req.VersionedParams(&corev1.PodExecOptions{ + Container: "desktop", + Command: command, + Stdin: stdin != nil, + Stdout: stdout != nil, + Stderr: stderr != nil, + TTY: false, + }, scheme.ParameterCodec) + exec, err := remotecommand.NewSPDYExecutor(client.Config, "POST", req.URL()) + if err != nil { + return fmt.Errorf("failed to initialize runtime workspace exec stream: %w", err) + } + return exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + Tty: false, + }) +} + +func runtimeWorkspaceBase(scope WorkspaceFileScope) string { + if base := strings.TrimSpace(scope.WorkspacePath); base != "" { + return base + } + return runtimeWorkspaceBaseDir +} + +func runtimeWorkspaceEntryFromPayload(payload runtimeWorkspaceEntryPayload) WorkspaceEntry { + isDir := payload.IsDir + return WorkspaceEntry{ + Name: payload.Name, + Path: filepath.ToSlash(payload.Path), + IsDir: isDir, + Size: payload.Size, + ModifiedAt: payload.ModifiedAt.UTC(), + Previewable: workspaceEntryPreviewable(payload.Name, payload.Size, isDir), + Downloadable: !isDir, + } +} + +func mapRuntimeWorkspaceError(execErr error, stderr string) error { + code := runtimeWorkspaceErrorCode(stderr) + switch code { + case "not_found": + return ErrWorkspacePathNotFound + case "escape": + return ErrWorkspacePathEscape + case "dir_required": + return ErrWorkspaceDirectoryExpected + case "file_required": + return ErrWorkspaceFileExpected + case "exists": + return ErrWorkspaceEntryExists + case "upload_too_large": + return ErrWorkspaceUploadTooLarge + case "filename_invalid", "invalid": + return ErrWorkspacePathInvalid + } + if strings.TrimSpace(stderr) != "" { + return fmt.Errorf("runtime workspace operation failed: %s", strings.TrimSpace(stderr)) + } + if execErr != nil { + return execErr + } + return errors.New("runtime workspace operation failed") +} + +func runtimeWorkspaceErrorCode(stderr string) string { + for _, line := range strings.Split(stderr, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, runtimeWorkspaceErrPrefix) { + remainder := strings.TrimPrefix(line, runtimeWorkspaceErrPrefix) + if index := strings.Index(remainder, ":"); index >= 0 { + return remainder[:index] + } + return remainder + } + } + return "" +} + +const runtimeWorkspacePythonPrelude = ` +import datetime +import json +import os +import shutil +import stat +import sys +import tempfile + +ERR_PREFIX = "CLAW_WORKSPACE_ERR:" + +def fail(code, message=""): + print(ERR_PREFIX + code + ":" + str(message), file=sys.stderr) + sys.exit(1) + +def clean_rel(value): + raw = (value or "").strip().replace("\\", "/") + if raw in ("", "."): + return "" + if "\x00" in raw: + fail("invalid", "path contains null byte") + if raw.startswith("/"): + fail("escape", "absolute paths are not allowed") + parts = [] + for part in raw.split("/"): + if part in ("", "."): + continue + if part == "..": + fail("escape", "traversal is not allowed") + parts.append(part) + return "/".join(parts) + +def safe_base(value): + base = os.path.realpath(value or "/config") + if not os.path.isdir(base): + fail("not_found", "workspace root") + return base + +def is_subpath(base, target): + try: + return os.path.commonpath([base, target]) == base + except ValueError: + return False + +def joined_path(base, rel): + target = os.path.abspath(os.path.join(base, rel)) if rel else base + if not is_subpath(base, target): + fail("escape", rel) + return target + +def target_for(base, rel, allow_missing=False): + rel = clean_rel(rel) + target = joined_path(base, rel) + if os.path.lexists(target): + real = os.path.realpath(target) + if not is_subpath(base, real): + fail("escape", rel) + return rel, target + if allow_missing: + parent = os.path.realpath(os.path.dirname(target)) + if not os.path.isdir(parent): + fail("invalid", "parent is not a directory") + if not is_subpath(base, parent): + fail("escape", rel) + return rel, target + fail("not_found", rel) + +def entry_payload(base, rel, target): + try: + st = os.lstat(target) + except FileNotFoundError: + fail("not_found", rel) + name = os.path.basename(target) if rel else os.path.basename(base) + modified = datetime.datetime.fromtimestamp(st.st_mtime, datetime.timezone.utc).isoformat().replace("+00:00", "Z") + return { + "name": name, + "path": rel, + "is_dir": stat.S_ISDIR(st.st_mode), + "size": int(st.st_size), + "modified_at": modified, + } + +def json_out(value): + print(json.dumps(value, separators=(",", ":"))) + +def chown_abc(path): + try: + import grp + import pwd + os.chown(path, pwd.getpwnam("abc").pw_uid, grp.getgrnam("abc").gr_gid) + except Exception: + pass +` + +var runtimeWorkspaceListScript = runtimeWorkspacePythonPrelude + ` +base = safe_base(sys.argv[1]) +rel, target = target_for(base, sys.argv[2] if len(sys.argv) > 2 else "") +if not os.path.isdir(target): + fail("dir_required", rel) +entries = [] +for entry in os.scandir(target): + child_rel = (rel + "/" + entry.name) if rel else entry.name + child_path = os.path.join(target, entry.name) + try: + real = os.path.realpath(child_path) + if not is_subpath(base, real): + continue + except Exception: + continue + entries.append(entry_payload(base, child_rel, child_path)) +json_out({"entries": entries}) +` + +var runtimeWorkspaceStatFileScript = runtimeWorkspacePythonPrelude + ` +base = safe_base(sys.argv[1]) +rel, target = target_for(base, sys.argv[2] if len(sys.argv) > 2 else "") +if not os.path.isfile(target): + fail("file_required", rel) +json_out(entry_payload(base, rel, target)) +` + +var runtimeWorkspaceStreamFileScript = runtimeWorkspacePythonPrelude + ` +base = safe_base(sys.argv[1]) +rel, target = target_for(base, sys.argv[2] if len(sys.argv) > 2 else "") +if not os.path.isfile(target): + fail("file_required", rel) +with open(target, "rb") as source: + shutil.copyfileobj(source, sys.stdout.buffer) +` + +var runtimeWorkspaceUploadScript = runtimeWorkspacePythonPrelude + ` +base = safe_base(sys.argv[1]) +dir_rel, dir_path = target_for(base, sys.argv[2] if len(sys.argv) > 2 else "") +name = sys.argv[3] if len(sys.argv) > 3 else "" +max_bytes = int(sys.argv[4]) if len(sys.argv) > 4 else 524288000 +if not name or name in (".", "..") or "/" in name or "\\" in name or "\x00" in name: + fail("filename_invalid", name) +if not os.path.isdir(dir_path): + fail("dir_required", dir_rel) +target_rel = (dir_rel + "/" + name) if dir_rel else name +target_rel, target = target_for(base, target_rel, allow_missing=True) +if os.path.isdir(target): + fail("file_required", target_rel) +tmp_fd, tmp_name = tempfile.mkstemp(prefix="." + name + ".tmp-", dir=dir_path) +written = 0 +try: + with os.fdopen(tmp_fd, "wb") as out: + while True: + chunk = sys.stdin.buffer.read(1024 * 1024) + if not chunk: + break + written += len(chunk) + if written > max_bytes: + fail("upload_too_large", target_rel) + out.write(chunk) + chown_abc(tmp_name) + os.replace(tmp_name, target) + chown_abc(target) + json_out(entry_payload(base, target_rel, target)) +finally: + try: + if os.path.exists(tmp_name): + os.unlink(tmp_name) + except Exception: + pass +` + +var runtimeWorkspaceMkdirScript = runtimeWorkspacePythonPrelude + ` +base = safe_base(sys.argv[1]) +rel = clean_rel(sys.argv[2] if len(sys.argv) > 2 else "") +if not rel: + fail("invalid", "workspace root cannot be modified") +rel, target = target_for(base, rel, allow_missing=True) +if os.path.lexists(target): + fail("exists", rel) +os.makedirs(target, mode=0o750, exist_ok=False) +chown_abc(target) +json_out(entry_payload(base, rel, target)) +` + +var runtimeWorkspaceRenameScript = runtimeWorkspacePythonPrelude + ` +base = safe_base(sys.argv[1]) +old_rel, old_target = target_for(base, sys.argv[2] if len(sys.argv) > 2 else "") +new_rel, new_target = target_for(base, sys.argv[3] if len(sys.argv) > 3 else "", allow_missing=True) +if not old_rel or not new_rel: + fail("invalid", "workspace root cannot be modified") +if os.path.lexists(new_target): + fail("exists", new_rel) +os.rename(old_target, new_target) +json_out(entry_payload(base, new_rel, new_target)) +` + +var runtimeWorkspaceDeleteScript = runtimeWorkspacePythonPrelude + ` +base = safe_base(sys.argv[1]) +rel, target = target_for(base, sys.argv[2] if len(sys.argv) > 2 else "") +if not rel: + fail("invalid", "workspace root cannot be modified") +if os.path.islink(target) or os.path.isfile(target): + os.unlink(target) +elif os.path.isdir(target): + shutil.rmtree(target) +else: + fail("not_found", rel) +` + +func runtimeWorkspaceContentType(name string) string { + contentType := mime.TypeByExtension(strings.ToLower(filepath.Ext(name))) + if contentType == "" { + return "application/octet-stream" + } + return contentType +} diff --git a/backend/internal/services/sync_service.go b/backend/internal/services/sync_service.go index da90987..937e22d 100644 --- a/backend/internal/services/sync_service.go +++ b/backend/internal/services/sync_service.go @@ -9,6 +9,7 @@ import ( "clawreef/internal/repository" "clawreef/internal/services/k8s" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" ) @@ -17,6 +18,7 @@ type SyncService struct { instanceRepo repository.InstanceRepository runtimeStatusService InstanceRuntimeStatusService podService *k8s.PodService + deploymentService *k8s.InstanceDeploymentService interval time.Duration stopChan chan struct{} } @@ -27,6 +29,7 @@ func NewSyncService(instanceRepo repository.InstanceRepository, runtimeStatusSer instanceRepo: instanceRepo, runtimeStatusService: runtimeStatusService, podService: k8s.NewPodService(), + deploymentService: k8s.NewInstanceDeploymentService(), interval: 5 * time.Second, // Sync every 5 seconds for more responsive status updates stopChan: make(chan struct{}), } @@ -96,6 +99,15 @@ func (s *SyncService) syncAllInstances() { // syncInstance synchronizes a single instance's state func (s *SyncService) syncInstance(ctx context.Context, instance *models.Instance) { + if _, ok := v2RuntimeTypeForInstance(instance); ok { + s.updateInfraStatus(instance.ID, instance.Status) + return + } + if instanceUsesDesktopRuntime(instance) { + s.syncDeploymentInstance(ctx, instance) + return + } + // Check if pod exists in K8s pod, err := s.podService.GetPod(ctx, instance.UserID, instance.ID) if err != nil { @@ -196,6 +208,73 @@ func (s *SyncService) syncInstance(ctx context.Context, instance *models.Instanc } } +func (s *SyncService) syncDeploymentInstance(ctx context.Context, instance *models.Instance) { + deployment, err := s.deploymentService.GetDeployment(ctx, instance.UserID, instance.ID) + if err != nil { + if instance.Status == "running" || instance.Status == "creating" { + nextStatus := "stopped" + if instance.Status == "creating" { + nextStatus = "error" + } + fmt.Printf("Instance %d marked as %s but deployment not found in K8s, updating status to %s\n", + instance.ID, instance.Status, nextStatus) + instance.Status = nextStatus + instance.PodName = nil + instance.PodNamespace = nil + instance.PodIP = nil + instance.UpdatedAt = time.Now() + if err := s.instanceRepo.Update(instance); err != nil { + fmt.Printf("Error updating instance %d status: %v\n", instance.ID, err) + } else { + s.updateInfraStatus(instance.ID, nextStatus) + GetHub().BroadcastInstanceStatus(instance.UserID, instance) + } + } + return + } + + desiredStatus := mapDeploymentToInstanceStatus(deployment) + needsUpdate := false + if instance.Status != desiredStatus { + fmt.Printf("Instance %d: Deployment replicas=%d available=%d but instance status is %s, updating to %s\n", + instance.ID, desiredReplicas(deployment), deployment.Status.AvailableReplicas, instance.Status, desiredStatus) + instance.Status = desiredStatus + needsUpdate = true + } + s.updateInfraStatus(instance.ID, desiredStatus) + + if pod, podErr := s.deploymentService.GetActivePod(ctx, instance.UserID, instance.ID); podErr == nil && pod != nil { + if pod.Status.PodIP != "" && (instance.PodIP == nil || *instance.PodIP != pod.Status.PodIP) { + instance.PodIP = &pod.Status.PodIP + needsUpdate = true + } + if instance.PodName == nil || *instance.PodName != pod.Name { + instance.PodName = &pod.Name + needsUpdate = true + } + if instance.PodNamespace == nil || *instance.PodNamespace != pod.Namespace { + instance.PodNamespace = &pod.Namespace + needsUpdate = true + } + } else if desiredStatus == "stopped" { + if instance.PodName != nil || instance.PodNamespace != nil || instance.PodIP != nil { + instance.PodName = nil + instance.PodNamespace = nil + instance.PodIP = nil + needsUpdate = true + } + } + + if needsUpdate { + instance.UpdatedAt = time.Now() + if err := s.instanceRepo.Update(instance); err != nil { + fmt.Printf("Error updating instance %d: %v\n", instance.ID, err) + } else { + GetHub().BroadcastInstanceStatus(instance.UserID, instance) + } + } +} + func (s *SyncService) updateInfraStatus(instanceID int, instanceStatus string) { if s.runtimeStatusService == nil { return @@ -243,6 +322,26 @@ func mapPodToInstanceStatus(pod *corev1.Pod) string { } } +func mapDeploymentToInstanceStatus(deployment *appsv1.Deployment) string { + if deployment == nil { + return "error" + } + if desiredReplicas(deployment) == 0 { + return "stopped" + } + if deployment.Status.AvailableReplicas > 0 { + return "running" + } + return "creating" +} + +func desiredReplicas(deployment *appsv1.Deployment) int32 { + if deployment == nil || deployment.Spec.Replicas == nil { + return 1 + } + return *deployment.Spec.Replicas +} + func isPodReady(pod *corev1.Pod) bool { for _, condition := range pod.Status.Conditions { if condition.Type == corev1.PodReady { diff --git a/backend/internal/services/system_image_setting_service.go b/backend/internal/services/system_image_setting_service.go index 93f535d..7a564c5 100644 --- a/backend/internal/services/system_image_setting_service.go +++ b/backend/internal/services/system_image_setting_service.go @@ -20,10 +20,10 @@ var orderedSystemImageTypes = []string{ } var supportedSystemImageTypes = map[string]string{ - "openclaw": "OpenClaw Desktop", + "openclaw": "OpenClaw Pro", "ubuntu": "Ubuntu Desktop", "webtop": "Webtop Desktop", - "hermes": "Hermes Runtime", + "hermes": "Hermes Pro", "debian": "Debian Desktop", "centos": "CentOS Desktop", "custom": "Custom Image", @@ -39,11 +39,11 @@ var defaultSystemImageSettings = map[string]string{ "custom": "registry.example.com/your-custom-image:latest", } -var defaultShellSystemImageSettings = map[string]string{ - "openclaw": "ghcr.io/yuan-lab-llm/agentsruntime/openclaw-shell:latest", +var defaultGatewaySystemImageSettings = map[string]string{ + "openclaw": "ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest", "ubuntu": "ubuntu:22.04", "webtop": "ubuntu:22.04", - "hermes": "ghcr.io/yuan-lab-llm/agentsruntime/hermes-shell:latest", + "hermes": "ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest", "debian": "debian:12", "centos": "quay.io/centos/centos:stream9", "custom": "registry.example.com/your-custom-shell-image:latest", @@ -55,8 +55,9 @@ var defaultEnabledSystemImageTypes = map[string]bool{ "hermes": true, } -var defaultEnabledShellSystemImageTypes = map[string]bool{ +var defaultEnabledGatewaySystemImageTypes = map[string]bool{ "openclaw": true, + "hermes": true, } // RuntimeImageConfig is the runtime card selected for an instance type. @@ -320,10 +321,16 @@ func displayNameForSystemImageType(instanceType string) string { func displayNameForSystemImagePreset(instanceType, runtimeType string) string { normalizedRuntimeType := normalizeSystemImageRuntimeType(runtimeType) if instanceType == "openclaw" { - if normalizedRuntimeType == "shell" { - return "OpenClaw Shell" + if normalizedRuntimeType == "gateway" { + return "OpenClaw Lite" } - return "OpenClaw Desktop" + return "OpenClaw Pro" + } + if instanceType == "hermes" { + if normalizedRuntimeType == "gateway" { + return "Hermes Lite" + } + return "Hermes Pro" } return displayNameForSystemImageType(instanceType) } @@ -337,12 +344,12 @@ func defaultSystemImagePresetsForType(instanceType string) []models.SystemImageS IsEnabled: defaultEnabledSystemImageTypes[instanceType], }} - if image := strings.TrimSpace(defaultShellSystemImageSettings[instanceType]); image != "" { - if defaultEnabledShellSystemImageTypes[instanceType] { + if image := strings.TrimSpace(defaultGatewaySystemImageSettings[instanceType]); image != "" { + if defaultEnabledGatewaySystemImageTypes[instanceType] { settings = append(settings, models.SystemImageSetting{ InstanceType: instanceType, - RuntimeType: "shell", - DisplayName: displayNameForSystemImagePreset(instanceType, "shell"), + RuntimeType: "gateway", + DisplayName: displayNameForSystemImagePreset(instanceType, "gateway"), Image: image, IsEnabled: true, }) @@ -359,8 +366,8 @@ func isSupportedSystemImageType(instanceType string) bool { func normalizeSystemImageRuntimeType(runtimeType string) string { normalized := strings.TrimSpace(strings.ToLower(runtimeType)) - if normalized == "shell" { - return "shell" + if normalized == "gateway" || normalized == "shell" { + return "gateway" } return "desktop" } @@ -370,7 +377,10 @@ func validateSystemImageRuntimeType(runtimeType string) (string, error) { if normalized == "" { return "desktop", nil } - if normalized != "desktop" && normalized != "shell" { + if normalized == "shell" { + return "gateway", nil + } + if normalized != "desktop" && normalized != "gateway" { return "", errors.New("unsupported runtime type") } return normalized, nil diff --git a/backend/internal/services/system_image_setting_service_test.go b/backend/internal/services/system_image_setting_service_test.go index 6e562d6..3fcd060 100644 --- a/backend/internal/services/system_image_setting_service_test.go +++ b/backend/internal/services/system_image_setting_service_test.go @@ -150,7 +150,7 @@ func TestSystemImageSettingServiceGetRuntimeImageFallsBackToDefaultWhenNoRowsExi } } -func TestSystemImageSettingServiceListIncludesOpenClawShellDefault(t *testing.T) { +func TestSystemImageSettingServiceListIncludesOpenClawGatewayDefault(t *testing.T) { service := NewSystemImageSettingService(&stubSystemImageSettingRepository{}) items, err := service.List() @@ -160,34 +160,63 @@ func TestSystemImageSettingServiceListIncludesOpenClawShellDefault(t *testing.T) found := false for _, item := range items { - if item.InstanceType == "openclaw" && item.RuntimeType == "shell" { + if item.InstanceType == "openclaw" && item.RuntimeType == "gateway" { found = true - if item.Image != defaultShellSystemImageSettings["openclaw"] { - t.Fatalf("expected default openclaw shell image %q, got %q", defaultShellSystemImageSettings["openclaw"], item.Image) + if item.Image != defaultGatewaySystemImageSettings["openclaw"] { + t.Fatalf("expected default openclaw gateway image %q, got %q", defaultGatewaySystemImageSettings["openclaw"], item.Image) } if !item.IsEnabled { - t.Fatalf("expected default openclaw shell image to be enabled") + t.Fatalf("expected default openclaw gateway image to be enabled") } } } if !found { - t.Fatalf("expected default openclaw shell runtime image") + t.Fatalf("expected default openclaw gateway runtime image") } } -func TestSystemImageSettingServiceGetRuntimeImageForImageFallsBackToOpenClawShellDefault(t *testing.T) { +func TestSystemImageSettingServiceListIncludesHermesLiteDefault(t *testing.T) { service := NewSystemImageSettingService(&stubSystemImageSettingRepository{}) - selection, ok := service.GetRuntimeImageForImage("openclaw", defaultShellSystemImageSettings["openclaw"]) + items, err := service.List() + if err != nil { + t.Fatalf("List returned error: %v", err) + } + + found := false + for _, item := range items { + if item.InstanceType == "hermes" && item.RuntimeType == "gateway" { + found = true + if item.Image != defaultGatewaySystemImageSettings["hermes"] { + t.Fatalf("expected default hermes lite image %q, got %q", defaultGatewaySystemImageSettings["hermes"], item.Image) + } + if item.DisplayName != "Hermes Lite" { + t.Fatalf("expected Hermes Lite display name, got %q", item.DisplayName) + } + if !item.IsEnabled { + t.Fatalf("expected default hermes lite image to be enabled") + } + } + } + + if !found { + t.Fatalf("expected default hermes lite runtime image") + } +} + +func TestSystemImageSettingServiceGetRuntimeImageForImageFallsBackToOpenClawGatewayDefault(t *testing.T) { + service := NewSystemImageSettingService(&stubSystemImageSettingRepository{}) + + selection, ok := service.GetRuntimeImageForImage("openclaw", defaultGatewaySystemImageSettings["openclaw"]) if !ok { - t.Fatalf("expected default openclaw shell runtime image to resolve") + t.Fatalf("expected default openclaw gateway runtime image to resolve") } - if selection.RuntimeType != "shell" { - t.Fatalf("expected runtime type shell, got %q", selection.RuntimeType) + if selection.RuntimeType != "gateway" { + t.Fatalf("expected runtime type gateway, got %q", selection.RuntimeType) } - if selection.Image != defaultShellSystemImageSettings["openclaw"] { - t.Fatalf("expected shell image %q, got %q", defaultShellSystemImageSettings["openclaw"], selection.Image) + if selection.Image != defaultGatewaySystemImageSettings["openclaw"] { + t.Fatalf("expected gateway image %q, got %q", defaultGatewaySystemImageSettings["openclaw"], selection.Image) } } @@ -195,20 +224,70 @@ func TestSystemImageSettingServiceGetRuntimeImageForImageUsesCardRuntimeType(t * repo := &stubSystemImageSettingRepository{ items: []models.SystemImageSetting{ {ID: 1, InstanceType: "openclaw", RuntimeType: "desktop", DisplayName: "OpenClaw Desktop", Image: "registry/openclaw-desktop:latest", IsEnabled: true}, - {ID: 2, InstanceType: "openclaw", RuntimeType: "shell", DisplayName: "OpenClaw Shell", Image: "registry/openclaw-shell:latest", IsEnabled: true}, + {ID: 2, InstanceType: "openclaw", RuntimeType: "gateway", DisplayName: "OpenClaw Lite", Image: "registry/openclaw-lite:latest", IsEnabled: true}, }, nextID: 2, } service := NewSystemImageSettingService(repo) - selection, ok := service.GetRuntimeImageForImage("openclaw", "registry/openclaw-shell:latest") + selection, ok := service.GetRuntimeImageForImage("openclaw", "registry/openclaw-lite:latest") if !ok { - t.Fatalf("expected selected shell runtime image to resolve") + t.Fatalf("expected selected gateway runtime image to resolve") } - if selection.RuntimeType != "shell" { - t.Fatalf("expected runtime type shell, got %q", selection.RuntimeType) + if selection.RuntimeType != "gateway" { + t.Fatalf("expected runtime type gateway, got %q", selection.RuntimeType) } - if selection.Image != "registry/openclaw-shell:latest" { - t.Fatalf("expected shell image to resolve, got %q", selection.Image) + if selection.Image != "registry/openclaw-lite:latest" { + t.Fatalf("expected gateway image to resolve, got %q", selection.Image) + } +} + +func TestSystemImageSettingServiceSaveAcceptsGatewayRuntimeType(t *testing.T) { + repo := &stubSystemImageSettingRepository{} + service := NewSystemImageSettingService(repo) + + saved, err := service.Save(&models.SystemImageSetting{ + InstanceType: "OpenClaw", + RuntimeType: "gateway", + DisplayName: "OpenClaw Lite", + Image: "registry/openclaw-lite:v2", + }) + if err != nil { + t.Fatalf("Save returned error: %v", err) + } + + if saved.InstanceType != "openclaw" { + t.Fatalf("expected normalized instance type openclaw, got %q", saved.InstanceType) + } + if saved.RuntimeType != "gateway" { + t.Fatalf("expected gateway runtime type, got %q", saved.RuntimeType) + } +} + +func TestSystemImageSettingServiceNormalizesLegacyShellRuntimeTypeToGateway(t *testing.T) { + repo := &stubSystemImageSettingRepository{ + items: []models.SystemImageSetting{ + {ID: 1, InstanceType: "hermes", RuntimeType: "shell", DisplayName: "Hermes Lite", Image: "registry/hermes-lite:legacy", IsEnabled: true}, + }, + nextID: 1, + } + service := NewSystemImageSettingService(repo) + + items, err := service.List() + if err != nil { + t.Fatalf("List returned error: %v", err) + } + for _, item := range items { + if item.ID == 1 && item.RuntimeType != "gateway" { + t.Fatalf("expected legacy shell row to be exposed as gateway, got %q", item.RuntimeType) + } + } + + selection, ok := service.GetRuntimeImageForImage("hermes", "registry/hermes-lite:legacy") + if !ok { + t.Fatalf("expected legacy shell image to resolve") + } + if selection.RuntimeType != "gateway" { + t.Fatalf("expected legacy shell image to resolve as gateway, got %q", selection.RuntimeType) } } diff --git a/backend/internal/services/team_redis.go b/backend/internal/services/team_redis.go index 4e426eb..7856551 100644 --- a/backend/internal/services/team_redis.go +++ b/backend/internal/services/team_redis.go @@ -70,6 +70,25 @@ func (b *redisBus) XAdd(ctx context.Context, key string, fields map[string]strin return id, nil } +func (b *redisBus) SetNX(ctx context.Context, key, value string, ttl time.Duration) (bool, error) { + if ttl <= 0 { + ttl = time.Second + } + reply, err := b.do(ctx, "SET", key, value, "NX", "PX", fmt.Sprintf("%d", ttl.Milliseconds())) + if err != nil { + return false, err + } + if reply == nil { + return false, nil + } + return reply == "OK", nil +} + +func (b *redisBus) Del(ctx context.Context, key string) error { + _, err := b.do(ctx, "DEL", key) + return err +} + func (b *redisBus) XRead(ctx context.Context, key, lastID string, block time.Duration) ([]redisStreamMessage, error) { blockMillis := int(block / time.Millisecond) if blockMillis <= 0 { diff --git a/backend/internal/services/team_service.go b/backend/internal/services/team_service.go index f85738c..a0fa5aa 100644 --- a/backend/internal/services/team_service.go +++ b/backend/internal/services/team_service.go @@ -31,6 +31,8 @@ const ( teamTaskStaleSweepInterval = 30 * time.Second initialLeaderTaskIntent = "team_bootstrap_introduction" + teamTaskCompletionTool = "team_complete_task" + teamTaskReplyTarget = "clawmanager" ) var ( @@ -66,6 +68,8 @@ type CreateTeamMemberRequest struct { MemberID string `json:"member_id,omitempty"` Name string `json:"name,omitempty"` Role string `json:"role"` + Mode string `json:"mode,omitempty"` + InstanceMode string `json:"instance_mode,omitempty"` RuntimeType string `json:"runtime_type,omitempty"` Description *string `json:"description,omitempty"` CPUCores float64 `json:"cpu_cores,omitempty"` @@ -130,36 +134,60 @@ type teamService struct { secretService *k8s.SecretService configMapService *k8s.ConfigMapService - ctx context.Context - cancel context.CancelFunc - mu sync.Mutex - consumers map[int]struct{} - staleMonitorStarted bool + ctx context.Context + cancel context.CancelFunc + mu sync.Mutex + consumers map[int]struct{} + staleMonitorStarted bool + runtimeWorkspaceRoot string } type plannedTeamMember struct { - Request CreateTeamMemberRequest - MemberKey string - DisplayName string - Role string - RuntimeType string - IsLeader bool + Request CreateTeamMemberRequest + MemberKey string + DisplayName string + Role string + RuntimeType string + InstanceMode string + IsLeader bool } -func NewTeamService(repo repository.TeamRepository, instanceService InstanceService) TeamService { - ctx, cancel := context.WithCancel(context.Background()) - return &teamService{ - repo: repo, - instanceService: instanceService, - pvcService: k8s.NewPVCService(), - secretService: k8s.NewSecretService(), - configMapService: k8s.NewConfigMapService(), - ctx: ctx, - cancel: cancel, - consumers: map[int]struct{}{}, +type teamRuntimeSecrets struct { + RedisURL string + Token string +} + +type TeamServiceOption func(*teamService) + +func WithTeamRuntimeWorkspaceRoot(root string) TeamServiceOption { + return func(s *teamService) { + if strings.TrimSpace(root) != "" { + s.runtimeWorkspaceRoot = strings.TrimSpace(root) + } } } +func NewTeamService(repo repository.TeamRepository, instanceService InstanceService, opts ...TeamServiceOption) TeamService { + ctx, cancel := context.WithCancel(context.Background()) + service := &teamService{ + repo: repo, + instanceService: instanceService, + pvcService: k8s.NewPVCService(), + secretService: k8s.NewSecretService(), + configMapService: k8s.NewConfigMapService(), + ctx: ctx, + cancel: cancel, + consumers: map[int]struct{}{}, + runtimeWorkspaceRoot: "/workspaces", + } + for _, opt := range opts { + if opt != nil { + opt(service) + } + } + return service +} + func (s *teamService) Start() { teams, err := s.repo.ListActiveTeams() if err != nil { @@ -252,15 +280,17 @@ func (s *teamService) CreateTeam(userID int, req CreateTeamRequest) (*TeamDetail return nil, s.rollbackTeamCreation(userID, team, err) } - if err := s.provisionTeamK8s(userID, team, redisURL, sharedStorageGB, strings.TrimSpace(req.StorageClass)); err != nil { + runtimeSecrets, err := s.provisionTeamK8s(userID, team, redisURL, sharedStorageGB, strings.TrimSpace(req.StorageClass)) + if err != nil { return nil, s.rollbackTeamCreation(userID, team, err) } - if err := s.upsertTeamRosterConfig(userID, team, memberPlans); err != nil { + rosterJSON, err := s.upsertTeamRosterConfig(userID, team, memberPlans) + if err != nil { return nil, s.rollbackTeamCreation(userID, team, err) } for _, memberPlan := range memberPlans { - member, err := s.createTeamMemberInstance(userID, team, memberPlan) + member, err := s.createTeamMemberInstance(userID, team, memberPlan, runtimeSecrets, rosterJSON) if err != nil { return nil, s.rollbackTeamCreation(userID, team, err) } @@ -339,16 +369,84 @@ func (s *teamService) recordInitialLeaderTaskDispatchFailure(teamID int, cause e }) } -func (s *teamService) provisionTeamK8s(userID int, team *models.Team, redisURL string, sharedStorageGB int, storageClass string) error { +func buildTeamTaskEnvelope(teamID int, memberKey string, task *models.TeamTask, messageID string, taskPayload map[string]interface{}, now time.Time) map[string]interface{} { + if taskPayload == nil { + taskPayload = map[string]interface{}{} + } + taskID := 0 + if task != nil { + taskID = task.ID + } + taskRef := fmt.Sprintf("team-%d-task-%d", teamID, taskID) + prompt := eventString(taskPayload, "prompt", "goal", "instruction", "instructions") + if prompt == "" { + rawPayload, _ := marshalJSON(taskPayload) + prompt = rawPayload + } + envelope := map[string]interface{}{ + "v": 1, + "messageId": messageID, + "teamId": strconv.Itoa(teamID), + "from": "clawmanager", + "to": memberKey, + "replyTo": teamTaskReplyTarget, + "requiresCompletion": true, + "completionTool": teamTaskCompletionTool, + "resultSink": map[string]interface{}{ + "type": "redis_stream", + "eventsKey": teamEventsKey(teamID), + "successEvent": "task_completed", + "failureEvent": "task_failed", + "replyEvent": "reply", + "resultField": "resultMarkdown", + "summaryField": "summary", + "artifactField": "artifactRefs", + "completionTool": teamTaskCompletionTool, + }, + "intent": eventString(taskPayload, "intent"), + "taskId": taskRef, + "title": eventString(taskPayload, "title"), + "prompt": appendTeamTaskCompletionInstruction(prompt), + "contextRefs": normalizeContextRefs(taskPayload["contextRefs"]), + "metadata": taskPayload, + "createdAt": now.Format(time.RFC3339Nano), + } + if envelope["intent"] == "" { + envelope["intent"] = "run_task" + } + if envelope["title"] == "" { + envelope["title"] = fmt.Sprintf("Team task %d", taskID) + } + return envelope +} + +func appendTeamTaskCompletionInstruction(prompt string) string { + base := strings.TrimSpace(prompt) + if strings.Contains(base, teamTaskCompletionTool) && strings.Contains(base, "task_completed") { + return base + } + instruction := strings.Join([]string{ + "Completion contract:", + "- When the final result is ready, call team_complete_task with status=\"succeeded\", summary, and resultMarkdown.", + "- If the task fails, call team_complete_task with status=\"failed\" and an error message.", + "- Do not send the final answer as a normal message to clawmanager; ClawManager consumes task_completed/task_failed events from the Team Redis event stream.", + }, "\n") + if base == "" { + return instruction + } + return base + "\n\n" + instruction +} + +func (s *teamService) provisionTeamK8s(userID int, team *models.Team, redisURL string, sharedStorageGB int, storageClass string) (*teamRuntimeSecrets, error) { ctx := context.Background() pvc, err := s.pvcService.CreateTeamSharedPVC(ctx, userID, team.ID, sharedStorageGB, storageClass) if err != nil { - return err + return nil, err } secretName := s.pvcService.GetClient().GetTeamSecretName(team.ID) teamToken, err := generatePrefixedToken("team") if err != nil { - return fmt.Errorf("failed to generate Team token: %w", err) + return nil, fmt.Errorf("failed to generate Team token: %w", err) } if err := s.secretService.UpsertSecret(ctx, userID, secretName, map[string]string{ teamRedisURLSecretKey: redisURL, @@ -358,7 +456,7 @@ func (s *teamService) provisionTeamK8s(userID int, team *models.Team, redisURL s "managed-by": "clawreef", "team-id": strconv.Itoa(team.ID), }); err != nil { - return err + return nil, err } team.RedisURLSecretName = &secretName @@ -368,21 +466,27 @@ func (s *teamService) provisionTeamK8s(userID int, team *models.Team, redisURL s team.SharedPVCName = &pvc.Name team.SharedPVCNamespace = &pvc.Namespace team.UpdatedAt = time.Now().UTC() - return s.repo.UpdateTeam(team) + if err := s.repo.UpdateTeam(team); err != nil { + return nil, err + } + return &teamRuntimeSecrets{RedisURL: redisURL, Token: teamToken}, nil } -func (s *teamService) upsertTeamRosterConfig(userID int, team *models.Team, members []plannedTeamMember) error { +func (s *teamService) upsertTeamRosterConfig(userID int, team *models.Team, members []plannedTeamMember) (string, error) { rosterJSON, err := buildTeamRosterConfig(team, members) if err != nil { - return err + return "", err } - return s.configMapService.UpsertConfigMap(context.Background(), userID, s.teamConfigMapName(team.ID), map[string]string{ + if err := s.configMapService.UpsertConfigMap(context.Background(), userID, s.teamConfigMapName(team.ID), map[string]string{ teamConfigFileName: rosterJSON, }, map[string]string{ "app": "clawreef", "managed-by": "clawreef", "team-id": strconv.Itoa(team.ID), - }) + }); err != nil { + return "", err + } + return rosterJSON, nil } func (s *teamService) teamConfigMapName(teamID int) string { @@ -393,7 +497,7 @@ func (s *teamService) teamConfigMapName(teamID int) string { return client.GetTeamConfigMapName(teamID) } -func (s *teamService) createTeamMemberInstance(userID int, team *models.Team, memberPlan plannedTeamMember) (*models.TeamMember, error) { +func (s *teamService) createTeamMemberInstance(userID int, team *models.Team, memberPlan plannedTeamMember, runtimeSecrets *teamRuntimeSecrets, rosterJSON string) (*models.TeamMember, error) { now := time.Now().UTC() member := &models.TeamMember{ TeamID: team.ID, @@ -402,6 +506,7 @@ func (s *teamService) createTeamMemberInstance(userID int, team *models.Team, me DisplayName: memberPlan.DisplayName, Role: memberPlan.Role, RuntimeType: memberPlan.RuntimeType, + InstanceMode: memberPlan.InstanceMode, Description: optionalString(strings.TrimSpace(derefTeamString(memberPlan.Request.Description))), Status: models.TeamMemberStatusCreating, Availability: models.TeamMemberAvailabilityUnknown, @@ -412,7 +517,7 @@ func (s *teamService) createTeamMemberInstance(userID int, team *models.Team, me return nil, err } - createReq := s.buildTeamMemberInstanceRequest(team, memberPlan) + createReq := s.buildTeamMemberInstanceRequestWithSecrets(team, memberPlan, runtimeSecrets, rosterJSON) instance, err := s.instanceService.Create(userID, createReq) if err != nil { member.Status = models.TeamMemberStatusFailed @@ -437,10 +542,36 @@ func (s *teamService) buildTeamMemberInstanceRequests(team *models.Team, memberP } func (s *teamService) buildTeamMemberInstanceRequest(team *models.Team, memberPlan plannedTeamMember) CreateInstanceRequest { + return s.buildTeamMemberInstanceRequestWithSecrets(team, memberPlan, nil, "") +} + +func (s *teamService) buildTeamMemberInstanceRequestWithSecrets(team *models.Team, memberPlan plannedTeamMember, runtimeSecrets *teamRuntimeSecrets, rosterJSON string) CreateInstanceRequest { req := memberPlan.Request + instanceMode := memberPlan.InstanceMode + if instanceMode == "" { + instanceMode = InstanceModeLite + } + runtimeBackendType, _ := RuntimeTypeForInstanceMode(instanceMode) + memberEnv := s.teamMemberEnv(team, memberPlan.MemberKey, memberPlan.Role) + if instanceMode == InstanceModeLite { + memberEnv["CLAWMANAGER_TEAM_SHARED_DIR"] = s.teamRuntimeSharedPath(team) + } + environmentOverrides := mergeEnvMaps(req.EnvironmentOverrides, memberEnv) + if instanceMode == InstanceModeLite && runtimeSecrets != nil { + environmentOverrides = mergeEnvMaps(environmentOverrides, map[string]string{ + teamRedisURLSecretKey: runtimeSecrets.RedisURL, + teamTokenSecretKey: runtimeSecrets.Token, + }) + if strings.TrimSpace(rosterJSON) != "" { + environmentOverrides["CLAWMANAGER_TEAM_CONFIG_JSON"] = rosterJSONWithSharedDir(rosterJSON, s.teamRuntimeSharedPath(team)) + } + } return CreateInstanceRequest{ Name: teamMemberInstanceName(team.Name, team.ID, memberPlan.MemberKey), Type: memberPlan.RuntimeType, + Mode: instanceMode, + InstanceMode: instanceMode, + RuntimeType: runtimeBackendType, CPUCores: defaultFloat(req.CPUCores, 2), MemoryGB: defaultInt(req.MemoryGB, 4), DiskGB: defaultInt(req.DiskGB, 20), @@ -450,7 +581,7 @@ func (s *teamService) buildTeamMemberInstanceRequest(team *models.Team, memberPl OSVersion: "latest", ImageRegistry: req.ImageRegistry, ImageTag: req.ImageTag, - EnvironmentOverrides: req.EnvironmentOverrides, + EnvironmentOverrides: environmentOverrides, StorageClass: derefTeamString(team.StorageClass), OpenClawConfigPlan: req.OpenClawConfigPlan, Team: &TeamInstanceConfig{ @@ -467,6 +598,13 @@ func (s *teamService) buildTeamMemberInstanceRequest(team *models.Team, memberPl } } +func (s *teamService) teamRuntimeSharedPath(team *models.Team) string { + if team == nil { + return k8s.TeamSharedWorkspacePath(s.runtimeWorkspaceRoot, 0, 0) + } + return k8s.TeamSharedWorkspacePath(s.runtimeWorkspaceRoot, team.UserID, team.ID) +} + func (s *teamService) teamMemberEnv(team *models.Team, memberKey, role string) map[string]string { managerBaseURL, _ := defaultTeamManagerBaseURL() return map[string]string{ @@ -651,30 +789,7 @@ func (s *teamService) DispatchTask(userID, teamID int, req DispatchTeamTaskReque } } now := time.Now().UTC() - envelope := map[string]interface{}{ - "v": 1, - "messageId": messageID, - "teamId": strconv.Itoa(teamID), - "from": "clawmanager", - "to": member.MemberKey, - "intent": eventString(taskPayload, "intent"), - "taskId": fmt.Sprintf("team-%d-task-%d", teamID, task.ID), - "title": eventString(taskPayload, "title"), - "prompt": eventString(taskPayload, "prompt", "goal", "instruction", "instructions"), - "contextRefs": normalizeContextRefs(taskPayload["contextRefs"]), - "metadata": taskPayload, - "createdAt": now.Format(time.RFC3339Nano), - } - if envelope["intent"] == "" { - envelope["intent"] = "run_task" - } - if envelope["title"] == "" { - envelope["title"] = fmt.Sprintf("Team task %d", task.ID) - } - if envelope["prompt"] == "" { - rawPayload, _ := marshalJSON(taskPayload) - envelope["prompt"] = rawPayload - } + envelope := buildTeamTaskEnvelope(teamID, member.MemberKey, task, messageID, taskPayload, now) envelopeJSON, err := marshalJSON(envelope) if err != nil { return nil, fmt.Errorf("failed to encode task envelope: %w", err) @@ -1090,6 +1205,7 @@ func (s *teamService) projectTeamEvent(team *models.Team, bus *redisBus, message } task = found } + eventType = normalizeFinalReplyTaskEvent(eventType, payload, task) payloadJSON, err := marshalOptionalJSON(payload) if err != nil { @@ -1184,6 +1300,35 @@ func (s *teamService) projectTeamEvent(team *models.Team, bus *redisBus, message return nil } +func normalizeFinalReplyTaskEvent(eventType string, payload map[string]interface{}, task *models.TeamTask) string { + if task == nil || !strings.EqualFold(strings.TrimSpace(eventType), "reply") { + return eventType + } + if !eventBool(payload, "final", "isFinal", "complete", "completed", "taskCompleted") { + return eventType + } + if !teamEventHasBody(payload) { + return eventType + } + payload["originalEvent"] = eventType + payload["event"] = "task_completed" + payload["type"] = "task_completed" + payload["status"] = "succeeded" + payload["availability"] = models.TeamMemberAvailabilityIdle + payload["runtimeStatus"] = "succeeded" + if eventString(payload, "resultMarkdown") == "" { + if text := eventString(payload, "text", "result", "summary"); text != "" { + payload["resultMarkdown"] = text + } + } + if eventString(payload, "summary") == "" { + if text := eventString(payload, "text", "resultMarkdown", "result"); text != "" { + payload["summary"] = text + } + } + return "task_completed" +} + func (s *teamService) enrichOutboundEventFromInbox(teamID int, bus *redisBus, payload map[string]interface{}, messageID string) (map[string]interface{}, error) { targetMember := eventString(payload, "to", "recipient", "target", "targetMemberId", "target_member_id") if targetMember == "" { @@ -1429,6 +1574,36 @@ func eventString(payload map[string]interface{}, keys ...string) string { return "" } +func eventBool(payload map[string]interface{}, keys ...string) bool { + for _, key := range keys { + value, ok := payload[key] + if !ok || value == nil { + continue + } + switch typed := value.(type) { + case bool: + return typed + case string: + switch strings.ToLower(strings.TrimSpace(typed)) { + case "1", "true", "yes", "y", "on": + return true + case "0", "false", "no", "n", "off": + return false + } + case float64: + return typed != 0 + case int: + return typed != 0 + default: + text := strings.ToLower(strings.TrimSpace(fmt.Sprintf("%v", typed))) + if text == "true" || text == "yes" || text == "1" { + return true + } + } + } + return false +} + func applyTeamMemberRuntimeProjection(member *models.TeamMember, payload map[string]interface{}, eventType string) { if member == nil { return @@ -1557,6 +1732,10 @@ func planTeamMembers(teamName string, members []CreateTeamMemberRequest) ([]plan if err != nil { return nil, err } + instanceMode, err := normalizeTeamMemberInstanceMode(memberReq.Mode, memberReq.InstanceMode) + if err != nil { + return nil, err + } isLeader := memberReq.IsLeader || isTeamLeaderRole(role) if isLeader { @@ -1568,12 +1747,13 @@ func planTeamMembers(teamName string, members []CreateTeamMemberRequest) ([]plan displayName = fmt.Sprintf("%s-%s", teamName, memberKey) } plans = append(plans, plannedTeamMember{ - Request: memberReq, - MemberKey: memberKey, - DisplayName: displayName, - Role: role, - RuntimeType: runtimeType, - IsLeader: isLeader, + Request: memberReq, + MemberKey: memberKey, + DisplayName: displayName, + Role: role, + RuntimeType: runtimeType, + InstanceMode: instanceMode, + IsLeader: isLeader, }) } if leaderCount != 1 { @@ -1636,6 +1816,22 @@ func normalizeTeamMemberRuntimeType(raw string) (string, error) { } } +func normalizeTeamMemberInstanceMode(rawMode, rawInstanceMode string) (string, error) { + if mode, ok := NormalizeInstanceMode(rawMode); ok { + return mode, nil + } + if strings.TrimSpace(rawMode) != "" { + return "", fmt.Errorf("unsupported team member instance mode: %s", rawMode) + } + if mode, ok := NormalizeInstanceMode(rawInstanceMode); ok { + return mode, nil + } + if strings.TrimSpace(rawInstanceMode) != "" { + return "", fmt.Errorf("unsupported team member instance mode: %s", rawInstanceMode) + } + return InstanceModeLite, nil +} + func normalizeTeamMemberRole(raw string, isLeader bool) string { role := strings.TrimSpace(raw) if isLeader || isTeamLeaderRole(role) { @@ -1682,12 +1878,13 @@ type teamRosterConfig struct { } type teamRosterMember struct { - MemberID string `json:"memberId"` - Role string `json:"role"` - RuntimeType string `json:"runtimeType"` - DisplayName string `json:"displayName"` - Description string `json:"description,omitempty"` - IsLeader bool `json:"isLeader"` + MemberID string `json:"memberId"` + Role string `json:"role"` + RuntimeType string `json:"runtimeType"` + InstanceMode string `json:"instanceMode"` + DisplayName string `json:"displayName"` + Description string `json:"description,omitempty"` + IsLeader bool `json:"isLeader"` } type teamRosterRedis struct { @@ -1714,12 +1911,13 @@ func buildTeamRosterConfig(team *models.Team, members []plannedTeamMember) (stri config.LeaderMemberID = member.MemberKey } config.Members = append(config.Members, teamRosterMember{ - MemberID: member.MemberKey, - Role: member.Role, - RuntimeType: member.RuntimeType, - DisplayName: member.DisplayName, - Description: derefTeamString(member.Request.Description), - IsLeader: member.IsLeader, + MemberID: member.MemberKey, + Role: member.Role, + RuntimeType: member.RuntimeType, + InstanceMode: member.InstanceMode, + DisplayName: member.DisplayName, + Description: derefTeamString(member.Request.Description), + IsLeader: member.IsLeader, }) } if config.LeaderMemberID == "" { @@ -1728,6 +1926,23 @@ func buildTeamRosterConfig(team *models.Team, members []plannedTeamMember) (stri return marshalJSON(config) } +func rosterJSONWithSharedDir(rosterJSON, sharedDir string) string { + sharedDir = strings.TrimSpace(sharedDir) + if sharedDir == "" { + return rosterJSON + } + var payload map[string]interface{} + if err := json.Unmarshal([]byte(rosterJSON), &payload); err != nil { + return rosterJSON + } + payload["sharedDir"] = sharedDir + raw, err := json.Marshal(payload) + if err != nil { + return rosterJSON + } + return string(raw) +} + func buildTeamRosterConfigFromMembers(team *models.Team, members []models.TeamMember) (string, error) { config := teamRosterConfig{ Version: 1, @@ -1747,16 +1962,21 @@ func buildTeamRosterConfigFromMembers(team *models.Team, members []models.TeamMe if runtimeType == "" { runtimeType = "openclaw" } + instanceMode := strings.TrimSpace(member.InstanceMode) + if instanceMode == "" { + instanceMode = InstanceModeLite + } if isLeader { config.LeaderMemberID = member.MemberKey } config.Members = append(config.Members, teamRosterMember{ - MemberID: member.MemberKey, - Role: member.Role, - RuntimeType: runtimeType, - DisplayName: member.DisplayName, - Description: derefTeamString(member.Description), - IsLeader: isLeader, + MemberID: member.MemberKey, + Role: member.Role, + RuntimeType: runtimeType, + InstanceMode: instanceMode, + DisplayName: member.DisplayName, + Description: derefTeamString(member.Description), + IsLeader: isLeader, }) } if config.LeaderMemberID == "" { diff --git a/backend/internal/services/team_service_test.go b/backend/internal/services/team_service_test.go index 2c27515..2d3374e 100644 --- a/backend/internal/services/team_service_test.go +++ b/backend/internal/services/team_service_test.go @@ -1,6 +1,7 @@ package services import ( + "encoding/json" "strings" "testing" "time" @@ -85,6 +86,81 @@ func TestBuildTeamMemberInstanceRequestUsesSharedPermissionDefaults(t *testing.T } } +func TestBuildTeamMemberInstanceRequestSupportsLiteMode(t *testing.T) { + service := &teamService{runtimeWorkspaceRoot: "/workspaces"} + team := &models.Team{ + UserID: 1, + ID: 8, + Name: "Lite Team", + SharedMountPath: "/team", + } + memberPlan := plannedTeamMember{ + Request: CreateTeamMemberRequest{ + Mode: "Lite", + InstanceMode: "Lite", + }, + MemberKey: "lite-worker", + DisplayName: "lite worker", + Role: "developer", + RuntimeType: "openclaw", + InstanceMode: InstanceModeLite, + } + req := service.buildTeamMemberInstanceRequest(team, memberPlan) + + if req.Mode != InstanceModeLite || req.InstanceMode != InstanceModeLite { + t.Fatalf("expected Team member instance request to preserve lite mode, got mode=%q instance_mode=%q", req.Mode, req.InstanceMode) + } + if req.RuntimeType != RuntimeBackendGateway { + t.Fatalf("expected lite Team member to target gateway runtime, got %q", req.RuntimeType) + } + + rosterJSON := `{"teamId":"8","members":[{"memberId":"lite-worker"}]}` + liteReq := service.buildTeamMemberInstanceRequestWithSecrets(team, memberPlan, &teamRuntimeSecrets{ + RedisURL: "redis://team-redis:6379/0", + Token: "team_test_token", + }, rosterJSON) + if liteReq.EnvironmentOverrides[teamRedisURLSecretKey] != "redis://team-redis:6379/0" || + liteReq.EnvironmentOverrides[teamTokenSecretKey] != "team_test_token" { + t.Fatalf("expected Lite Team runtime secrets in gateway env overrides, got %#v", liteReq.EnvironmentOverrides) + } + if !strings.Contains(liteReq.EnvironmentOverrides["CLAWMANAGER_TEAM_CONFIG_JSON"], `"sharedDir":"/workspaces/teams/user-1/team-8-shared"`) { + t.Fatalf("expected Lite Team roster JSON to include runtime shared dir, got %#v", liteReq.EnvironmentOverrides) + } +} + +func TestBuildTeamMemberInstanceRequestPointsLiteSharedDirAtRuntimeWorkspace(t *testing.T) { + service := &teamService{runtimeWorkspaceRoot: "/workspaces"} + team := &models.Team{ + UserID: 1, + ID: 28, + Name: "Mixed Team", + SharedMountPath: "/team", + } + memberPlan := plannedTeamMember{ + MemberKey: "backend", + DisplayName: "backend", + Role: "developer", + RuntimeType: "openclaw", + InstanceMode: InstanceModeLite, + } + + req := service.buildTeamMemberInstanceRequestWithSecrets(team, memberPlan, &teamRuntimeSecrets{ + RedisURL: "redis://team-redis:6379/0", + Token: "team_test_token", + }, `{"sharedDir":"/team"}`) + + wantSharedDir := "/workspaces/teams/user-1/team-28-shared" + if req.EnvironmentOverrides["CLAWMANAGER_TEAM_SHARED_DIR"] != wantSharedDir { + t.Fatalf("expected Lite shared dir %q, got %#v", wantSharedDir, req.EnvironmentOverrides) + } + if !strings.Contains(req.EnvironmentOverrides["CLAWMANAGER_TEAM_CONFIG_JSON"], `"sharedDir":"`+wantSharedDir+`"`) { + t.Fatalf("expected Lite roster JSON to use runtime shared dir, got %s", req.EnvironmentOverrides["CLAWMANAGER_TEAM_CONFIG_JSON"]) + } + if req.Team.SharedMountPath != "/team" { + t.Fatalf("Pro Team mount path should remain /team, got %q", req.Team.SharedMountPath) + } +} + func TestNewRedisBusParsesURLWithoutNetwork(t *testing.T) { bus, err := newRedisBus("redis://:pass@redis.example:6380/3") if err != nil { @@ -141,7 +217,7 @@ func TestPlanTeamMembersRequiresExactlyOneLeader(t *testing.T) { func TestPlanTeamMembersSupportsHermesRuntime(t *testing.T) { plans, err := planTeamMembers("team", []CreateTeamMemberRequest{ {MemberID: "lead", Role: "leader"}, - {MemberID: "hermes-writer", Role: "writer", RuntimeType: "Hermes"}, + {MemberID: "hermes-writer", Role: "writer", RuntimeType: "Hermes", InstanceMode: "Pro"}, }) if err != nil { t.Fatalf("planTeamMembers returned error: %v", err) @@ -149,6 +225,9 @@ func TestPlanTeamMembersSupportsHermesRuntime(t *testing.T) { if plans[1].RuntimeType != "hermes" { t.Fatalf("expected Hermes runtime to be normalized, got %#v", plans[1]) } + if plans[1].InstanceMode != InstanceModePro { + t.Fatalf("expected Pro instance mode to be normalized, got %#v", plans[1]) + } _, err = planTeamMembers("team", []CreateTeamMemberRequest{ {MemberID: "lead", Role: "leader"}, @@ -157,6 +236,14 @@ func TestPlanTeamMembersSupportsHermesRuntime(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "unsupported team member runtime type") { t.Fatalf("expected unsupported runtime validation error, got %v", err) } + + _, err = planTeamMembers("team", []CreateTeamMemberRequest{ + {MemberID: "lead", Role: "leader"}, + {MemberID: "worker", Role: "developer", Mode: "mini"}, + }) + if err == nil || !strings.Contains(err.Error(), "unsupported team member instance mode") { + t.Fatalf("expected unsupported instance mode validation error, got %v", err) + } } func TestTeamMemberInstanceNameUsesTeamIDAndMemberKey(t *testing.T) { @@ -205,6 +292,9 @@ func TestBuildTeamRosterConfigOmitsSecrets(t *testing.T) { if !strings.Contains(roster, `"runtimeType":"openclaw"`) { t.Fatalf("roster missing member runtime type: %s", roster) } + if !strings.Contains(roster, `"instanceMode":"lite"`) { + t.Fatalf("roster missing member instance mode: %s", roster) + } } func TestBuildInitialLeaderTaskPayloadDescribesRosterAndTeamSend(t *testing.T) { @@ -236,6 +326,107 @@ func TestBuildInitialLeaderTaskPayloadDescribesRosterAndTeamSend(t *testing.T) { } } +func TestBuildTeamTaskEnvelopeIncludesCompletionContract(t *testing.T) { + task := &models.TeamTask{ID: 67} + payload := map[string]interface{}{ + "intent": "team_bootstrap_introduction", + "title": "Introduce the team", + "prompt": "Generate the team report.", + } + + envelope := buildTeamTaskEnvelope(31, "leader", task, "team-31-bootstrap-introduction", payload, time.Unix(123, 0).UTC()) + + if envelope["replyTo"] != "clawmanager" { + t.Fatalf("expected replyTo clawmanager, got %#v", envelope["replyTo"]) + } + if envelope["requiresCompletion"] != true { + t.Fatalf("expected requiresCompletion=true, got %#v", envelope["requiresCompletion"]) + } + if envelope["completionTool"] != "team_complete_task" { + t.Fatalf("expected completion tool team_complete_task, got %#v", envelope["completionTool"]) + } + resultSink, ok := envelope["resultSink"].(map[string]interface{}) + if !ok { + t.Fatalf("expected resultSink map, got %#v", envelope["resultSink"]) + } + if resultSink["type"] != "redis_stream" || resultSink["eventsKey"] != "claw:team:31:events" { + t.Fatalf("unexpected resultSink: %#v", resultSink) + } + if resultSink["successEvent"] != "task_completed" || resultSink["failureEvent"] != "task_failed" { + t.Fatalf("unexpected resultSink events: %#v", resultSink) + } + prompt, ok := envelope["prompt"].(string) + if !ok { + t.Fatalf("expected prompt string, got %#v", envelope["prompt"]) + } + for _, expected := range []string{"Generate the team report.", "team_complete_task", "resultMarkdown", "task_completed"} { + if !strings.Contains(prompt, expected) { + t.Fatalf("completion prompt missing %q: %s", expected, prompt) + } + } +} + +func TestProjectTeamEventTreatsFinalReplyAsTaskCompleted(t *testing.T) { + taskID := 67 + messageID := "team-31-bootstrap-introduction" + task := &models.TeamTask{ + ID: taskID, + TeamID: 31, + TargetMemberID: 120, + MessageID: messageID, + Status: models.TeamTaskStatusRunning, + UpdatedAt: time.Now().UTC(), + } + member := &models.TeamMember{ + ID: 120, + TeamID: 31, + MemberKey: "leader", + Status: models.TeamMemberStatusBusy, + CurrentTaskID: &taskID, + Availability: models.TeamMemberAvailabilityBusy, + } + repo := &teamRepositoryStub{ + tasksByMessageID: map[string]*models.TeamTask{messageID: task}, + membersByKey: map[string]*models.TeamMember{"leader": member}, + } + service := &teamService{repo: repo} + payload := map[string]interface{}{ + "event": "reply", + "messageId": messageID, + "memberId": "leader", + "taskId": "team-31-task-67", + "final": true, + "summary": "Team report ready", + "resultMarkdown": "Full report", + "text": "Full report", + } + payloadJSON, err := json.Marshal(payload) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + + err = service.projectTeamEvent(&models.Team{ID: 31}, nil, redisStreamMessage{ + ID: "1781171178655-0", + Fields: map[string]string{"payload": string(payloadJSON)}, + }) + if err != nil { + t.Fatalf("projectTeamEvent returned error: %v", err) + } + + if repo.updatedTask == nil || repo.updatedTask.Status != models.TeamTaskStatusSucceeded { + t.Fatalf("expected final reply to mark task succeeded, got %#v", repo.updatedTask) + } + if repo.updatedTask.ResultJSON == nil || !strings.Contains(*repo.updatedTask.ResultJSON, "Full report") { + t.Fatalf("expected reply payload to become task result, got %#v", repo.updatedTask.ResultJSON) + } + if repo.updatedMember == nil || repo.updatedMember.Status != models.TeamMemberStatusIdle || repo.updatedMember.Availability != models.TeamMemberAvailabilityIdle { + t.Fatalf("expected member to become idle, got %#v", repo.updatedMember) + } + if len(repo.createdEvents) != 1 || repo.createdEvents[0].EventType != "task_completed" { + t.Fatalf("expected stored event type task_completed, got %#v", repo.createdEvents) + } +} + func TestActiveTeamMembersFiltersDeletedMembers(t *testing.T) { members := activeTeamMembers([]models.TeamMember{ {MemberKey: "leader", Status: models.TeamMemberStatusIdle}, @@ -343,3 +534,111 @@ func TestMergeMissingEventFieldsEnrichesOutboundPayload(t *testing.T) { t.Fatalf("expected enriched event to have displayable body: %#v", merged) } } + +type teamRepositoryStub struct { + teamsByID map[int]*models.Team + membersByID map[int]*models.TeamMember + membersByKey map[string]*models.TeamMember + tasksByID map[int]*models.TeamTask + tasksByMessageID map[string]*models.TeamTask + createdEvents []models.TeamEvent + updatedTask *models.TeamTask + updatedMember *models.TeamMember + updatedTeam *models.Team +} + +func (s *teamRepositoryStub) CreateTeam(team *models.Team) error { return nil } +func (s *teamRepositoryStub) UpdateTeam(team *models.Team) error { + clone := *team + s.updatedTeam = &clone + return nil +} +func (s *teamRepositoryStub) GetTeamByID(id int) (*models.Team, error) { + if s.teamsByID != nil { + return s.teamsByID[id], nil + } + return nil, nil +} +func (s *teamRepositoryStub) GetTeamByUserIDAndName(userID int, name string) (*models.Team, error) { + return nil, nil +} +func (s *teamRepositoryStub) ExistsByUserIDAndName(userID int, name string) (bool, error) { + return false, nil +} +func (s *teamRepositoryStub) ListTeamsByUserID(userID int, offset, limit int) ([]models.Team, error) { + return nil, nil +} +func (s *teamRepositoryStub) ListActiveTeams() ([]models.Team, error) { return nil, nil } +func (s *teamRepositoryStub) CountTeamsByUserID(userID int) (int, error) { + return 0, nil +} +func (s *teamRepositoryStub) CreateMember(member *models.TeamMember) error { return nil } +func (s *teamRepositoryStub) UpdateMember(member *models.TeamMember) error { + clone := *member + s.updatedMember = &clone + return nil +} +func (s *teamRepositoryStub) GetMemberByID(id int) (*models.TeamMember, error) { + if s.membersByID != nil { + return s.membersByID[id], nil + } + return nil, nil +} +func (s *teamRepositoryStub) GetMemberByTeamKey(teamID int, memberKey string) (*models.TeamMember, error) { + if s.membersByKey == nil { + return nil, nil + } + member := s.membersByKey[memberKey] + if member == nil || member.TeamID != teamID { + return nil, nil + } + return member, nil +} +func (s *teamRepositoryStub) ListMembersByTeamID(teamID int) ([]models.TeamMember, error) { + return nil, nil +} +func (s *teamRepositoryStub) CreateTask(task *models.TeamTask) error { return nil } +func (s *teamRepositoryStub) UpdateTask(task *models.TeamTask) error { + clone := *task + s.updatedTask = &clone + return nil +} +func (s *teamRepositoryStub) GetTaskByID(id int) (*models.TeamTask, error) { + if s.tasksByID != nil { + return s.tasksByID[id], nil + } + return nil, nil +} +func (s *teamRepositoryStub) GetTaskByMessageID(teamID int, messageID string) (*models.TeamTask, error) { + if s.tasksByMessageID == nil { + return nil, nil + } + task := s.tasksByMessageID[messageID] + if task == nil || task.TeamID != teamID { + return nil, nil + } + return task, nil +} +func (s *teamRepositoryStub) ListTasksByTeamID(teamID int, limit int) ([]models.TeamTask, error) { + return nil, nil +} +func (s *teamRepositoryStub) ListTasksBeforeID(teamID, beforeID, limit int) ([]models.TeamTask, error) { + return nil, nil +} +func (s *teamRepositoryStub) ListStaleCandidateTasks(cutoff time.Time, limit int) ([]models.TeamTask, error) { + return nil, nil +} +func (s *teamRepositoryStub) CreateEvent(event *models.TeamEvent) error { + clone := *event + s.createdEvents = append(s.createdEvents, clone) + return nil +} +func (s *teamRepositoryStub) EventExistsByStreamID(teamID int, streamID string) (bool, error) { + return false, nil +} +func (s *teamRepositoryStub) ListEventsByTeamID(teamID int, limit int) ([]models.TeamEvent, error) { + return nil, nil +} +func (s *teamRepositoryStub) ListEventsBeforeID(teamID, beforeID, limit int) ([]models.TeamEvent, error) { + return nil, nil +} diff --git a/backend/internal/services/websocket_service.go b/backend/internal/services/websocket_service.go index 255d464..5f13491 100644 --- a/backend/internal/services/websocket_service.go +++ b/backend/internal/services/websocket_service.go @@ -1,9 +1,11 @@ package services import ( + "context" "encoding/json" "log" "net/http" + "strings" "sync" "time" @@ -24,11 +26,20 @@ var upgrader = websocket.Upgrader{ type Client struct { ID int UserID int + Role string + Topic WebSocketTopic Conn *websocket.Conn Send chan []byte hub *Hub } +type WebSocketTopic string + +const ( + WebSocketTopicUser WebSocketTopic = "user" + WebSocketTopicRuntimeAdmin WebSocketTopic = "runtime_admin" +) + // Hub maintains the set of active clients and broadcasts messages type Hub struct { clients map[*Client]bool @@ -181,16 +192,59 @@ func (h *Hub) BroadcastToAll(msgType string, data interface{}) { h.broadcast <- msg } +func (h *Hub) BroadcastRuntimeAdmin(msgType string, data interface{}) { + msg := &Message{ + Type: msgType, + Data: data, + Timestamp: time.Now(), + } + h.broadcastRuntimeAdminMessage(msg) +} + +func (h *Hub) broadcastRuntimeAdminMessage(msg *Message) { + h.mu.RLock() + clients := make([]*Client, 0, len(h.clients)) + for client := range h.clients { + if client.Role == "admin" && client.Topic == WebSocketTopicRuntimeAdmin { + clients = append(clients, client) + } + } + h.mu.RUnlock() + + encoded := mustEncode(msg) + for _, client := range clients { + select { + case client.Send <- encoded: + default: + h.mu.Lock() + if _, ok := h.clients[client]; ok { + close(client.Send) + delete(h.clients, client) + } + h.mu.Unlock() + } + } +} + // ServeWS handles WebSocket connections func ServeWS(hub *Hub, w http.ResponseWriter, r *http.Request, userID int) { + ServeWSWithOptions(hub, w, r, userID, "", WebSocketTopicUser) +} + +func ServeWSWithOptions(hub *Hub, w http.ResponseWriter, r *http.Request, userID int, role string, topic WebSocketTopic) { conn, err := upgrader.Upgrade(w, r, nil) if err != nil { log.Printf("Failed to upgrade WebSocket: %v", err) return } + if topic == "" { + topic = WebSocketTopicUser + } client := &Client{ UserID: userID, + Role: strings.TrimSpace(role), + Topic: topic, Conn: conn, Send: make(chan []byte, 256), hub: hub, @@ -203,6 +257,92 @@ func ServeWS(hub *Hub, w http.ResponseWriter, r *http.Request, userID int) { go client.readPump() } +func StartRuntimeAdminEventBridge(ctx context.Context, events RuntimeEventService, hub *Hub) { + if events == nil || hub == nil { + return + } + go func() { + lastID := "$" + for { + select { + case <-ctx.Done(): + return + default: + } + + messages, err := events.Read(ctx, lastID, 5*time.Second) + if err != nil { + if ctx.Err() != nil { + return + } + log.Printf("runtime admin event bridge read failed: %v", err) + sleepOrDone(ctx, time.Second) + continue + } + if len(messages) == 0 { + sleepOrDone(ctx, 200*time.Millisecond) + continue + } + for _, message := range messages { + if strings.TrimSpace(message.ID) != "" { + lastID = message.ID + } + if msg, ok := runtimeEventWebSocketMessage(message); ok { + hub.broadcastRuntimeAdminMessage(msg) + } + } + } + }() +} + +func runtimeEventWebSocketMessage(message redisStreamMessage) (*Message, bool) { + eventType := strings.TrimSpace(message.Fields["type"]) + if eventType == "" { + var event RuntimeEvent + if raw := strings.TrimSpace(message.Fields["event"]); raw != "" && json.Unmarshal([]byte(raw), &event) == nil { + eventType = strings.TrimSpace(event.Type) + if eventType == "" { + return nil, false + } + return &Message{ + Type: eventType, + Data: json.RawMessage(event.Payload), + Timestamp: event.CreatedAt, + }, true + } + return nil, false + } + + createdAt := time.Now().UTC() + if rawCreatedAt := strings.TrimSpace(message.Fields["created_at"]); rawCreatedAt != "" { + if parsed, err := time.Parse(time.RFC3339Nano, rawCreatedAt); err == nil { + createdAt = parsed + } + } + var data interface{} = map[string]any{} + if rawPayload := strings.TrimSpace(message.Fields["payload"]); rawPayload != "" { + if json.Valid([]byte(rawPayload)) { + data = json.RawMessage(rawPayload) + } else { + data = rawPayload + } + } + return &Message{ + Type: eventType, + Data: data, + Timestamp: createdAt, + }, true +} + +func sleepOrDone(ctx context.Context, d time.Duration) { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + case <-timer.C: + } +} + // readPump pumps messages from the websocket connection to the hub func (c *Client) readPump() { defer func() { @@ -278,7 +418,6 @@ func (h *Hub) GetClientCount() int { return len(h.clients) } - // Stop gracefully shuts down the hub, closing all client connections. func (h *Hub) Stop() { close(h.stop) diff --git a/backend/internal/services/websocket_service_test.go b/backend/internal/services/websocket_service_test.go index d19c018..06b088c 100644 --- a/backend/internal/services/websocket_service_test.go +++ b/backend/internal/services/websocket_service_test.go @@ -1,6 +1,7 @@ package services import ( + "encoding/json" "sync" "testing" "time" @@ -86,3 +87,57 @@ func TestHubStopClosesClients(t *testing.T) { } } +func TestHubBroadcastRuntimeAdminFiltersToAdminRuntimeClients(t *testing.T) { + h := NewHub() + adminRuntime := &Client{ + UserID: 1, + Role: "admin", + Topic: WebSocketTopicRuntimeAdmin, + Send: make(chan []byte, 1), + hub: h, + } + adminUserTopic := &Client{ + UserID: 2, + Role: "admin", + Topic: WebSocketTopicUser, + Send: make(chan []byte, 1), + hub: h, + } + normalRuntime := &Client{ + UserID: 3, + Role: "user", + Topic: WebSocketTopicRuntimeAdmin, + Send: make(chan []byte, 1), + hub: h, + } + h.clients[adminRuntime] = true + h.clients[adminUserTopic] = true + h.clients[normalRuntime] = true + + h.BroadcastRuntimeAdmin("runtime_pod_metrics", map[string]any{"pod_id": int64(9)}) + + select { + case raw := <-adminRuntime.Send: + var msg Message + if err := json.Unmarshal(raw, &msg); err != nil { + t.Fatalf("runtime admin message is not json: %v", err) + } + if msg.Type != "runtime_pod_metrics" { + t.Fatalf("message type = %q, want runtime_pod_metrics", msg.Type) + } + default: + t.Fatalf("admin runtime client did not receive runtime admin message") + } + + select { + case raw := <-adminUserTopic.Send: + t.Fatalf("admin user-topic client received runtime admin message: %s", string(raw)) + default: + } + + select { + case raw := <-normalRuntime.Send: + t.Fatalf("normal runtime client received runtime admin message: %s", string(raw)) + default: + } +} diff --git a/backend/internal/services/workspace_file_service.go b/backend/internal/services/workspace_file_service.go new file mode 100644 index 0000000..cea67c9 --- /dev/null +++ b/backend/internal/services/workspace_file_service.go @@ -0,0 +1,645 @@ +package services + +import ( + "context" + "errors" + "fmt" + "io" + "mime" + "net/url" + "os" + "path" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + "unicode" + + "clawreef/internal/models" + "clawreef/internal/repository" +) + +const ( + WorkspaceUploadMaxBytesEnv = "WORKSPACE_UPLOAD_MAX_BYTES" + defaultWorkspaceUploadMax = int64(524288000) + textPreviewMaxBytes = int64(1024 * 1024) + imagePreviewMaxBytes = int64(10 * 1024 * 1024) +) + +var ( + ErrWorkspaceDirectoryExpected = errors.New("workspace directory is required") + ErrWorkspaceFileExpected = errors.New("workspace file is required") + ErrWorkspacePreviewTooLarge = errors.New("workspace preview exceeds maximum size") + ErrWorkspaceUploadTooLarge = errors.New("workspace upload exceeds maximum size") + ErrWorkspaceRootOperation = errors.New("workspace root cannot be modified") + ErrWorkspaceFileNameInvalid = errors.New("workspace filename is invalid") + ErrWorkspaceEntryExists = errors.New("workspace entry already exists") +) + +type WorkspaceFileScope struct { + InstanceID int + UserID int + WorkspacePath string +} + +type WorkspaceEntry struct { + Name string `json:"name"` + Path string `json:"path"` + IsDir bool `json:"is_dir"` + Size int64 `json:"size"` + ModifiedAt time.Time `json:"modified_at"` + Previewable bool `json:"previewable"` + Downloadable bool `json:"downloadable"` +} + +type WorkspacePreview struct { + Kind string `json:"kind"` + ContentType string `json:"content_type"` + Text string `json:"text,omitempty"` + PreviewURL string `json:"preview_url,omitempty"` + DownloadURL string `json:"download_url,omitempty"` +} + +type WorkspaceFileService interface { + List(ctx context.Context, scope WorkspaceFileScope, relativePath string) ([]WorkspaceEntry, error) + Preview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspacePreview, error) + OpenPreview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) + OpenDownload(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) + Upload(ctx context.Context, scope WorkspaceFileScope, relativeDir string, filename string, reader io.Reader, size int64) (*WorkspaceEntry, error) + Mkdir(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspaceEntry, error) + Rename(ctx context.Context, scope WorkspaceFileScope, oldPath, newPath string) (*WorkspaceEntry, error) + Delete(ctx context.Context, scope WorkspaceFileScope, relativePath string) error +} + +type workspaceFileService struct { + auditRepo repository.WorkspaceFileAuditRepository +} + +func NewWorkspaceFileService(auditRepo repository.WorkspaceFileAuditRepository) WorkspaceFileService { + return &workspaceFileService{auditRepo: auditRepo} +} + +func WorkspaceUploadMaxBytes() int64 { + raw := strings.TrimSpace(os.Getenv(WorkspaceUploadMaxBytesEnv)) + if raw == "" { + return defaultWorkspaceUploadMax + } + value, err := strconv.ParseInt(raw, 10, 64) + if err != nil || value <= 0 { + return defaultWorkspaceUploadMax + } + return value +} + +func (s *workspaceFileService) List(ctx context.Context, scope WorkspaceFileScope, relativePath string) ([]WorkspaceEntry, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + resolved, err := ResolveWorkspacePath(scope.WorkspacePath, relativePath, false) + if err != nil { + return nil, err + } + root, err := openWorkspaceRoot(scope.WorkspacePath) + if err != nil { + return nil, err + } + defer root.Close() + + dir, err := root.Open(workspaceRootName(resolved.RelativePath)) + if err != nil { + return nil, err + } + defer dir.Close() + info, err := dir.Stat() + if err != nil { + return nil, err + } + if !info.IsDir() { + return nil, ErrWorkspaceDirectoryExpected + } + + items, err := dir.ReadDir(-1) + if err != nil { + return nil, err + } + entries := make([]WorkspaceEntry, 0, len(items)) + for _, item := range items { + itemInfo, infoErr := item.Info() + if infoErr != nil { + return nil, infoErr + } + childRelative := joinWorkspaceRelative(resolved.RelativePath, item.Name()) + entries = append(entries, buildWorkspaceEntry(childRelative, itemInfo)) + } + sort.Slice(entries, func(i, j int) bool { + if entries[i].IsDir != entries[j].IsDir { + return entries[i].IsDir + } + left := strings.ToLower(entries[i].Name) + right := strings.ToLower(entries[j].Name) + if left == right { + return entries[i].Name < entries[j].Name + } + return left < right + }) + return entries, nil +} + +func (s *workspaceFileService) Preview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspacePreview, error) { + resolved, file, info, err := s.openExistingFile(ctx, scope, relativePath) + if err != nil { + return nil, err + } + defer file.Close() + + kind, contentType, previewable, maxBytes := workspacePreviewKind(info.Name(), info.Size()) + downloadURL := workspaceDownloadURL(scope.InstanceID, resolved.RelativePath) + if !previewable { + return &WorkspacePreview{ + Kind: "binary", + ContentType: "application/octet-stream", + DownloadURL: downloadURL, + }, nil + } + if maxBytes > 0 && info.Size() > maxBytes { + return nil, ErrWorkspacePreviewTooLarge + } + if kind == "text" { + data, err := io.ReadAll(io.LimitReader(file, textPreviewMaxBytes+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > textPreviewMaxBytes { + return nil, ErrWorkspacePreviewTooLarge + } + return &WorkspacePreview{ + Kind: "text", + ContentType: contentType, + Text: string(data), + DownloadURL: downloadURL, + }, nil + } + + return &WorkspacePreview{ + Kind: kind, + ContentType: contentType, + PreviewURL: workspacePreviewURL(scope.InstanceID, resolved.RelativePath), + DownloadURL: downloadURL, + }, nil +} + +func (s *workspaceFileService) OpenPreview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) { + file, info, contentType, err := s.openPreviewableFile(ctx, scope, relativePath) + if err != nil { + return nil, "", 0, err + } + return file, contentType, info.Size(), nil +} + +func (s *workspaceFileService) openPreviewableFile(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, os.FileInfo, string, error) { + _, file, info, err := s.openExistingFile(ctx, scope, relativePath) + if err != nil { + return nil, nil, "", err + } + _, contentType, previewable, maxBytes := workspacePreviewKind(info.Name(), info.Size()) + if !previewable { + file.Close() + return nil, nil, "", ErrWorkspaceFileExpected + } + if maxBytes > 0 && info.Size() > maxBytes { + file.Close() + return nil, nil, "", ErrWorkspacePreviewTooLarge + } + return file, info, contentType, nil +} + +func (s *workspaceFileService) OpenDownload(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) { + resolved, file, info, err := s.openExistingFile(ctx, scope, relativePath) + if err != nil { + return nil, "", 0, err + } + if err := s.recordAudit(ctx, scope, "download", resolved.RelativePath, info.Size()); err != nil { + file.Close() + return nil, "", 0, err + } + return file, info.Name(), info.Size(), nil +} + +func (s *workspaceFileService) Upload(ctx context.Context, scope WorkspaceFileScope, relativeDir string, filename string, reader io.Reader, size int64) (*WorkspaceEntry, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + maxBytes := WorkspaceUploadMaxBytes() + if size > maxBytes { + return nil, ErrWorkspaceUploadTooLarge + } + cleanName, err := sanitizeWorkspaceFilename(filename) + if err != nil { + return nil, err + } + dir, err := ResolveWorkspacePath(scope.WorkspacePath, relativeDir, false) + if err != nil { + return nil, err + } + root, err := openWorkspaceRoot(scope.WorkspacePath) + if err != nil { + return nil, err + } + defer root.Close() + dirInfo, err := root.Stat(workspaceRootName(dir.RelativePath)) + if err != nil { + return nil, err + } + if !dirInfo.IsDir() { + return nil, ErrWorkspaceDirectoryExpected + } + + targetRelative := joinWorkspaceRelative(dir.RelativePath, cleanName) + target, err := ResolveWorkspacePath(scope.WorkspacePath, targetRelative, true) + if err != nil { + return nil, err + } + if err := ensureWorkspaceRuntimeOwnership(scope, dir.RelativePath); err != nil { + return nil, err + } + targetName := workspaceRootName(target.RelativePath) + if targetInfo, statErr := root.Stat(targetName); statErr == nil && targetInfo.IsDir() { + return nil, ErrWorkspaceFileExpected + } else if statErr != nil && !os.IsNotExist(statErr) { + return nil, statErr + } + + tempRelative := joinWorkspaceRelative(dir.RelativePath, fmt.Sprintf(".%s.tmp-%d", cleanName, time.Now().UnixNano())) + tempName := workspaceRootName(tempRelative) + file, err := root.OpenFile(tempName, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0640) + if err != nil { + return nil, err + } + written, copyErr := io.Copy(file, io.LimitReader(reader, maxBytes+1)) + closeErr := file.Close() + if copyErr != nil { + _ = root.Remove(tempName) + return nil, copyErr + } + if closeErr != nil { + _ = root.Remove(tempName) + return nil, closeErr + } + if written > maxBytes { + _ = root.Remove(tempName) + return nil, ErrWorkspaceUploadTooLarge + } + if err := applyWorkspaceRuntimeOwnership(scope, tempRelative, 0640); err != nil { + _ = root.Remove(tempName) + return nil, err + } + if err := s.recordAudit(ctx, scope, "upload", target.RelativePath, written); err != nil { + _ = root.Remove(tempName) + return nil, err + } + if err := root.Rename(tempName, targetName); err != nil { + _ = root.Remove(tempName) + return nil, err + } + info, err := root.Stat(targetName) + if err != nil { + return nil, err + } + entry := buildWorkspaceEntry(target.RelativePath, info) + return &entry, nil +} + +func (s *workspaceFileService) Mkdir(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspaceEntry, error) { + if isWorkspaceRootRelative(relativePath) { + return nil, ErrWorkspaceRootOperation + } + resolved, err := ResolveWorkspacePath(scope.WorkspacePath, relativePath, true) + if err != nil { + return nil, err + } + root, err := openWorkspaceRoot(scope.WorkspacePath) + if err != nil { + return nil, err + } + defer root.Close() + resolvedName := workspaceRootName(resolved.RelativePath) + if _, err := root.Lstat(resolvedName); err == nil { + return nil, ErrWorkspaceEntryExists + } else if !os.IsNotExist(err) { + return nil, err + } + if err := s.recordAudit(ctx, scope, "mkdir", resolved.RelativePath, 0); err != nil { + return nil, err + } + if err := root.MkdirAll(resolvedName, 0750); err != nil { + return nil, err + } + if err := ensureWorkspaceRuntimeOwnership(scope, resolved.RelativePath); err != nil { + _ = root.RemoveAll(resolvedName) + return nil, err + } + info, err := root.Stat(resolvedName) + if err != nil { + return nil, err + } + entry := buildWorkspaceEntry(resolved.RelativePath, info) + return &entry, nil +} + +func (s *workspaceFileService) Rename(ctx context.Context, scope WorkspaceFileScope, oldPath, newPath string) (*WorkspaceEntry, error) { + if isWorkspaceRootRelative(oldPath) || isWorkspaceRootRelative(newPath) { + return nil, ErrWorkspaceRootOperation + } + oldResolved, err := ResolveWorkspacePath(scope.WorkspacePath, oldPath, false) + if err != nil { + return nil, err + } + newResolved, err := ResolveWorkspacePath(scope.WorkspacePath, newPath, true) + if err != nil { + return nil, err + } + root, err := openWorkspaceRoot(scope.WorkspacePath) + if err != nil { + return nil, err + } + defer root.Close() + oldName := workspaceRootName(oldResolved.RelativePath) + newName := workspaceRootName(newResolved.RelativePath) + if _, err := root.Lstat(newName); err == nil { + return nil, ErrWorkspaceEntryExists + } else if !os.IsNotExist(err) { + return nil, err + } + if _, err := root.Lstat(oldName); err != nil { + return nil, err + } + auditPath := oldResolved.RelativePath + " -> " + newResolved.RelativePath + if err := s.recordAudit(ctx, scope, "rename", auditPath, 0); err != nil { + return nil, err + } + if err := root.Rename(oldName, newName); err != nil { + return nil, err + } + if err := ensureWorkspaceRuntimeOwnership(scope, newResolved.RelativePath); err != nil { + return nil, err + } + info, err := root.Stat(newName) + if err != nil { + return nil, err + } + entry := buildWorkspaceEntry(newResolved.RelativePath, info) + return &entry, nil +} + +func (s *workspaceFileService) Delete(ctx context.Context, scope WorkspaceFileScope, relativePath string) error { + if isWorkspaceRootRelative(relativePath) { + return ErrWorkspaceRootOperation + } + resolved, err := ResolveWorkspacePath(scope.WorkspacePath, relativePath, false) + if err != nil { + return err + } + if err := s.recordAudit(ctx, scope, "delete", resolved.RelativePath, 0); err != nil { + return err + } + root, err := openWorkspaceRoot(scope.WorkspacePath) + if err != nil { + return err + } + defer root.Close() + return root.RemoveAll(workspaceRootName(resolved.RelativePath)) +} + +func (s *workspaceFileService) openExistingFile(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*ResolvedWorkspacePath, *os.File, os.FileInfo, error) { + if err := ctx.Err(); err != nil { + return nil, nil, nil, err + } + resolved, err := ResolveWorkspacePath(scope.WorkspacePath, relativePath, false) + if err != nil { + return nil, nil, nil, err + } + root, err := openWorkspaceRoot(scope.WorkspacePath) + if err != nil { + return nil, nil, nil, err + } + defer root.Close() + file, err := root.Open(workspaceRootName(resolved.RelativePath)) + if err != nil { + return nil, nil, nil, err + } + info, err := file.Stat() + if err != nil { + file.Close() + return nil, nil, nil, err + } + if info.IsDir() { + file.Close() + return nil, nil, nil, ErrWorkspaceFileExpected + } + return resolved, file, info, nil +} + +func (s *workspaceFileService) recordAudit(ctx context.Context, scope WorkspaceFileScope, action, relativePath string, bytes int64) error { + if s.auditRepo == nil { + return nil + } + if err := ctx.Err(); err != nil { + return err + } + return s.auditRepo.Create(ctx, &models.WorkspaceFileAudit{ + InstanceID: scope.InstanceID, + UserID: scope.UserID, + Action: action, + RelativePath: relativePath, + Bytes: bytes, + CreatedAt: time.Now().UTC(), + }) +} + +func buildWorkspaceEntry(relativePath string, info os.FileInfo) WorkspaceEntry { + isDir := info.IsDir() + return WorkspaceEntry{ + Name: info.Name(), + Path: filepath.ToSlash(relativePath), + IsDir: isDir, + Size: info.Size(), + ModifiedAt: info.ModTime().UTC(), + Previewable: workspaceEntryPreviewable(info.Name(), info.Size(), isDir), + Downloadable: !isDir, + } +} + +func workspaceEntryPreviewable(name string, size int64, isDir bool) bool { + if isDir { + return false + } + kind, _, previewable, maxBytes := workspacePreviewKind(name, size) + if !previewable { + return false + } + return kind == "pdf" || maxBytes <= 0 || size <= maxBytes +} + +func workspacePreviewKind(name string, size int64) (string, string, bool, int64) { + ext := strings.ToLower(filepath.Ext(name)) + switch ext { + case ".txt", ".md", ".json", ".yaml", ".yml", ".log", ".py", ".js", ".ts", ".go", ".sh": + return "text", "text/plain; charset=utf-8", true, textPreviewMaxBytes + case ".png", ".jpg", ".jpeg", ".gif", ".webp": + contentType := mime.TypeByExtension(ext) + if contentType == "" { + contentType = "application/octet-stream" + } + return "image", contentType, true, imagePreviewMaxBytes + case ".pdf": + return "pdf", "application/pdf", true, 0 + default: + return "binary", "application/octet-stream", false, 0 + } +} + +func sanitizeWorkspaceFilename(name string) (string, error) { + name = strings.TrimSpace(name) + if name == "" || name == "." || name == ".." { + return "", ErrWorkspaceFileNameInvalid + } + if strings.ContainsAny(name, `/\`) || strings.Contains(name, "\x00") { + return "", ErrWorkspaceFileNameInvalid + } + cleaned := strings.Map(func(r rune) rune { + if unicode.IsControl(r) { + return -1 + } + switch r { + case ':', '*', '?', '"', '<', '>', '|': + return '-' + default: + return r + } + }, name) + cleaned = strings.TrimSpace(cleaned) + if cleaned == "" || cleaned == "." || cleaned == ".." { + return "", ErrWorkspaceFileNameInvalid + } + return cleaned, nil +} + +func joinWorkspaceRelative(base, name string) string { + base = filepath.ToSlash(strings.TrimSpace(base)) + name = filepath.ToSlash(strings.TrimSpace(name)) + if base == "" { + return path.Clean(name) + } + return path.Clean(base + "/" + name) +} + +func ensureWorkspaceRuntimeOwnership(scope WorkspaceFileScope, relativePath string) error { + if _, _, ok := runtimeWorkspaceOwner(scope); !ok { + return nil + } + relative, err := cleanWorkspaceRelativePath(relativePath) + if err != nil { + return err + } + if relative == "" { + return nil + } + + parts := strings.Split(relative, "/") + for idx := range parts { + current := strings.Join(parts[:idx+1], "/") + resolved, err := ResolveWorkspacePath(scope.WorkspacePath, current, false) + if err != nil { + return err + } + info, err := os.Stat(resolved.RealPath) + if err != nil { + return err + } + mode := os.FileMode(0640) + if info.IsDir() { + mode = 0750 + } + if err := applyWorkspaceRuntimeOwnershipToPath(scope, resolved.RealPath, mode); err != nil { + return err + } + } + return nil +} + +func applyWorkspaceRuntimeOwnership(scope WorkspaceFileScope, relativePath string, mode os.FileMode) error { + if _, _, ok := runtimeWorkspaceOwner(scope); !ok { + return nil + } + resolved, err := ResolveWorkspacePath(scope.WorkspacePath, relativePath, false) + if err != nil { + return err + } + return applyWorkspaceRuntimeOwnershipToPath(scope, resolved.RealPath, mode) +} + +func applyWorkspaceRuntimeOwnershipToPath(scope WorkspaceFileScope, targetPath string, mode os.FileMode) error { + uid, gid, ok := runtimeWorkspaceOwner(scope) + if !ok { + return nil + } + if err := os.Chown(targetPath, uid, gid); err != nil { + return err + } + if err := os.Chmod(targetPath, mode); err != nil { + return err + } + return nil +} + +func runtimeWorkspaceOwner(scope WorkspaceFileScope) (int, int, bool) { + if scope.InstanceID <= 0 || scope.UserID <= 0 { + return 0, 0, false + } + cleaned := filepath.ToSlash(filepath.Clean(strings.TrimSpace(scope.WorkspacePath))) + parts := strings.Split(strings.Trim(cleaned, "/"), "/") + if len(parts) < 3 { + return 0, 0, false + } + instancePart := fmt.Sprintf("instance-%d", scope.InstanceID) + userPart := fmt.Sprintf("user-%d", scope.UserID) + runtimeType := parts[len(parts)-3] + if parts[len(parts)-1] != instancePart || parts[len(parts)-2] != userPart { + return 0, 0, false + } + if runtimeType != RuntimeTypeOpenClaw && runtimeType != RuntimeTypeHermes { + return 0, 0, false + } + linuxID := RuntimeLinuxID(scope.InstanceID) + return linuxID, linuxID, true +} + +func isWorkspaceRootRelative(relativePath string) bool { + cleaned, err := cleanWorkspaceRelativePath(relativePath) + return err == nil && cleaned == "" +} + +func openWorkspaceRoot(workspacePath string) (*os.Root, error) { + resolved, err := ResolveWorkspacePath(workspacePath, "", false) + if err != nil { + return nil, err + } + return os.OpenRoot(resolved.Root) +} + +func workspaceRootName(relativePath string) string { + relativePath = filepath.ToSlash(strings.TrimSpace(relativePath)) + if relativePath == "" { + return "." + } + return filepath.FromSlash(relativePath) +} + +func workspaceDownloadURL(instanceID int, relativePath string) string { + return fmt.Sprintf("/api/v1/instances/%d/workspace/download?path=%s", instanceID, url.QueryEscape(filepath.ToSlash(relativePath))) +} + +func workspacePreviewURL(instanceID int, relativePath string) string { + return fmt.Sprintf("/api/v1/instances/%d/workspace/preview?path=%s&raw=1", instanceID, url.QueryEscape(filepath.ToSlash(relativePath))) +} diff --git a/backend/internal/services/workspace_file_service_test.go b/backend/internal/services/workspace_file_service_test.go new file mode 100644 index 0000000..2c9d7fd --- /dev/null +++ b/backend/internal/services/workspace_file_service_test.go @@ -0,0 +1,302 @@ +package services + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "clawreef/internal/models" +) + +type recordingWorkspaceFileAuditRepo struct { + items []models.WorkspaceFileAudit + err error +} + +func (r *recordingWorkspaceFileAuditRepo) Create(ctx context.Context, audit *models.WorkspaceFileAudit) error { + if r.err != nil { + return r.err + } + r.items = append(r.items, *audit) + return nil +} + +func workspaceTestScope(root string) WorkspaceFileScope { + return WorkspaceFileScope{ + InstanceID: 12, + UserID: 34, + WorkspacePath: root, + } +} + +func TestWorkspaceFileServiceListSortsDirectoriesFirstAndMarksCapabilities(t *testing.T) { + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "docs"), 0750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "alpha.log"), []byte("hello"), 0640); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "binary.dat"), []byte{0, 1, 2}, 0640); err != nil { + t.Fatal(err) + } + + service := NewWorkspaceFileService(&recordingWorkspaceFileAuditRepo{}) + entries, err := service.List(context.Background(), workspaceTestScope(root), "") + if err != nil { + t.Fatalf("List returned error: %v", err) + } + + names := make([]string, 0, len(entries)) + for _, entry := range entries { + names = append(names, entry.Name) + } + if strings.Join(names, ",") != "docs,alpha.log,binary.dat" { + t.Fatalf("entry order = %v, want dirs first then names", names) + } + if entries[1].Path != "alpha.log" || !entries[1].Previewable || !entries[1].Downloadable { + t.Fatalf("alpha.log capabilities = %#v, want previewable downloadable file", entries[1]) + } + if entries[2].Previewable || !entries[2].Downloadable { + t.Fatalf("binary.dat capabilities = %#v, want download-only file", entries[2]) + } +} + +func TestWorkspaceFileServicePreviewTextAndDoesNotAudit(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "notes.md"), []byte("# hello"), 0640); err != nil { + t.Fatal(err) + } + auditRepo := &recordingWorkspaceFileAuditRepo{} + + service := NewWorkspaceFileService(auditRepo) + preview, err := service.Preview(context.Background(), workspaceTestScope(root), "notes.md") + if err != nil { + t.Fatalf("Preview returned error: %v", err) + } + + if preview.Kind != "text" || preview.Text != "# hello" { + t.Fatalf("preview = %#v, want text preview", preview) + } + if len(auditRepo.items) != 0 { + t.Fatalf("preview audited %d events, want none", len(auditRepo.items)) + } +} + +func TestWorkspaceFileServiceTreatsSVGAsDownloadOnly(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "icon.svg"), []byte(``), 0640); err != nil { + t.Fatal(err) + } + + service := NewWorkspaceFileService(&recordingWorkspaceFileAuditRepo{}) + entries, err := service.List(context.Background(), workspaceTestScope(root), "") + if err != nil { + t.Fatalf("List returned error: %v", err) + } + if len(entries) != 1 || entries[0].Previewable { + t.Fatalf("svg entry = %#v, want download-only", entries) + } + preview, err := service.Preview(context.Background(), workspaceTestScope(root), "icon.svg") + if err != nil { + t.Fatalf("Preview returned error: %v", err) + } + if preview.Kind != "binary" || preview.PreviewURL != "" { + t.Fatalf("svg preview = %#v, want binary without preview url", preview) + } +} + +func TestWorkspaceFileServiceRejectsLargeTextPreview(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "large.log"), []byte(strings.Repeat("a", 1024*1024+1)), 0640); err != nil { + t.Fatal(err) + } + + service := NewWorkspaceFileService(&recordingWorkspaceFileAuditRepo{}) + _, err := service.Preview(context.Background(), workspaceTestScope(root), "large.log") + if !errors.Is(err, ErrWorkspacePreviewTooLarge) { + t.Fatalf("Preview error = %v, want ErrWorkspacePreviewTooLarge", err) + } +} + +func TestWorkspaceFileServiceDownloadAuditsSuccessfulFileOpen(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "report.txt"), []byte("report"), 0640); err != nil { + t.Fatal(err) + } + auditRepo := &recordingWorkspaceFileAuditRepo{} + + service := NewWorkspaceFileService(auditRepo) + file, filename, size, err := service.OpenDownload(context.Background(), workspaceTestScope(root), "report.txt") + if err != nil { + t.Fatalf("OpenDownload returned error: %v", err) + } + file.Close() + + if filename != "report.txt" || size != int64(len("report")) { + t.Fatalf("download metadata filename=%q size=%d", filename, size) + } + if len(auditRepo.items) != 1 || auditRepo.items[0].Action != "download" || auditRepo.items[0].RelativePath != "report.txt" { + t.Fatalf("audit items = %#v, want download audit", auditRepo.items) + } +} + +func TestWorkspaceFileServiceUploadRejectsEscapingSymlinkParent(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + if err := os.Symlink(outside, filepath.Join(root, "linked")); err != nil { + t.Skipf("symlink creation is not available: %v", err) + } + + service := NewWorkspaceFileService(&recordingWorkspaceFileAuditRepo{}) + _, err := service.Upload(context.Background(), workspaceTestScope(root), "linked", "owned.txt", strings.NewReader("owned"), int64(len("owned"))) + if !errors.Is(err, ErrWorkspacePathEscape) { + t.Fatalf("Upload error = %v, want ErrWorkspacePathEscape", err) + } + if _, statErr := os.Stat(filepath.Join(outside, "owned.txt")); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("outside file stat error = %v, want not exist", statErr) + } +} + +func TestWorkspaceFileServiceUploadEnforcesMaxBytesBeforeWriting(t *testing.T) { + t.Setenv(WorkspaceUploadMaxBytesEnv, "3") + root := t.TempDir() + + service := NewWorkspaceFileService(&recordingWorkspaceFileAuditRepo{}) + _, err := service.Upload(context.Background(), workspaceTestScope(root), "", "big.txt", strings.NewReader("abcd"), int64(len("abcd"))) + if !errors.Is(err, ErrWorkspaceUploadTooLarge) { + t.Fatalf("Upload error = %v, want ErrWorkspaceUploadTooLarge", err) + } + if _, statErr := os.Stat(filepath.Join(root, "big.txt")); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("uploaded file stat error = %v, want not exist", statErr) + } +} + +func TestWorkspaceFileServiceRuntimeWorkspaceOwner(t *testing.T) { + scope := WorkspaceFileScope{ + InstanceID: 74, + UserID: 1, + WorkspacePath: "/workspaces/openclaw/user-1/instance-74", + } + uid, gid, ok := runtimeWorkspaceOwner(scope) + if !ok { + t.Fatal("runtimeWorkspaceOwner ok = false, want true") + } + if uid != RuntimeLinuxID(74) || gid != RuntimeLinuxID(74) { + t.Fatalf("runtime owner uid:gid = %d:%d, want %d:%d", uid, gid, RuntimeLinuxID(74), RuntimeLinuxID(74)) + } + + scope.WorkspacePath = "/tmp/workspaces/openclaw/user-1/instance-74" + if _, _, ok := runtimeWorkspaceOwner(scope); !ok { + t.Fatal("runtimeWorkspaceOwner should support configurable workspace roots") + } + + scope.WorkspacePath = "/workspaces/openclaw/user-2/instance-74" + if _, _, ok := runtimeWorkspaceOwner(scope); ok { + t.Fatal("runtimeWorkspaceOwner ok = true for mismatched user") + } + + scope.WorkspacePath = "/var/lib/clawmanager/user-1/instance-74" + if _, _, ok := runtimeWorkspaceOwner(scope); ok { + t.Fatal("runtimeWorkspaceOwner ok = true for non-runtime workspace") + } +} + +func TestWorkspaceFileServiceUploadAuditFailureRemovesFile(t *testing.T) { + root := t.TempDir() + auditRepo := &recordingWorkspaceFileAuditRepo{err: errors.New("audit unavailable")} + service := NewWorkspaceFileService(auditRepo) + + _, err := service.Upload(context.Background(), workspaceTestScope(root), "", "created.txt", strings.NewReader("body"), int64(len("body"))) + if err == nil || !strings.Contains(err.Error(), "audit unavailable") { + t.Fatalf("Upload error = %v, want audit failure", err) + } + if _, statErr := os.Stat(filepath.Join(root, "created.txt")); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("created file stat error = %v, want not exist", statErr) + } +} + +func TestWorkspaceFileServiceMkdirAuditFailureRemovesDirectory(t *testing.T) { + root := t.TempDir() + auditRepo := &recordingWorkspaceFileAuditRepo{err: errors.New("audit unavailable")} + service := NewWorkspaceFileService(auditRepo) + + _, err := service.Mkdir(context.Background(), workspaceTestScope(root), "docs") + if err == nil || !strings.Contains(err.Error(), "audit unavailable") { + t.Fatalf("Mkdir error = %v, want audit failure", err) + } + if _, statErr := os.Stat(filepath.Join(root, "docs")); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("created directory stat error = %v, want not exist", statErr) + } +} + +func TestWorkspaceFileServiceRenameAuditFailureRollsBack(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "old.txt"), []byte("body"), 0640); err != nil { + t.Fatal(err) + } + auditRepo := &recordingWorkspaceFileAuditRepo{err: errors.New("audit unavailable")} + service := NewWorkspaceFileService(auditRepo) + + _, err := service.Rename(context.Background(), workspaceTestScope(root), "old.txt", "new.txt") + if err == nil || !strings.Contains(err.Error(), "audit unavailable") { + t.Fatalf("Rename error = %v, want audit failure", err) + } + if _, statErr := os.Stat(filepath.Join(root, "old.txt")); statErr != nil { + t.Fatalf("old file stat error = %v, want rollback", statErr) + } + if _, statErr := os.Stat(filepath.Join(root, "new.txt")); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("new file stat error = %v, want not exist", statErr) + } +} + +func TestWorkspaceFileServiceDeleteAuditFailureKeepsFile(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "keep.txt"), []byte("body"), 0640); err != nil { + t.Fatal(err) + } + auditRepo := &recordingWorkspaceFileAuditRepo{err: errors.New("audit unavailable")} + service := NewWorkspaceFileService(auditRepo) + + err := service.Delete(context.Background(), workspaceTestScope(root), "keep.txt") + if err == nil || !strings.Contains(err.Error(), "audit unavailable") { + t.Fatalf("Delete error = %v, want audit failure", err) + } + if _, statErr := os.Stat(filepath.Join(root, "keep.txt")); statErr != nil { + t.Fatalf("kept file stat error = %v", statErr) + } +} + +func TestWorkspaceFileServiceMutationsAuditAndProtectRoot(t *testing.T) { + root := t.TempDir() + auditRepo := &recordingWorkspaceFileAuditRepo{} + service := NewWorkspaceFileService(auditRepo) + scope := workspaceTestScope(root) + + if err := service.Delete(context.Background(), scope, ""); !errors.Is(err, ErrWorkspaceRootOperation) { + t.Fatalf("Delete root error = %v, want ErrWorkspaceRootOperation", err) + } + if _, err := service.Rename(context.Background(), scope, "", "renamed"); !errors.Is(err, ErrWorkspaceRootOperation) { + t.Fatalf("Rename root error = %v, want ErrWorkspaceRootOperation", err) + } + if _, err := service.Mkdir(context.Background(), scope, "docs"); err != nil { + t.Fatalf("Mkdir returned error: %v", err) + } + if _, err := service.Rename(context.Background(), scope, "docs", "notes"); err != nil { + t.Fatalf("Rename returned error: %v", err) + } + if err := service.Delete(context.Background(), scope, "notes"); err != nil { + t.Fatalf("Delete returned error: %v", err) + } + + actions := make([]string, 0, len(auditRepo.items)) + for _, item := range auditRepo.items { + actions = append(actions, item.Action) + } + if strings.Join(actions, ",") != "mkdir,rename,delete" { + t.Fatalf("audit actions = %v, want mkdir,rename,delete", actions) + } +} diff --git a/backend/internal/services/workspace_path_guard.go b/backend/internal/services/workspace_path_guard.go new file mode 100644 index 0000000..c8b1e31 --- /dev/null +++ b/backend/internal/services/workspace_path_guard.go @@ -0,0 +1,202 @@ +package services + +import ( + "errors" + "fmt" + "os" + "path" + "path/filepath" + "runtime" + "strings" +) + +var ( + ErrWorkspacePathInvalid = errors.New("workspace path is invalid") + ErrWorkspacePathEscape = errors.New("workspace path escapes workspace") + ErrWorkspacePathNotFound = errors.New("workspace entry not found") +) + +type ResolvedWorkspacePath struct { + Root string + Path string + RealPath string + RelativePath string +} + +func ResolveWorkspacePath(workspaceRoot, relativePath string, allowMissingLeaf bool) (*ResolvedWorkspacePath, error) { + root := strings.TrimSpace(workspaceRoot) + if root == "" { + return nil, fmt.Errorf("%w: workspace root is required", ErrWorkspacePathInvalid) + } + if strings.Contains(root, "\x00") { + return nil, fmt.Errorf("%w: workspace root contains null byte", ErrWorkspacePathInvalid) + } + + rootAbs, err := filepath.Abs(root) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, err) + } + rootReal, err := filepath.EvalSymlinks(rootAbs) + if err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("%w: workspace root", ErrWorkspacePathNotFound) + } + return nil, fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, err) + } + rootReal = filepath.Clean(rootReal) + rootInfo, err := os.Stat(rootReal) + if err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("%w: workspace root", ErrWorkspacePathNotFound) + } + return nil, fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, err) + } + if !rootInfo.IsDir() { + return nil, fmt.Errorf("%w: workspace root is not a directory", ErrWorkspacePathInvalid) + } + + relative, err := cleanWorkspaceRelativePath(relativePath) + if err != nil { + return nil, err + } + + targetPath := rootReal + if relative != "" { + targetPath = filepath.Join(rootReal, filepath.FromSlash(relative)) + } + targetPath = filepath.Clean(targetPath) + if !isWorkspaceSubpath(rootReal, targetPath) { + return nil, fmt.Errorf("%w: %s", ErrWorkspacePathEscape, relativePath) + } + + realPath, err := resolveWorkspaceRealPath(targetPath, allowMissingLeaf) + if err != nil { + return nil, err + } + realPath = filepath.Clean(realPath) + if !isWorkspaceSubpath(rootReal, realPath) { + return nil, fmt.Errorf("%w: %s", ErrWorkspacePathEscape, relativePath) + } + + return &ResolvedWorkspacePath{ + Root: rootReal, + Path: targetPath, + RealPath: realPath, + RelativePath: relative, + }, nil +} + +func cleanWorkspaceRelativePath(relativePath string) (string, error) { + raw := strings.TrimSpace(relativePath) + if raw == "" || raw == "." { + return "", nil + } + if strings.Contains(raw, "\x00") { + return "", fmt.Errorf("%w: path contains null byte", ErrWorkspacePathInvalid) + } + if filepath.IsAbs(raw) || filepath.VolumeName(raw) != "" { + return "", fmt.Errorf("%w: absolute paths are not allowed", ErrWorkspacePathEscape) + } + + normalized := strings.ReplaceAll(raw, "\\", "/") + if path.IsAbs(normalized) || isWindowsDrivePath(normalized) { + return "", fmt.Errorf("%w: absolute paths are not allowed", ErrWorkspacePathEscape) + } + + parts := strings.Split(normalized, "/") + cleaned := make([]string, 0, len(parts)) + for _, part := range parts { + switch part { + case "", ".": + continue + case "..": + return "", fmt.Errorf("%w: traversal is not allowed", ErrWorkspacePathEscape) + default: + cleaned = append(cleaned, part) + } + } + if len(cleaned) == 0 { + return "", nil + } + return path.Clean(strings.Join(cleaned, "/")), nil +} + +func isWindowsDrivePath(value string) bool { + if len(value) < 2 || value[1] != ':' { + return false + } + first := value[0] + return (first >= 'a' && first <= 'z') || (first >= 'A' && first <= 'Z') +} + +func resolveWorkspaceRealPath(targetPath string, allowMissingLeaf bool) (string, error) { + realPath, err := filepath.EvalSymlinks(targetPath) + if err == nil { + return realPath, nil + } + if !os.IsNotExist(err) { + return "", fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, err) + } + if !allowMissingLeaf { + return "", fmt.Errorf("%w: %s", ErrWorkspacePathNotFound, targetPath) + } + + existingPath := targetPath + missingParts := []string{} + for { + info, statErr := os.Lstat(existingPath) + if statErr == nil { + if len(missingParts) > 0 { + info, statErr = os.Stat(existingPath) + if statErr != nil { + if os.IsNotExist(statErr) { + return "", fmt.Errorf("%w: %s", ErrWorkspacePathNotFound, existingPath) + } + return "", fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, statErr) + } + if !info.IsDir() { + return "", fmt.Errorf("%w: parent is not a directory", ErrWorkspacePathInvalid) + } + } + parentReal, evalErr := filepath.EvalSymlinks(existingPath) + if evalErr != nil { + if os.IsNotExist(evalErr) { + return "", fmt.Errorf("%w: %s", ErrWorkspacePathNotFound, existingPath) + } + return "", fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, evalErr) + } + parts := append([]string{parentReal}, missingParts...) + return filepath.Join(parts...), nil + } + if !os.IsNotExist(statErr) { + return "", fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, statErr) + } + + parent := filepath.Dir(existingPath) + if parent == existingPath { + return "", fmt.Errorf("%w: %s", ErrWorkspacePathNotFound, targetPath) + } + missingParts = append([]string{filepath.Base(existingPath)}, missingParts...) + existingPath = parent + } +} + +func isWorkspaceSubpath(root, target string) bool { + root = filepath.Clean(root) + target = filepath.Clean(target) + if sameWorkspacePath(root, target) { + return true + } + relative, err := filepath.Rel(root, target) + if err != nil || relative == "." || filepath.IsAbs(relative) { + return false + } + return relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) +} + +func sameWorkspacePath(left, right string) bool { + if runtime.GOOS == "windows" { + return strings.EqualFold(left, right) + } + return left == right +} diff --git a/backend/internal/services/workspace_path_guard_test.go b/backend/internal/services/workspace_path_guard_test.go new file mode 100644 index 0000000..1ca2f13 --- /dev/null +++ b/backend/internal/services/workspace_path_guard_test.go @@ -0,0 +1,76 @@ +package services + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestResolveWorkspacePathRejectsTraversalAndAbsolutePaths(t *testing.T) { + root := t.TempDir() + + for _, relativePath := range []string{ + "../secret.txt", + "docs/../../secret.txt", + "/etc/passwd", + `C:\Windows\System32\drivers\etc\hosts`, + "C:/Windows/System32/drivers/etc/hosts", + } { + t.Run(relativePath, func(t *testing.T) { + _, err := ResolveWorkspacePath(root, relativePath, false) + if !errors.Is(err, ErrWorkspacePathEscape) && !errors.Is(err, ErrWorkspacePathInvalid) { + t.Fatalf("ResolveWorkspacePath(%q) error = %v, want path safety error", relativePath, err) + } + }) + } +} + +func TestResolveWorkspacePathAllowsMissingChildInsideWorkspace(t *testing.T) { + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "docs"), 0750); err != nil { + t.Fatal(err) + } + + resolved, err := ResolveWorkspacePath(root, "docs/new.txt", true) + if err != nil { + t.Fatalf("ResolveWorkspacePath returned error: %v", err) + } + + if resolved.RelativePath != "docs/new.txt" { + t.Fatalf("relative path = %q, want docs/new.txt", resolved.RelativePath) + } + if !strings.HasSuffix(filepath.ToSlash(resolved.Path), "/docs/new.txt") { + t.Fatalf("target path = %q, want path under docs/new.txt", resolved.Path) + } +} + +func TestResolveWorkspacePathRejectsExistingSymlinkEscape(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + if err := os.WriteFile(filepath.Join(outside, "secret.txt"), []byte("secret"), 0640); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(outside, "secret.txt"), filepath.Join(root, "leak.txt")); err != nil { + t.Skipf("symlink creation is not available: %v", err) + } + + _, err := ResolveWorkspacePath(root, "leak.txt", false) + if !errors.Is(err, ErrWorkspacePathEscape) { + t.Fatalf("ResolveWorkspacePath through symlink error = %v, want ErrWorkspacePathEscape", err) + } +} + +func TestResolveWorkspacePathRejectsMissingTargetUnderEscapingSymlinkParent(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + if err := os.Symlink(outside, filepath.Join(root, "linked")); err != nil { + t.Skipf("symlink creation is not available: %v", err) + } + + _, err := ResolveWorkspacePath(root, "linked/new.txt", true) + if !errors.Is(err, ErrWorkspacePathEscape) { + t.Fatalf("ResolveWorkspacePath through symlink parent error = %v, want ErrWorkspacePathEscape", err) + } +} diff --git a/deployments/container/start.sh b/deployments/container/start.sh index 61174d1..a58a977 100644 --- a/deployments/container/start.sh +++ b/deployments/container/start.sh @@ -3,26 +3,84 @@ set -eu TLS_DIR="/etc/nginx/tls" TLS_SOURCE_DIR="${TLS_SOURCE_DIR:-/var/run/clawreef-tls}" +TLS_SHARED_DIR="${TLS_SHARED_DIR:-/workspaces/.clawmanager/tls}" +TLS_CN="${TLS_CN:-clawmanager.local}" +TLS_SUBJECT_ALT_NAME="${TLS_SUBJECT_ALT_NAME:-DNS:clawmanager.local,DNS:localhost,IP:127.0.0.1}" TLS_CERT="${TLS_DIR}/tls.crt" TLS_KEY="${TLS_DIR}/tls.key" SOURCE_CERT="${TLS_SOURCE_DIR}/tls.crt" SOURCE_KEY="${TLS_SOURCE_DIR}/tls.key" +SHARED_CERT="${TLS_SHARED_DIR}/tls.crt" +SHARED_KEY="${TLS_SHARED_DIR}/tls.key" +SHARED_LOCK_DIR="${TLS_SHARED_DIR}.lock" -mkdir -p "${TLS_DIR}" - -if [ -f "${SOURCE_CERT}" ] && [ -f "${SOURCE_KEY}" ]; then - cp "${SOURCE_CERT}" "${TLS_CERT}" - cp "${SOURCE_KEY}" "${TLS_KEY}" -elif [ ! -f "${TLS_CERT}" ] || [ ! -f "${TLS_KEY}" ]; then - echo "TLS certificate not found, generating a self-signed certificate for bootstrap use." +generate_self_signed_cert() { + cert_path="$1" + key_path="$2" openssl req \ -x509 \ -nodes \ -days 365 \ -newkey rsa:2048 \ - -subj "/CN=clawreef.local" \ - -keyout "${TLS_KEY}" \ - -out "${TLS_CERT}" + -subj "/CN=${TLS_CN}" \ + -addext "subjectAltName=${TLS_SUBJECT_ALT_NAME}" \ + -keyout "${key_path}" \ + -out "${cert_path}" || \ + openssl req \ + -x509 \ + -nodes \ + -days 365 \ + -newkey rsa:2048 \ + -subj "/CN=${TLS_CN}" \ + -keyout "${key_path}" \ + -out "${cert_path}" +} + +copy_cert_pair() { + cp "$1" "${TLS_CERT}" + cp "$2" "${TLS_KEY}" + chmod 0644 "${TLS_CERT}" + chmod 0600 "${TLS_KEY}" +} + +ensure_shared_cert() { + if [ -f "${SHARED_CERT}" ] && [ -f "${SHARED_KEY}" ]; then + return 0 + fi + + mkdir -p "${TLS_SHARED_DIR}" 2>/dev/null || return 1 + if mkdir "${SHARED_LOCK_DIR}" 2>/dev/null; then + if [ ! -f "${SHARED_CERT}" ] || [ ! -f "${SHARED_KEY}" ]; then + tmp_cert="${SHARED_CERT}.$$" + tmp_key="${SHARED_KEY}.$$" + generate_self_signed_cert "${tmp_cert}" "${tmp_key}" + chmod 0644 "${tmp_cert}" + chmod 0600 "${tmp_key}" + mv "${tmp_cert}" "${SHARED_CERT}" + mv "${tmp_key}" "${SHARED_KEY}" + fi + rmdir "${SHARED_LOCK_DIR}" 2>/dev/null || true + return 0 + fi + + for _ in $(seq 1 60); do + if [ -f "${SHARED_CERT}" ] && [ -f "${SHARED_KEY}" ]; then + return 0 + fi + sleep 1 + done + return 1 +} + +mkdir -p "${TLS_DIR}" + +if [ -f "${SOURCE_CERT}" ] && [ -f "${SOURCE_KEY}" ]; then + copy_cert_pair "${SOURCE_CERT}" "${SOURCE_KEY}" +elif ensure_shared_cert; then + copy_cert_pair "${SHARED_CERT}" "${SHARED_KEY}" +elif [ ! -f "${TLS_CERT}" ] || [ ! -f "${TLS_KEY}" ]; then + echo "TLS certificate not found, generating a self-signed certificate for bootstrap use." + generate_self_signed_cert "${TLS_CERT}" "${TLS_KEY}" fi export SERVER_ADDRESS="${SERVER_ADDRESS:-:9001}" diff --git a/deployments/hermes-runtime/Dockerfile.tui-dist b/deployments/hermes-runtime/Dockerfile.tui-dist new file mode 100644 index 0000000..ae45802 --- /dev/null +++ b/deployments/hermes-runtime/Dockerfile.tui-dist @@ -0,0 +1,9 @@ +ARG HERMES_RUNTIME_BASE_IMAGE=172.16.1.12:5010/hermes:team-lite-envfix-20260610123402 +FROM ${HERMES_RUNTIME_BASE_IMAGE} + +USER root + +RUN cd /usr/local/lib/hermes-agent/ui-tui \ + && npm run build \ + && test -f dist/entry.js \ + && chmod -R a+rX dist diff --git a/deployments/k3s/clawmanager.yaml b/deployments/k3s/clawmanager.yaml index 7c72dac..a264109 100644 --- a/deployments/k3s/clawmanager.yaml +++ b/deployments/k3s/clawmanager.yaml @@ -13,6 +13,9 @@ stringData: mysql-root-password: root123 mysql-password: clawreef123 jwt-secret: change-me-in-production + runtime-agent-control-token: change-me-runtime-control-token + runtime-agent-report-token: change-me-runtime-report-token + openclaw-gateway-token: change-me-openclaw-gateway-token --- apiVersion: v1 kind: ConfigMap @@ -635,8 +638,8 @@ data: SELECT 'openclaw', 'shell', - 'OpenClaw Shell', - 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-shell:latest', + 'OpenClaw Lite', + 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest', TRUE WHERE NOT EXISTS ( SELECT 1 @@ -671,6 +674,39 @@ data: updated_at = CURRENT_TIMESTAMP WHERE type IN ('openclaw', 'ubuntu', 'webtop', 'hermes') AND mount_path IN ('/data', '/home/user/data', '/config/.hermes'); + 031_add_hermes_lite_runtime_image.sql: | + USE clawmanager; + UPDATE system_image_settings + SET + image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest', + display_name = 'Hermes Lite', + is_enabled = TRUE + WHERE instance_type = 'hermes' + AND runtime_type = 'shell' + AND image IN ( + 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-shell:latest', + 'registry.example.com/your-custom-shell-image:latest' + ); + + INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) + SELECT + 'hermes', + 'shell', + 'Hermes Lite', + 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest', + TRUE + WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'hermes' + AND runtime_type = 'shell' + ) + AND NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'hermes' + AND is_enabled = FALSE + ); --- apiVersion: v1 kind: PersistentVolumeClaim @@ -792,6 +828,19 @@ spec: --- apiVersion: v1 kind: Service +metadata: + name: clawmanager-redis + namespace: clawmanager-system +spec: + selector: + app: clawmanager-team-redis + ports: + - name: redis + port: 6379 + targetPort: redis +--- +apiVersion: v1 +kind: Service metadata: name: clawmanager-team-redis namespace: clawmanager-system @@ -804,12 +853,120 @@ spec: targetPort: redis --- apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: clawmanager-workspaces + namespace: clawmanager-system +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 50Gi + storageClassName: local-path +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: workspace-store + namespace: clawmanager-system +spec: + replicas: 1 + selector: + matchLabels: + app: workspace-store + template: + metadata: + labels: + app: workspace-store + spec: + containers: + - name: nfs-server + image: itsthenetwork/nfs-server-alpine:12 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + env: + - name: SHARED_DIRECTORY + value: /exports/workspaces + ports: + - name: nfs + containerPort: 2049 + volumeMounts: + - name: workspaces + mountPath: /exports/workspaces + volumes: + - name: workspaces + persistentVolumeClaim: + claimName: clawmanager-workspaces +--- +apiVersion: v1 +kind: Service +metadata: + name: workspace-store + namespace: clawmanager-system +spec: + selector: + app: workspace-store + ports: + - name: nfs + port: 2049 + targetPort: nfs +--- +apiVersion: v1 kind: ServiceAccount metadata: name: clawmanager-app namespace: clawmanager-system --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: clawmanager-runtime-manager +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + - apiGroups: + - apps + resources: + - deployments + - deployments/scale + verbs: + - get + - list + - watch + - create + - update + - patch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: clawmanager-runtime-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: clawmanager-runtime-manager +subjects: + - kind: ServiceAccount + name: clawmanager-app + namespace: clawmanager-system +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: clawmanager-app-cluster-admin @@ -828,7 +985,7 @@ metadata: name: clawmanager-app namespace: clawmanager-system spec: - replicas: 1 + replicas: 3 selector: matchLabels: app: clawmanager-app @@ -878,10 +1035,47 @@ spec: value: "local-path" - name: CLAWMANAGER_WORKSPACE_ARCHIVE_MAX_MIB value: "500" + - name: PLATFORM_REDIS_URL + value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0" + - name: TEAM_REDIS_URL + value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0" - name: CLAWMANAGER_TEAM_REDIS_URL - value: "redis://clawmanager-team-redis.clawmanager-system.svc.cluster.local:6379/0" + value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0" - name: CLAWMANAGER_TEAM_MANAGER_BASE_URL value: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + - name: RUNTIME_NAMESPACE + value: "clawmanager-system" + - name: RUNTIME_WORKSPACE_ROOT + value: "/workspaces" + - name: RUNTIME_WORKSPACE_NFS_SERVER + value: "workspace-store.clawmanager-system.svc.cluster.local" + - name: RUNTIME_WORKSPACE_NFS_PATH + value: "/" + - name: RUNTIME_MAX_GATEWAYS_PER_POD + value: "100" + - name: RUNTIME_GATEWAY_PORT_START + value: "20000" + - name: RUNTIME_GATEWAY_PORT_END + value: "20099" + - name: RUNTIME_SCHEDULER_ENABLED + value: "true" + - name: RUNTIME_HEARTBEAT_TIMEOUT + value: "10s" + - name: RUNTIME_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: RUNTIME_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: OPENCLAW_GATEWAY_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: openclaw-gateway-token - name: SKILL_SCANNER_ENABLED value: "true" - name: SKILL_SCANNER_BASE_URL @@ -895,8 +1089,13 @@ spec: - name: OBJECT_STORAGE_LOCAL_FALLBACK value: "/data/object-storage" volumeMounts: + - name: tls-cert + mountPath: /var/run/clawreef-tls + readOnly: true - name: object-storage mountPath: /data/object-storage + - name: workspaces + mountPath: /workspaces readinessProbe: httpGet: path: /healthz @@ -914,8 +1113,203 @@ spec: periodSeconds: 20 timeoutSeconds: 5 volumes: + - name: tls-cert + secret: + secretName: clawmanager-tls + optional: true - name: object-storage emptyDir: {} + - name: workspaces + nfs: + server: workspace-store.clawmanager-system.svc.cluster.local + path: /exports/workspaces +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: openclaw-runtime + namespace: clawmanager-system + labels: + app: openclaw-runtime + clawmanager.io/runtime-type: openclaw +spec: + replicas: 1 + selector: + matchLabels: + app: openclaw-runtime + clawmanager.io/runtime-type: openclaw + template: + metadata: + labels: + app: openclaw-runtime + clawmanager.io/runtime-type: openclaw + spec: + containers: + - name: runtime + image: ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest + imagePullPolicy: Always + env: + - name: CLAWMANAGER_RUNTIME_TYPE + value: openclaw + - name: CLAWMANAGER_BACKEND_URL + value: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + - name: CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME + value: openclaw-runtime + - name: CLAWMANAGER_RUNTIME_IMAGE_REF + value: ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest + - name: RUNTIME_WORKSPACE_ROOT + value: /workspaces + - name: RUNTIME_GATEWAY_PORT_START + value: "20000" + - name: RUNTIME_GATEWAY_PORT_END + value: "20099" + - name: RUNTIME_AGENT_LISTEN_ADDR + value: "0.0.0.0:19090" + - name: RUNTIME_AGENT_PUBLIC_PORT + value: "19090" + - name: RUNTIME_GATEWAY_COMMAND + value: "/usr/local/bin/openclaw gateway run --allow-unconfigured --auth token --bind auto --force" + - name: OPENCLAW_GATEWAY_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: openclaw-gateway-token + - name: RUNTIME_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: RUNTIME_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - name: agent + containerPort: 19090 + volumeMounts: + - name: workspaces + mountPath: /workspaces + volumes: + - name: workspaces + nfs: + server: workspace-store.clawmanager-system.svc.cluster.local + path: /exports/workspaces +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-runtime + namespace: clawmanager-system + labels: + app: hermes-runtime + clawmanager.io/runtime-type: hermes +spec: + replicas: 1 + selector: + matchLabels: + app: hermes-runtime + clawmanager.io/runtime-type: hermes + template: + metadata: + labels: + app: hermes-runtime + clawmanager.io/runtime-type: hermes + spec: + containers: + - name: runtime + image: ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest + imagePullPolicy: IfNotPresent + env: + - name: CLAWMANAGER_RUNTIME_TYPE + value: hermes + - name: CLAWMANAGER_BACKEND_URL + value: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + - name: CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME + value: hermes-runtime + - name: CLAWMANAGER_RUNTIME_IMAGE_REF + value: ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest + - name: CLAWMANAGER_AGENT_PORT + value: "19090" + - name: CLAWMANAGER_GATEWAY_PORT_START + value: "20000" + - name: CLAWMANAGER_GATEWAY_PORT_END + value: "20099" + - name: HERMES_TUI_DIR + value: /usr/local/lib/hermes-agent/ui-tui + - name: CLAWMANAGER_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: CLAWMANAGER_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: RUNTIME_WORKSPACE_ROOT + value: /workspaces + - name: RUNTIME_AGENT_LISTEN_ADDR + value: "0.0.0.0:19090" + - name: RUNTIME_AGENT_PUBLIC_PORT + value: "19090" + - name: RUNTIME_GATEWAY_PORT_START + value: "20000" + - name: RUNTIME_GATEWAY_PORT_END + value: "20099" + - name: RUNTIME_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: RUNTIME_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - name: agent + containerPort: 19090 + volumeMounts: + - name: workspaces + mountPath: /workspaces + volumes: + - name: workspaces + nfs: + server: workspace-store.clawmanager-system.svc.cluster.local + path: /exports/workspaces --- apiVersion: v1 kind: Service @@ -930,7 +1324,7 @@ spec: - name: https port: 443 targetPort: https - nodePort: 30443 + nodePort: 39443 --- apiVersion: v1 kind: Service diff --git a/deployments/k8s/clawmanager.yaml b/deployments/k8s/clawmanager.yaml index 27ac892..167fc14 100644 --- a/deployments/k8s/clawmanager.yaml +++ b/deployments/k8s/clawmanager.yaml @@ -13,6 +13,9 @@ stringData: mysql-root-password: root123 mysql-password: clawreef123 jwt-secret: change-me-in-production + runtime-agent-control-token: change-me-runtime-control-token + runtime-agent-report-token: change-me-runtime-report-token + openclaw-gateway-token: change-me-openclaw-gateway-token minio-root-user: minioadmin minio-root-password: minioadmin123 minio-access-key: minioadmin @@ -639,8 +642,8 @@ data: SELECT 'openclaw', 'shell', - 'OpenClaw Shell', - 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-shell:latest', + 'OpenClaw Lite', + 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest', TRUE WHERE NOT EXISTS ( SELECT 1 @@ -675,6 +678,39 @@ data: updated_at = CURRENT_TIMESTAMP WHERE type IN ('openclaw', 'ubuntu', 'webtop', 'hermes') AND mount_path IN ('/data', '/home/user/data', '/config/.hermes'); + 031_add_hermes_lite_runtime_image.sql: | + USE clawmanager; + UPDATE system_image_settings + SET + image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest', + display_name = 'Hermes Lite', + is_enabled = TRUE + WHERE instance_type = 'hermes' + AND runtime_type = 'shell' + AND image IN ( + 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-shell:latest', + 'registry.example.com/your-custom-shell-image:latest' + ); + + INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled) + SELECT + 'hermes', + 'shell', + 'Hermes Lite', + 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest', + TRUE + WHERE NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'hermes' + AND runtime_type = 'shell' + ) + AND NOT EXISTS ( + SELECT 1 + FROM system_image_settings + WHERE instance_type = 'hermes' + AND is_enabled = FALSE + ); --- apiVersion: v1 kind: PersistentVolume @@ -912,6 +948,19 @@ spec: --- apiVersion: v1 kind: Service +metadata: + name: clawmanager-redis + namespace: clawmanager-system +spec: + selector: + app: clawmanager-team-redis + ports: + - name: redis + port: 6379 + targetPort: redis +--- +apiVersion: v1 +kind: Service metadata: name: clawmanager-team-redis namespace: clawmanager-system @@ -924,12 +973,137 @@ spec: targetPort: redis --- apiVersion: v1 +kind: PersistentVolume +metadata: + name: clawmanager-workspaces-pv +spec: + capacity: + storage: 50Gi + volumeMode: Filesystem + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: manual + hostPath: + path: /data/clawmanager/system/workspaces + type: DirectoryOrCreate +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: clawmanager-workspaces + namespace: clawmanager-system +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 50Gi + storageClassName: manual + volumeName: clawmanager-workspaces-pv +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: workspace-store + namespace: clawmanager-system +spec: + replicas: 1 + selector: + matchLabels: + app: workspace-store + template: + metadata: + labels: + app: workspace-store + spec: + containers: + - name: nfs-server + image: itsthenetwork/nfs-server-alpine:12 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + env: + - name: SHARED_DIRECTORY + value: /exports/workspaces + ports: + - name: nfs + containerPort: 2049 + volumeMounts: + - name: workspaces + mountPath: /exports/workspaces + volumes: + - name: workspaces + persistentVolumeClaim: + claimName: clawmanager-workspaces +--- +apiVersion: v1 +kind: Service +metadata: + name: workspace-store + namespace: clawmanager-system +spec: + selector: + app: workspace-store + ports: + - name: nfs + port: 2049 + targetPort: nfs +--- +apiVersion: v1 kind: ServiceAccount metadata: name: clawmanager-app namespace: clawmanager-system --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: clawmanager-runtime-manager +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + - apiGroups: + - apps + resources: + - deployments + - deployments/scale + verbs: + - get + - list + - watch + - create + - update + - patch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: clawmanager-runtime-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: clawmanager-runtime-manager +subjects: + - kind: ServiceAccount + name: clawmanager-app + namespace: clawmanager-system +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: clawmanager-app-cluster-admin @@ -1003,7 +1177,7 @@ metadata: name: clawmanager-app namespace: clawmanager-system spec: - replicas: 1 + replicas: 3 selector: matchLabels: app: clawmanager-app @@ -1077,10 +1251,47 @@ spec: value: "/data/clawreef" - name: CLAWMANAGER_WORKSPACE_ARCHIVE_MAX_MIB value: "500" + - name: PLATFORM_REDIS_URL + value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0" + - name: TEAM_REDIS_URL + value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0" - name: CLAWMANAGER_TEAM_REDIS_URL - value: "redis://clawmanager-team-redis.clawmanager-system.svc.cluster.local:6379/0" + value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0" - name: CLAWMANAGER_TEAM_MANAGER_BASE_URL value: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + - name: RUNTIME_NAMESPACE + value: "clawmanager-system" + - name: RUNTIME_WORKSPACE_ROOT + value: "/workspaces" + - name: RUNTIME_WORKSPACE_NFS_SERVER + value: "workspace-store.clawmanager-system.svc.cluster.local" + - name: RUNTIME_WORKSPACE_NFS_PATH + value: "/" + - name: RUNTIME_MAX_GATEWAYS_PER_POD + value: "100" + - name: RUNTIME_GATEWAY_PORT_START + value: "20000" + - name: RUNTIME_GATEWAY_PORT_END + value: "20099" + - name: RUNTIME_SCHEDULER_ENABLED + value: "true" + - name: RUNTIME_HEARTBEAT_TIMEOUT + value: "10s" + - name: RUNTIME_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: RUNTIME_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: OPENCLAW_GATEWAY_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: openclaw-gateway-token - name: SKILL_SCANNER_ENABLED value: "true" - name: SKILL_SCANNER_BASE_URL @@ -1095,6 +1306,8 @@ spec: - name: tls-cert mountPath: /var/run/clawreef-tls readOnly: true + - name: workspaces + mountPath: /workspaces readinessProbe: httpGet: scheme: HTTPS @@ -1114,6 +1327,197 @@ spec: secret: secretName: clawmanager-tls optional: true + - name: workspaces + nfs: + server: workspace-store.clawmanager-system.svc.cluster.local + path: /exports/workspaces +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: openclaw-runtime + namespace: clawmanager-system + labels: + app: openclaw-runtime + clawmanager.io/runtime-type: openclaw +spec: + replicas: 1 + selector: + matchLabels: + app: openclaw-runtime + clawmanager.io/runtime-type: openclaw + template: + metadata: + labels: + app: openclaw-runtime + clawmanager.io/runtime-type: openclaw + spec: + containers: + - name: runtime + image: ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest + imagePullPolicy: Always + env: + - name: CLAWMANAGER_RUNTIME_TYPE + value: openclaw + - name: CLAWMANAGER_BACKEND_URL + value: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + - name: CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME + value: openclaw-runtime + - name: CLAWMANAGER_RUNTIME_IMAGE_REF + value: ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest + - name: RUNTIME_WORKSPACE_ROOT + value: /workspaces + - name: RUNTIME_GATEWAY_PORT_START + value: "20000" + - name: RUNTIME_GATEWAY_PORT_END + value: "20099" + - name: RUNTIME_AGENT_LISTEN_ADDR + value: "0.0.0.0:19090" + - name: RUNTIME_AGENT_PUBLIC_PORT + value: "19090" + - name: RUNTIME_GATEWAY_COMMAND + value: "/usr/local/bin/openclaw gateway run --allow-unconfigured --auth token --bind auto --force" + - name: OPENCLAW_GATEWAY_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: openclaw-gateway-token + - name: RUNTIME_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: RUNTIME_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - name: agent + containerPort: 19090 + volumeMounts: + - name: workspaces + mountPath: /workspaces + volumes: + - name: workspaces + nfs: + server: workspace-store.clawmanager-system.svc.cluster.local + path: /exports/workspaces +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hermes-runtime + namespace: clawmanager-system + labels: + app: hermes-runtime + clawmanager.io/runtime-type: hermes +spec: + replicas: 1 + selector: + matchLabels: + app: hermes-runtime + clawmanager.io/runtime-type: hermes + template: + metadata: + labels: + app: hermes-runtime + clawmanager.io/runtime-type: hermes + spec: + containers: + - name: runtime + image: ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest + imagePullPolicy: IfNotPresent + env: + - name: CLAWMANAGER_RUNTIME_TYPE + value: hermes + - name: CLAWMANAGER_BACKEND_URL + value: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + - name: CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME + value: hermes-runtime + - name: CLAWMANAGER_RUNTIME_IMAGE_REF + value: ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest + - name: CLAWMANAGER_AGENT_PORT + value: "19090" + - name: CLAWMANAGER_GATEWAY_PORT_START + value: "20000" + - name: CLAWMANAGER_GATEWAY_PORT_END + value: "20099" + - name: HERMES_TUI_DIR + value: /usr/local/lib/hermes-agent/ui-tui + - name: CLAWMANAGER_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: CLAWMANAGER_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: RUNTIME_WORKSPACE_ROOT + value: /workspaces + - name: RUNTIME_AGENT_LISTEN_ADDR + value: "0.0.0.0:19090" + - name: RUNTIME_AGENT_PUBLIC_PORT + value: "19090" + - name: RUNTIME_GATEWAY_PORT_START + value: "20000" + - name: RUNTIME_GATEWAY_PORT_END + value: "20099" + - name: RUNTIME_AGENT_CONTROL_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-control-token + - name: RUNTIME_AGENT_REPORT_TOKEN + valueFrom: + secretKeyRef: + name: clawmanager-secrets + key: runtime-agent-report-token + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - name: agent + containerPort: 19090 + volumeMounts: + - name: workspaces + mountPath: /workspaces + volumes: + - name: workspaces + nfs: + server: workspace-store.clawmanager-system.svc.cluster.local + path: /exports/workspaces --- apiVersion: v1 kind: Service @@ -1128,7 +1532,7 @@ spec: - name: https port: 443 targetPort: https - nodePort: 30443 + nodePort: 39443 --- apiVersion: v1 kind: Service diff --git a/deployments/nginx/nginx.conf b/deployments/nginx/nginx.conf index c0d51fb..488d82a 100644 --- a/deployments/nginx/nginx.conf +++ b/deployments/nginx/nginx.conf @@ -11,6 +11,20 @@ http { sendfile on; tcp_nopush on; tcp_nodelay on; + gzip on; + gzip_comp_level 6; + gzip_min_length 1024; + gzip_proxied any; + gzip_vary on; + gzip_types + application/javascript + application/json + application/xml + image/svg+xml + text/css + text/javascript + text/plain + text/xml; keepalive_timeout 75; keepalive_requests 10000; server_tokens off; @@ -53,6 +67,43 @@ http { return 200 'ok'; } + location ^~ /assets/ { + try_files $uri =404; + access_log off; + expires 1y; + add_header Cache-Control "public, max-age=31536000, immutable"; + } + + location ~* ^/(?:lobster_logo|lobster_transparent|openclaw|hermes)\.png$ { + try_files $uri =404; + access_log off; + expires 30d; + add_header Cache-Control "public, max-age=2592000"; + } + + location ~* ^/vendor-icons/.+\.(?:png|jpg|jpeg|gif|webp|ico|svg)$ { + try_files $uri =404; + access_log off; + expires 30d; + add_header Cache-Control "public, max-age=2592000"; + } + + location ~ ^/s/ { + proxy_pass http://clawreef_backend; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_connect_timeout 15s; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + proxy_buffering off; + proxy_request_buffering off; + } + location ~ ^/api/v1/instances/[0-9]+/proxy/? { proxy_pass http://clawreef_backend; proxy_http_version 1.1; diff --git a/docs/agent-control-plane.md b/docs/agent-control-plane.md index 91dcc00..5c04bbe 100644 --- a/docs/agent-control-plane.md +++ b/docs/agent-control-plane.md @@ -37,7 +37,9 @@ Examples of platform-driven runtime actions include: ## Related Guides +- [ClawManager Agent V2 Development Guide](./clawmanager-agent-v2-development-guide.md) - [Runtime Agent Integration Guide](./runtime-agent-integration-guide.md) +- [Hermes Lite / Pro Agent Development Guide](./hermes-lite-pro-agent-development.md) - [Hermes Runtime Image and Agent Development Guide](./hermes-runtime-agent-development.md) - [Admin and User Guide](./admin-user-guide.md) - [Resource Management Guide](./resource-management.md) diff --git a/docs/agent-runtime-development-spec.md b/docs/agent-runtime-development-spec.md new file mode 100644 index 0000000..dcfaf86 --- /dev/null +++ b/docs/agent-runtime-development-spec.md @@ -0,0 +1,512 @@ +# Agent Runtime 开发规范 + +本文是 ClawManager 托管 runtime 的 agent 侧开发规范,适用于 OpenClaw、Hermes,以及后续新增的任意 runtime。开发新 runtime 时,应先满足本文的通用要求,再补充 runtime 自己的启动命令、配置文件和健康检查细节。 + +相关协议和背景请同时参考: + +- `docs/clawmanager-agent-v2-development-guide.md` +- `docs/clawmanager-agent-v2-contract.md` +- `docs/runtime-agent-integration-guide.md` +- `docs/hermes-lite-pro-agent-development.md` +- `docs/hermes-runtime-agent-development.md` + +## 1. 范围和目标 + +一个托管 runtime 镜像不能只启动业务进程。它必须内置一个常驻 runtime agent,由 agent 在 Pod 内负责: + +- Runtime Pod 注册、heartbeat、metrics、gateway 状态上报。 +- 接收 ClawManager 下发的 create、delete、drain、restart 等控制指令。 +- 管理本 Pod 内多个 gateway 子进程。 +- 管理端口池、进程、workspace、UID/GID、资源限制和健康检查。 +- 把 ClawManager 注入的 AI Gateway、实例 token、代理信任和 runtime 配置写入每个实例自己的 workspace。 + +一句话原则:agent 是 Pod 内 gateway 管理器,不是同步启动脚本。 + +## 2. 架构约束 + +每个 Runtime Pod 内只能有一个 runtime agent 控制进程。一个用户实例对应一个 gateway 子进程和一个 workspace。 + +```mermaid +flowchart LR + CM["ClawManager Backend"] + POD["Runtime Pod"] + AG["runtime-agent"] + GW1["gateway: instance 101"] + GW2["gateway: instance 102"] + WS["/workspaces shared volume"] + + CM -- "POST /v1/gateways" --> AG + CM -- "DELETE /v1/gateways/{id}" --> AG + CM -- "POST /v1/drain" --> AG + AG -- "register / heartbeat / metrics / gateways report" --> CM + AG --> GW1 + AG --> GW2 + GW1 --> WS + GW2 --> WS +``` + +浏览器只访问 ClawManager 的 HTTPS 入口。ClawManager 再反代到 Pod 内部 gateway。runtime agent 不应把每个 gateway 的自签 HTTPS 地址直接暴露给前端。 + +## 3. 必须遵守的原则 + +| 原则 | 要求 | +| --- | --- | +| 异步创建 | `POST /v1/gateways` 必须快速返回,通常 1 到 3 秒内返回 `status=starting`,不能阻塞等待业务 gateway 完全启动。 | +| 幂等创建 | 以 `instance_id + generation` 为主要幂等键,重复请求必须返回同一个 gateway 元数据,不能重复启动进程。 | +| 状态以真实进程为准 | `running` 只能在进程存在且健康检查通过时上报,不能只读旧缓存。 | +| 端口统一管理 | 端口由 agent 本地分配器加锁分配,启动前检查监听占用,停止后释放。 | +| 配置按 workspace 隔离 | 每个实例的 runtime 配置写入自己的 workspace,不能写全局配置。 | +| 不覆盖用户配置 | 修改 runtime 配置必须做 JSON merge,保留已有字段,数组 append 后去重。 | +| 不写死环境地址 | 不写死外部 IP、域名、CIDR,必须从 env 或 ClawManager 请求中读取。 | +| 内部服务优先 | runtime 内部访问 ClawManager 使用 Kubernetes Service DNS,不使用浏览器访问的外部地址。 | +| 时间必须可信 | token、设备签名、JWT 和 WebSocket 校验依赖时间,agent 必须检测时钟偏移,不能在明显偏移时报告健康。 | +| 安全默认开启 | 控制接口校验 token,路径校验不能越权,命令执行不能拼 shell 字符串。 | + +## 4. Runtime Pod 环境变量 + +agent 至少支持以下环境变量: + +| 变量 | 说明 | +| --- | --- | +| `CLAWMANAGER_RUNTIME_TYPE` | runtime 类型,例如 `openclaw`、`hermes`。 | +| `RUNTIME_WORKSPACE_ROOT` | workspace 根目录,默认 `/workspaces`。 | +| `RUNTIME_GATEWAY_PORT_START` | gateway 端口起始值,默认 `20000`。 | +| `RUNTIME_GATEWAY_PORT_END` | gateway 端口结束值,默认 `20099`。 | +| `RUNTIME_AGENT_CONTROL_TOKEN` | ClawManager 调 agent 控制接口使用。 | +| `RUNTIME_AGENT_REPORT_TOKEN` | agent 上报 ClawManager 使用。 | +| `CLAWMANAGER_BACKEND_URL` | ClawManager backend 内部地址,推荐 `http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001`。 | +| `RUNTIME_AGENT_LISTEN_ADDR` | agent 控制接口监听地址,默认 `0.0.0.0:19090`。 | +| `RUNTIME_AGENT_PUBLIC_PORT` | 注册给 ClawManager 的 agent 端口,默认 `19090`。 | +| `CLAWMANAGER_RUNTIME_IMAGE_REF` | 当前 runtime 镜像标识。 | +| `POD_NAME` / `POD_NAMESPACE` / `POD_IP` / `NODE_NAME` | 通过 Downward API 注入的 Pod 身份。 | +| `CLAWMANAGER_TRUSTED_PROXY_CIDRS` | ClawManager app 或 gateway Pod 的来源网段,供 runtime 的 trusted proxy 配置使用。 | +| `CLAWMANAGER_CONTROL_UI_ORIGIN` | ClawManager 内部 Control UI origin,默认可由 `CLAWMANAGER_BACKEND_URL` 推导。 | + +要求: + +- `CLAWMANAGER_BACKEND_URL` 必须是内部服务地址,不能使用 `https://172.16.1.12:39443` 这类浏览器外部入口。 +- 只有对浏览器生成页面 URL 时才使用外部入口。runtime gateway 的信任来源和 LLM base URL 都应使用内部服务 DNS。 +- agent 启动时要校验必填 env,缺失时上报 `unhealthy`,并在 `/v1/health` 返回 503。 + +## 5. Gateway 创建协议 + +ClawManager 调用: + +```http +POST /v1/gateways +X-ClawManager-Control-Token: ${RUNTIME_AGENT_CONTROL_TOKEN} +Content-Type: application/json +``` + +请求字段以 `docs/clawmanager-agent-v2-contract.md` 为准,当前应支持: + +```json +{ + "instance_id": 123, + "user_id": 45, + "agent_type": "hermes", + "workspace_path": "/workspaces/hermes/user-45/instance-123", + "port_range": { + "start": 20000, + "end": 20099 + }, + "uid": 200123, + "gid": 200123, + "cpu_cores": 2, + "memory_mb": 4096, + "disk_quota_mb": 20480, + "generation": 7, + "environment": { + "CLAWMANAGER_LLM_BASE_URL": "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001/api/v1/gateway/llm", + "CLAWMANAGER_LLM_API_KEY": "instance-token", + "CLAWMANAGER_LLM_MODEL": "[\"auto\"]", + "CLAWMANAGER_LLM_PROVIDER": "openai-compatible", + "CLAWMANAGER_INSTANCE_TOKEN": "instance-token", + "OPENAI_API_KEY": "instance-token" + } +} +``` + +成功响应必须快速返回: + +```json +{ + "gateway_id": "gw-123-7", + "instance_id": 123, + "port": 20017, + "pid": null, + "status": "starting", + "workspace_path": "/workspaces/hermes/user-45/instance-123" +} +``` + +创建流程: + +1. 校验控制 token、`agent_type`、`instance_id`、`user_id`、`generation`、`workspace_path`、端口范围。 +2. 检查当前 Pod 是否 draining。draining 时返回 409 或 503。 +3. 按 `instance_id + generation` 查询本地状态。已存在 `starting` 或 `running` 时直接返回已有元数据。 +4. 若收到更高 generation,停止并释放同实例旧 generation gateway。 +5. 加锁分配端口,先标记 reserved。 +6. 创建 workspace 和 home 目录,设置 owner 为请求中的 `uid/gid`。 +7. 写入或合并 runtime 配置。 +8. 记录 gateway 元数据为 `starting`。 +9. 后台启动进程,立即返回。 +10. 后台健康检查通过后,上报 `/api/v1/runtime-agent/gateways/report`,状态为 `running`。 +11. 启动失败或健康检查失败时,上报 `error`,释放端口,并保留错误信息。 + +禁止事项: + +- 禁止在 HTTP handler 里直接运行前台命令并等待它退出。 +- 禁止同一个 create 请求触发多个 gateway 进程。 +- 禁止把请求里的任意字符串拼接成 shell 命令。 +- 禁止创建不在端口范围内的监听端口。 + +## 6. 端口和资源管理 + +agent 必须实现并发安全的端口分配器: + +```text +reserve(instance_id, generation, count) -> port_set +commit(instance_id, generation, port_set) +release(instance_id, generation) +list_used() +``` + +要求: + +- 分配时持有互斥锁。 +- 在锁内检查 agent 内存状态和系统监听端口。 +- runtime 如果需要主端口、control 端口、browser 端口或 sidecar 端口,必须一次分配一组端口。 +- 启动失败、健康检查失败、DELETE 成功后必须释放端口。 +- `used_slots` 来自真实 `starting` 和健康 `running` gateway 数量,不能来自旧缓存。 + +资源隔离要求: + +- 每个 gateway 使用独立进程组。 +- 业务进程最终以请求里的 `uid/gid` 运行。 +- CPU、Memory 使用 cgroup 限制。 +- Disk 使用 filesystem quota 或周期扫描加错误上报。 +- 删除 gateway 不删除 workspace。 + +## 7. Workspace 和配置写入 + +workspace 固定格式: + +```text +/workspaces/{runtime}/user-{user_id}/instance-{instance_id} +``` + +agent 必须: + +- 对 workspace root 和请求路径执行 realpath。 +- 拒绝 `..`、符号链接逃逸、跨用户或跨实例路径。 +- 创建 `${workspace}/home`,并设置 `HOME=${workspace}/home`。 +- 密钥、token、provider 配置文件权限建议 `0600`。 +- owner 设置为该 gateway 的 `uid/gid`。 + +runtime 配置必须写在 workspace 内。例如: + +| Runtime | 推荐配置路径 | +| --- | --- | +| OpenClaw | `{workspace}/home/.openclaw/openclaw.json` | +| Hermes | `{workspace}/home/.hermes/hermes.json` 或 Hermes 原生配置路径 | + +JSON merge 规则: + +- 对象字段递归 merge。 +- 已存在字段默认保留。 +- 平台必须控制的字段可以覆盖,但要在代码中显式列出。 +- 数组字段 append 后去重,例如 `allowedOrigins`。 +- 不删除用户已有的未知字段。 +- 写入前先生成临时文件,再原子 rename。 + +## 8. AI Gateway 和模型凭证注入 + +ClawManager 会在 `POST /v1/gateways` 的 `environment` 字段中下发实例级 LLM 环境变量。agent 必须把这些变量传给 gateway 子进程,并按 runtime 需要写入配置文件。 + +允许转发给子进程的变量应使用白名单: + +```text +CLAWMANAGER_LLM_BASE_URL +CLAWMANAGER_LLM_API_KEY +CLAWMANAGER_LLM_MODEL +CLAWMANAGER_LLM_PROVIDER +CLAWMANAGER_INSTANCE_TOKEN +OPENAI_BASE_URL +OPENAI_API_BASE +OPENAI_API_KEY +OPENAI_MODEL +``` + +要求: + +- `CLAWMANAGER_LLM_BASE_URL` 使用 ClawManager 内部服务 DNS。 +- `CLAWMANAGER_LLM_API_KEY` 和 `OPENAI_API_KEY` 使用当前实例 token。 +- 不能要求用户在 runtime 页面手工填写 OpenAI key。 +- 不能把 API key、实例 token、bootstrap token 打到日志。 +- 如果 `environment` 里缺少 API key,agent 不应报告 gateway `running`,应上报 `error`,错误消息明确说明缺少实例 LLM token。 + +Hermes 落地时应把上述 env 转换成 Hermes 的 provider 配置,推荐语义: + +```json +{ + "models": { + "providers": { + "auto": { + "type": "openai-compatible", + "baseUrl": "${CLAWMANAGER_LLM_BASE_URL}", + "apiKey": "${CLAWMANAGER_LLM_API_KEY}" + } + }, + "primary": "auto/auto" + } +} +``` + +如果 Hermes 使用不同配置 schema,也必须满足同样的安全和可用性要求:通过 ClawManager AI Gateway 访问模型,不在镜像内写死厂商 key。 + +## 9. 代理信任和浏览器访问 + +runtime gateway 在 Pod 内部推荐使用 HTTP。浏览器访问链路应为: + +```text +Browser HTTPS -> ClawManager HTTPS -> ClawManager backend proxy -> Runtime Pod HTTP gateway +``` + +agent 写入 runtime 配置时必须配置: + +- Control UI 的 `basePath` 为 `/api/v1/instances/{instance_id}/proxy`。 +- 允许的 origin 使用 ClawManager 内部 origin,例如 `http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001`。 +- trusted proxies 使用 `CLAWMANAGER_TRUSTED_PROXY_CIDRS` 或等价 env,不写死 `100.68.x.x`。 +- 不使用外部地址 `https://172.16.1.12:39443` 作为 runtime 内部信任来源。 + +OpenClaw 参考配置: + +```json +{ + "gateway": { + "auth": { + "mode": "trusted-proxy" + }, + "controlUi": { + "allowedOrigins": [ + "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001" + ], + "basePath": "/api/v1/instances/123/proxy", + "dangerouslyDisableDeviceAuth": true + }, + "trustedProxies": [ + "100.68.0.0/16" + ] + } +} +``` + +说明: + +- `dangerouslyDisableDeviceAuth` 只能在 ClawManager trusted proxy 模式下使用。 +- 如果 runtime 不支持 trusted proxy 模式,必须明确实现一种等价机制,保证浏览器通过 ClawManager 代理即可自动进入,不需要用户手工粘贴 gateway token。 +- 如果出现 `origin not allowed`,优先检查 `allowedOrigins` 是否使用内部 origin,以及 WebSocket 代理是否正确传递 Origin。 +- 如果出现 `Proxy headers detected from untrusted address`,优先检查 trusted proxies CIDR。 + +## 10. 时间同步规范 + +runtime 侧不能忽略时间问题。以下问题通常与时钟偏移有关: + +- `device signature expired` +- token 未过期但被拒绝。 +- JWT、签名 URL、WebSocket 鉴权异常。 +- ClawManager 显示状态和 gateway 实际状态不一致。 + +agent 必须实现: + +- 注册和 heartbeat 响应如果带 `server_time`,记录本地时间与服务端时间偏移。 +- 若偏移超过 30 秒,在日志和 health 中上报 `clock_skew_warning`。 +- 若偏移超过 120 秒,不应报告 `ready` 或 gateway `running`,应上报 `unhealthy` 或 `error`。 +- 生成或校验任何带时间的签名时使用 UTC。 +- 不在容器里自行修改系统时间。时间同步由 Kubernetes 节点的 NTP、chrony 或宿主机配置负责。 + +运维要求: + +- 所有 Kubernetes 节点必须启用 NTP 或 chrony。 +- 发现签名过期类错误时,先比较浏览器机器、ClawManager Pod、runtime Pod、Kubernetes Node 的 UTC 时间。 +- agent 健康检查输出应包含当前本地时间、最近服务端时间和估算偏移,便于排查。 + +## 11. 状态上报和恢复语义 + +Runtime Pod 注册 payload 必须包含容量: + +```json +{ + "runtime_type": "hermes", + "capacity": 100, + "used_slots": 12, + "available_slots": 88, + "state": "ready", + "draining": false +} +``` + +Gateway report 每个 gateway 建议包含: + +```json +{ + "instance_id": 66, + "gateway_id": "gw-66-7", + "gateway_port": 20000, + "gateway_pid": 1234, + "state": "running", + "generation": 7, + "workspace_path": "/workspaces/hermes/user-1/instance-66", + "started_at": "2026-06-03T06:00:00Z", + "updated_at": "2026-06-03T06:00:10Z", + "error_message": "" +} +``` + +状态规则: + +- `starting`:进程准备中或健康检查尚未通过。 +- `running`:进程存在、端口监听、HTTP 健康检查通过、必要的代理和 LLM 检查通过。 +- `error`:启动失败、配置失败、健康检查失败、凭证缺失或端口冲突。 +- `stopped`:进程已停止,端口已释放。 +- `unhealthy`:进程存在但健康检查失败。 + +Pod 重启或升级时: + +- agent 启动后先初始化 workspace root、端口池、控制服务和报告客户端。 +- 控制服务可用后注册 Runtime Pod。 +- 只有初始化完成后才能上报 `ready`。 +- 不要把旧 Pod 的 gateway 当成运行中,除非进程确实存在且健康。 +- 收到 SIGTERM 时进入 draining,拒绝新 create,尽量上报最后状态。 +- 支持 ClawManager 下发 drain、stop、restart gateway 指令。 + +## 12. 健康检查 + +`/v1/health` 只有在 agent 能接受控制指令时才返回 2xx。以下情况必须返回 503: + +- 必填环境变量缺失。 +- workspace root 不可访问。 +- 端口池不可用。 +- report token 或 control token 缺失。 +- 时钟偏移超过健康阈值。 +- agent 正在关闭且不再接受请求。 + +gateway 从 `starting` 转为 `running` 前至少校验: + +- 端口已监听。 +- HTTP `/` 或 runtime 约定健康端点可访问。 +- 代理 base path 配置已写入。 +- 使用 ClawManager 内部 Origin 的 Control UI 或 WebSocket 握手不再返回 `origin not allowed`。 +- runtime 如果需要 LLM,使用实例 token 调用 ClawManager AI Gateway 可以返回模型列表或完成一次轻量探测。 + +健康检查失败时不要上报 `running`。应上报 `error`,并带短错误文本。 + +## 13. 安全规范 + +agent 控制接口必须校验: + +- `X-ClawManager-Control-Token`。 +- `instance_id`、`user_id`、`generation` 类型和值范围。 +- `agent_type` 必须等于当前 Pod runtime type。 +- `workspace_path` 不越权。 +- 端口只能来自允许范围。 +- `environment` 只允许白名单 key。 + +进程启动必须使用 argv 数组,不使用 shell 拼接: + +```text +execve(binary, ["hermes", "gateway", "run", "--port", "20017"], env) +``` + +禁止: + +- 把请求参数拼成 `sh -c`。 +- 日志打印 token、API key、channel secret。 +- 允许用户覆盖 agent 自己的控制 token 和上报 token。 +- 删除 gateway 时顺手删除 workspace。 +- 为了调通临时写死 IP、CIDR 或 `allowedOrigins=["*"]`,除非明确是本地开发环境。 + +## 14. Hermes Runtime 开发落地清单 + +开发 Hermes runtime 时,至少完成以下事项: + +| 项目 | 要求 | +| --- | --- | +| Runtime 类型 | `CLAWMANAGER_RUNTIME_TYPE=hermes`,注册和上报 payload 中都使用 `hermes`。 | +| Agent 控制服务 | 监听 `0.0.0.0:19090`,实现 `/v1/health`、`/v1/gateways`、`DELETE /v1/gateways/{id}`、`/v1/drain`。 | +| Gateway 启动 | 后台启动 Hermes gateway,使用实例 workspace、端口、UID/GID、HOME。 | +| 配置文件 | 写入 `{workspace}/home/.hermes/hermes.json` 或 Hermes 原生配置,必须 JSON merge。 | +| AI Gateway | 将 `CLAWMANAGER_LLM_*` 和 `OPENAI_*` 注入 Hermes provider 配置。 | +| 代理路径 | Hermes UI 或 WebSocket 必须支持 `/api/v1/instances/{instance_id}/proxy` base path。 | +| Trusted proxy | 信任 ClawManager 内部 proxy 来源,不信任任意外部来源。 | +| 自动登录 | 浏览器通过 ClawManager proxy 进入时不要求手工粘贴 token 或重复确认自签证书。 | +| 端口组 | 如果 Hermes 除主端口外还有 control/browser/sidecar 端口,按端口组分配。 | +| Skills | skill inventory 和状态上报按 `instance_id`、`workspace_path` 分组。 | +| 时间 | 实现 clock skew 检测;偏移过大时不报告 ready/running。 | +| 验收 | 100 个实例并发创建无端口冲突,Pod drain 和重启后状态正确。 | + +Hermes 启动命令由 Hermes 项目确定,但 agent 必须保证: + +- 命令参数固定且可审计。 +- 端口和 workspace 只能来自 agent 校验后的值。 +- LLM、proxy、auth 配置在进程启动前已经写入。 +- `POST /v1/gateways` 返回时进程可以仍在启动,但 metadata 必须已经持久化。 + +## 15. 测试和验收 + +单元测试至少覆盖: + +- create gateway 幂等。 +- 高 generation 替换低 generation。 +- 端口分配并发无冲突。 +- 端口被系统占用时跳过。 +- workspace path 越权被拒绝。 +- JSON merge 保留用户字段,数组去重。 +- LLM env 白名单转发。 +- 缺少 LLM token 时 gateway 不报告 running。 +- trusted proxy 和 allowed origin 配置生成。 +- clock skew 阈值判断。 + +集成测试至少覆盖: + +- Runtime Pod 注册为 `ready`,capacity、used_slots、available_slots 正确。 +- 创建 100 个 gateway,端口唯一,状态最终为 running。 +- 重复创建同一个 `instance_id + generation` 不重复启动进程。 +- ClawManager 反代 WebSocket 不出现 `origin not allowed`。 +- 通过 ClawManager proxy 进入 runtime 不要求手工粘贴 gateway token。 +- runtime 内模型调用经过 ClawManager AI Gateway,不出现 `No API key found`。 +- Pod drain 后拒绝新 create,已有 gateway 持续上报,DELETE 后释放端口。 +- Pod 重启后不误报旧 gateway running。 + +上线前 Kubernetes 验证建议: + +```bash +kubectl -n clawmanager-system get pods -l app.kubernetes.io/component=runtime-agent -o wide +kubectl -n clawmanager-system logs deploy/hermes-runtime --tail=200 +kubectl -n clawmanager-system exec deploy/hermes-runtime -- curl -fsS http://127.0.0.1:19090/v1/health +kubectl -n clawmanager-system exec deploy/hermes-runtime -- date -u +kubectl -n clawmanager-system exec deploy/clawmanager-app -- date -u +``` + +如果出现认证或签名问题,排查顺序: + +1. 时间偏移。 +2. 实例 token 是否下发到 `environment`。 +3. runtime 配置文件是否包含 provider 和 API key。 +4. ClawManager 内部 LLM base URL 是否可达。 +5. allowed origins 和 trusted proxies 是否使用内部服务地址和 env CIDR。 +6. gateway 是否真正健康,而不是缓存状态误报。 + +## 16. Do / Don't + +| Do | Don't | +| --- | --- | +| 用内部 Kubernetes Service DNS 配置 runtime 到 ClawManager 的访问。 | 写死 `172.16.1.12`、NodePort、外部 HTTPS 入口。 | +| `POST /v1/gateways` 快速返回 `starting`,后台拉起进程。 | 在 HTTP 请求里阻塞等待 `gateway run` 前台进程。 | +| 按 `instance_id + generation` 幂等。 | 重试时重复启动多个 gateway。 | +| 上报状态前检查真实进程和健康。 | 只读旧状态文件就上报 `running`。 | +| JSON merge runtime 配置。 | 覆盖用户的整个配置文件。 | +| 白名单转发 env。 | 把用户请求里的所有 env 原样传给进程。 | +| 使用 trusted proxy 让浏览器通过 ClawManager 自动进入。 | 要求用户手工粘贴 gateway token。 | +| 检测并上报 clock skew。 | 忽略时间,导致 `device signature expired` 反复出现。 | diff --git a/docs/clawmanager-agent-v2-contract.md b/docs/clawmanager-agent-v2-contract.md new file mode 100644 index 0000000..8879cd2 --- /dev/null +++ b/docs/clawmanager-agent-v2-contract.md @@ -0,0 +1,178 @@ +# clawmanager-agent V2 Contract + +## Purpose + +The agent runs inside each shared OpenClaw or Hermes runtime Pod. It manages gateway subprocesses, ports, cgroups, Linux users, workspace quota, health checks, skill/status reporting, and metrics. + +## Agent-To-Backend Reports + +All report requests use header `X-ClawManager-Agent-Token`. + +### POST /api/v1/runtime-agent/register + +Request body: + +```json +{ + "runtime_type": "openclaw", + "namespace": "clawmanager-system", + "pod_name": "openclaw-runtime-6f77f8b8c7-abcde", + "pod_uid": "pod-uid", + "pod_ip": "10.42.0.31", + "node_name": "node-a", + "deployment_name": "openclaw-runtime", + "image_ref": "ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest", + "agent_endpoint": "http://10.42.0.31:19090", + "capacity": 100 +} +``` + +### POST /api/v1/runtime-agent/heartbeat + +Request body: + +```json +{ + "pod_name": "openclaw-runtime-6f77f8b8c7-abcde", + "runtime_type": "openclaw", + "state": "ready", + "used_slots": 37, + "draining": false +} +``` + +### POST /api/v1/runtime-agent/metrics/report + +Request body: + +```json +{ + "pod_name": "openclaw-runtime-6f77f8b8c7-abcde", + "cpu_millis_used": 13600, + "memory_bytes_used": 42949672960, + "disk_bytes_used": 214748364800, + "network_rx_bytes": 9223372, + "network_tx_bytes": 19223372, + "gateways": [ + { + "instance_id": 123, + "gateway_id": "gw-123-7", + "port": 20017, + "state": "running", + "last_health_at": "2026-06-01T10:00:00Z" + } + ] +} +``` + +## Backend-To-Agent Commands + +All command requests use header `X-ClawManager-Control-Token`. + +ClawManager treats redirects from agent command endpoints as errors. Agents should return direct 2xx, 4xx, or 5xx responses instead of 3xx redirects. + +### GET /v1/health + +The agent reports whether it can accept control-plane commands. + +Success status: HTTP 200. + +Success response body: + +```json +{ + "status": "ready" +} +``` + +ClawManager currently treats any HTTP 2xx response as success and does not require a response body for this endpoint. Any non-2xx status is treated as an error. + +### POST /v1/gateways + +The agent creates a gateway subprocess and returns the bound port. If no port is free in the requested range, return HTTP 409. + +Request body: + +```json +{ + "instance_id": 123, + "user_id": 45, + "agent_type": "openclaw", + "workspace_path": "/workspaces/openclaw/user-45/instance-123", + "port_range": { + "start": 20000, + "end": 20099 + }, + "uid": 200123, + "gid": 200123, + "cpu_cores": 2, + "memory_mb": 4096, + "disk_quota_mb": 20480, + "generation": 7 +} +``` + +Success status: HTTP 200 or HTTP 201. + +Success response body: + +```json +{ + "gateway_id": "gw-123-7", + "port": 20017, + "pid": 8842, + "status": "running" +} +``` + +### DELETE /v1/gateways/{gateway_id} + +The agent stops the gateway and releases cgroup, process, and port resources. + +`gateway_id` is URL path-escaped by ClawManager. Agents must decode the path segment before lookup. + +Success status: HTTP 200, HTTP 202, or HTTP 204. ClawManager does not require a response body. + +### POST /v1/drain + +The agent marks the Pod as draining and rejects new gateway creation. Existing gateways continue until ClawManager migrates them. + +Request body: + +```json +{ + "draining": true +} +``` + +Success status: HTTP 200 or HTTP 202. ClawManager does not require a response body. + +## Backend-To-Agent Error Responses + +For now, agents should return a plain text response body for command errors. + +When no port is free for `POST /v1/gateways`, return: + +```http +HTTP/1.1 409 Conflict +Content-Type: text/plain + +no free port +``` + +ClawManager maps this to `runtime agent conflict: no free port`. + +For general failures, return an appropriate non-2xx status with a short plain text body: + +```http +HTTP/1.1 500 Internal Server Error +Content-Type: text/plain + +failed to start gateway process +``` + +ClawManager includes both the status code and response body in the returned error. + +## Isolation Requirements + +Every gateway runs as UID/GID `200000 + instance_id`. The agent rejects workspace paths outside `/workspaces/{runtime}/user-{user_id}/instance-{instance_id}` after resolving symlinks. CPU and memory limits are enforced with cgroups. Workspace quota is enforced before writes. diff --git a/docs/clawmanager-agent-v2-development-guide.md b/docs/clawmanager-agent-v2-development-guide.md new file mode 100644 index 0000000..ec62088 --- /dev/null +++ b/docs/clawmanager-agent-v2-development-guide.md @@ -0,0 +1,888 @@ +# ClawManager Agent V2 开发规范 + +本文整理当前 ClawManager 代码中的最新 agent 约定,面向 OpenClaw、Hermes 以及后续新增的托管 runtime。开发或改造 runtime 镜像时,应优先遵守本文;字段级契约以 `docs/clawmanager-agent-v2-contract.md` 和后端代码为准。 + +## 1. 架构定位 + +最新架构中有两类 agent 通信面: + +1. **Runtime Pod Agent,也就是本文主线的 V2 agent** + 运行在共享 runtime Pod 内,负责本 Pod 内多个 gateway 子进程的创建、删除、端口、workspace、资源隔离、健康检查和状态上报。ClawManager 通过它把一个实例绑定到 `pod_ip + gateway_port`。 + +2. **Instance Agent Control Plane,旧的 `/api/v1/agent/*` 协议** + 面向单个实例的状态、命令、skill inventory、skill package 上传和配置 revision 下载。当前后端仍保留这套协议,runtime 可以继续复用它完成实例级状态与 skill 管理;但 gateway 生命周期管理必须走 Runtime Pod Agent。 + +一句话原则:**Runtime Pod Agent 是 Pod 内 gateway 管理器,不是一个阻塞式启动脚本。** + +## 2. 支持范围 + +当前 V2 托管 runtime 类型: + +| 类型 | 说明 | +| --- | --- | +| `openclaw` | OpenClaw runtime | +| `hermes` | Hermes runtime | + +当前平台默认值: + +| 项 | 默认值 | +| --- | --- | +| workspace root | `/workspaces` | +| workspace path | `/workspaces/{runtime}/user-{user_id}/instance-{instance_id}` | +| gateway 端口范围 | `20000-20099` | +| 单 Runtime Pod 容量 | `100` | +| 实例 UID/GID | `200000 + instance_id` | +| agent 控制端口 | `19090` | + +## 3. Runtime Pod Agent 职责 + +Runtime Pod Agent 必须实现: + +- 启动本地控制服务:`GET /v1/health`、`POST /v1/gateways`、`DELETE /v1/gateways/{gateway_id}`、`POST /v1/drain`。 +- 向 ClawManager 上报 runtime pod 注册、heartbeat、metrics、gateway 状态和 skill 状态。 +- 管理本 Pod 内 gateway 子进程生命周期。 +- 管理端口池,避免同一 Pod 内端口冲突。 +- 创建并校验 workspace,设置正确 UID/GID。 +- 将 ClawManager 下发的 LLM、代理、实例 token、runtime 配置写入实例自己的 workspace。 +- 实现 drain,拒绝新 gateway,保持已有 gateway 运行,等待 ClawManager 迁移和删除。 +- 避免任何敏感 token、API key、Authorization header 出现在日志中。 + +不应该由 Runtime Pod Agent 实现: + +- 直接对浏览器暴露 gateway 自签地址。 +- 管理 ClawManager 用户权限。 +- 删除用户 workspace。 +- 把外部 NodePort、外部 HTTPS 入口或固定内网 IP 写死进 runtime 配置。 + +## 4. 运行环境变量 + +当前 Runtime Deployment 构建器注入以下变量,agent 实现应优先支持: + +| 变量 | 说明 | +| --- | --- | +| `CLAWMANAGER_RUNTIME_TYPE` | `openclaw` 或 `hermes` | +| `CLAWMANAGER_AGENT_PORT` | agent 控制端口,当前为 `19090` | +| `CLAWMANAGER_GATEWAY_PORT_START` | gateway 起始端口 | +| `CLAWMANAGER_GATEWAY_PORT_END` | gateway 结束端口 | +| `CLAWMANAGER_AGENT_CONTROL_TOKEN` | ClawManager 调 agent 控制接口使用 | +| `CLAWMANAGER_AGENT_REPORT_TOKEN` | agent 上报 ClawManager 使用 | + +为了兼容早期文档和旧镜像,agent 建议同时识别以下别名: + +| 新变量 | 兼容别名 | +| --- | --- | +| `CLAWMANAGER_AGENT_CONTROL_TOKEN` | `RUNTIME_AGENT_CONTROL_TOKEN` | +| `CLAWMANAGER_AGENT_REPORT_TOKEN` | `RUNTIME_AGENT_REPORT_TOKEN` | +| `CLAWMANAGER_GATEWAY_PORT_START` | `RUNTIME_GATEWAY_PORT_START` | +| `CLAWMANAGER_GATEWAY_PORT_END` | `RUNTIME_GATEWAY_PORT_END` | +| `CLAWMANAGER_AGENT_PORT` | `RUNTIME_AGENT_PUBLIC_PORT` | + +建议支持的附加变量: + +| 变量 | 默认值/要求 | +| --- | --- | +| `RUNTIME_WORKSPACE_ROOT` | 默认 `/workspaces` | +| `RUNTIME_AGENT_LISTEN_ADDR` | 默认 `0.0.0.0:19090` | +| `CLAWMANAGER_BACKEND_URL` | ClawManager backend 内部地址,推荐 Kubernetes Service DNS | +| `CLAWMANAGER_RUNTIME_IMAGE_REF` | 当前镜像标识,用于注册上报 | +| `POD_NAME` / `POD_NAMESPACE` / `POD_IP` / `NODE_NAME` | 建议通过 Downward API 注入 | +| `CLAWMANAGER_TRUSTED_PROXY_CIDRS` | ClawManager app/gateway 的可信代理网段 | + +`CLAWMANAGER_BACKEND_URL` 必须使用集群内部地址,例如: + +```text +http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001 +``` + +不要在 runtime 内部配置中写入浏览器外部入口,例如 `https://172.16.1.12:39443`。 + +## 5. Agent 到 ClawManager 的上报接口 + +所有 Runtime Pod Agent 上报请求必须带: + +```http +X-ClawManager-Agent-Token: ${CLAWMANAGER_AGENT_REPORT_TOKEN} +Content-Type: application/json +``` + +后端统一返回: + +```json +{ + "success": true, + "message": "...", + "data": {} +} +``` + +### 5.1 注册 Runtime Pod + +```http +POST {CLAWMANAGER_BACKEND_URL}/api/v1/runtime-agent/register +``` + +必填字段: + +- `runtime_type` +- `namespace` +- `pod_name` +- `deployment_name` +- `image_ref` + +推荐完整 payload: + +```json +{ + "runtime_type": "openclaw", + "namespace": "clawmanager-system", + "pod_name": "openclaw-runtime-6f77f8b8c7-abcde", + "pod_uid": "pod-uid", + "pod_ip": "10.42.0.31", + "node_name": "node-a", + "deployment_name": "openclaw-runtime", + "image_ref": "ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest", + "agent_endpoint": "http://10.42.0.31:19090", + "state": "ready", + "capacity": 100, + "used_slots": 0, + "draining": false, + "metrics": { + "agent_version": "0.1.0" + }, + "reported_at": "2026-06-08T09:00:00Z" +} +``` + +要求: + +- `agent_endpoint` 必须是 ClawManager Pod 可以访问的地址,通常是 `http://{POD_IP}:19090`,不能是 `127.0.0.1`。 +- `capacity <= 0` 时后端会按 `100` 处理,但 agent 应主动上报真实容量。 +- `state` 为空时后端按 `ready` 处理;agent 不应在初始化未完成时上报 ready。 +- `metrics` 必须是合法 JSON。 + +### 5.2 Heartbeat + +```http +POST {CLAWMANAGER_BACKEND_URL}/api/v1/runtime-agent/heartbeat +``` + +payload: + +```json +{ + "pod_id": 17, + "namespace": "clawmanager-system", + "pod_name": "openclaw-runtime-6f77f8b8c7-abcde", + "state": "ready", + "used_slots": 37, + "draining": false, + "reported_at": "2026-06-08T09:00:02Z" +} +``` + +要求: + +- 优先使用注册返回的 `data.pod.id` 作为 `pod_id`。 +- 没有 `pod_id` 时必须带 `namespace + pod_name`。 +- `state` 必填。 +- `used_slots` 必须来自当前真实 `starting/running` gateway 数量,不要使用过期缓存。 +- 建议 heartbeat 间隔 2 秒;如果环境压力较大,可放宽,但必须小于 ClawManager 的 heartbeat 超时窗口。 + +### 5.3 Metrics + +```http +POST {CLAWMANAGER_BACKEND_URL}/api/v1/runtime-agent/metrics/report +``` + +payload: + +```json +{ + "pod_id": 17, + "cpu_millis_used": 13600, + "memory_bytes_used": 42949672960, + "disk_bytes_used": 214748364800, + "network_rx_bytes": 9223372, + "network_tx_bytes": 19223372, + "metrics": { + "gateway_count": 37, + "load_1m": 2.4 + }, + "reported_at": "2026-06-08T09:00:05Z" +} +``` + +要求: + +- CPU 单位是 millicore。 +- memory/disk/network 单位是 bytes。 +- network 字段必须是单调递增 counter,不要填瞬时速率。 +- `metrics` 是扩展 JSON,必须合法。 + +### 5.4 Gateway Report + +```http +POST {CLAWMANAGER_BACKEND_URL}/api/v1/runtime-agent/gateways/report +``` + +payload: + +```json +{ + "pod_id": 17, + "gateways": [ + { + "instance_id": 123, + "gateway_id": "gw-123-7", + "gateway_port": 20017, + "gateway_pid": 8842, + "state": "running", + "generation": 7, + "health_at": "2026-06-08T09:00:05Z" + } + ] +} +``` + +要求: + +- `gateways` 必填。 +- 每个 gateway 的 `instance_id`、`state`、`generation` 必填。 +- 后端只接受“当前绑定 Pod + 当前 generation”的上报;旧 Pod 或旧 generation 上报会被忽略。 +- `state=running` 或 `state=healthy` 会把绑定更新为 running。 +- `starting`、`stopped`、`error`、`unhealthy` 等状态会更新绑定状态。 +- 错误状态必须带 `error_message`,不要只上报空状态。 + +### 5.5 Skills Report + +```http +POST {CLAWMANAGER_BACKEND_URL}/api/v1/runtime-agent/skills/report +``` + +当前后端接收并发布任意 JSON payload。推荐按实例分组: + +```json +{ + "pod_id": 17, + "runtime_type": "openclaw", + "mode": "full", + "reported_at": "2026-06-08T09:00:05Z", + "instances": [ + { + "instance_id": 123, + "workspace_path": "/workspaces/openclaw/user-45/instance-123", + "skills": [ + { + "skill_id": "weather", + "skill_version": "1.0.0", + "identifier": "weather", + "install_path": "/workspaces/openclaw/user-45/instance-123/.openclaw/skills/weather", + "content_md5": "0123456789abcdef0123456789abcdef", + "source": "runtime", + "type": "agent-skill" + } + ] + } + ] +} +``` + +skill inventory 必须按 `instance_id` 和 `workspace_path` 隔离,不要把一个实例的 skill 混到另一个实例。 + +## 6. ClawManager 到 Agent 的控制接口 + +所有控制请求必须校验: + +```http +X-ClawManager-Control-Token: ${CLAWMANAGER_AGENT_CONTROL_TOKEN} +``` + +token 错误返回 `401`。不要返回 `3xx` redirect;后端会把 redirect 当成失败。 + +### 6.1 Health + +```http +GET /v1/health +``` + +任意 HTTP 2xx 都会被后端视为成功。建议响应: + +```json +{ + "status": "ready" +} +``` + +以下情况必须返回 `503`: + +- 必填环境变量缺失。 +- workspace root 不可访问。 +- 端口池不可用。 +- control/report token 缺失。 +- clock skew 超过健康阈值。 +- agent 正在关闭,不再接受新请求。 + +### 6.2 Create Gateway + +```http +POST /v1/gateways +``` + +请求: + +```json +{ + "instance_id": 123, + "user_id": 45, + "agent_type": "openclaw", + "workspace_path": "/workspaces/openclaw/user-45/instance-123", + "port_range": { + "start": 20000, + "end": 20099 + }, + "uid": 200123, + "gid": 200123, + "cpu_cores": 2, + "memory_mb": 4096, + "disk_quota_mb": 20480, + "generation": 7, + "environment": { + "CLAWMANAGER_LLM_BASE_URL": "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001/api/v1/gateway/llm", + "CLAWMANAGER_LLM_API_KEY": "instance-token", + "CLAWMANAGER_LLM_MODEL": "[\"auto\"]", + "CLAWMANAGER_LLM_PROVIDER": "openai-compatible", + "CLAWMANAGER_INSTANCE_TOKEN": "instance-token", + "OPENAI_API_KEY": "instance-token" + } +} +``` + +响应: + +```json +{ + "gateway_id": "gw-123-7", + "port": 20017, + "pid": 8842, + "status": "starting" +} +``` + +实现要求: + +- handler 必须快速返回,通常 1 到 3 秒内返回 `starting` 或已有 gateway 元数据。 +- 不能在 HTTP handler 内阻塞等待前台 gateway 进程完整运行。 +- `agent_type` 必须等于当前 Pod runtime type。 +- Pod draining 时返回 `409` 或 `503`,拒绝新建。 +- 以 `instance_id + generation` 为幂等键;重复请求不能重复启动进程。 +- 收到更高 generation 时,停止同实例旧 generation gateway 并释放端口。 +- 端口必须来自请求的 `port_range`。 +- 没有可用端口时返回: + +```http +HTTP/1.1 409 Conflict +Content-Type: text/plain + +no free port +``` + +- 启动失败、健康检查失败、配置写入失败时,上报 gateway `error` 并释放端口。 + +### 6.3 Delete Gateway + +```http +DELETE /v1/gateways/{gateway_id} +``` + +要求: + +- `gateway_id` 是 URL path escaped,agent 必须 decode 后再查找。 +- 优雅停止进程组,超时后强制 kill。 +- 释放端口、cgroup、进程表和本地状态。 +- 不删除 workspace。 +- gateway 不存在时建议返回 `204`,保持删除幂等。 +- 成功状态可以是 `200`、`202` 或 `204`。 + +### 6.4 Drain + +```http +POST /v1/drain +``` + +请求: + +```json +{ + "draining": true +} +``` + +要求: + +- 设置本地 draining 状态。 +- 立即拒绝新的 `POST /v1/gateways`。 +- 已有 gateway 不主动停止,继续健康检查和上报。 +- 立即补发 heartbeat,`state=draining, draining=true`。 +- 等 ClawManager 迁移实例后,逐个收到 DELETE 再清理。 + +## 7. Gateway 生命周期 + +推荐状态机: + +```text +requested -> reserved_port -> starting -> running + -> error +running -> stopping -> stopped +running -> unhealthy -> running/error +``` + +创建流程: + +1. 校验 token、runtime type、instance/user/generation、workspace、端口范围。 +2. 如果 draining,拒绝创建。 +3. 查本地状态,处理幂等。 +4. 如果 generation 更高,清理旧 generation。 +5. 加锁分配端口,标记为 reserved。 +6. 创建 workspace 和 home,设置 owner 为请求中的 `uid/gid`。 +7. 写入 runtime 配置和 LLM/proxy 配置。 +8. 持久化 gateway 元数据为 `starting`。 +9. 后台启动 gateway 子进程,立即返回。 +10. 健康检查通过后上报 `running`。 +11. 失败时上报 `error`,记录短错误信息并释放端口。 + +`running` 必须以真实进程和真实健康检查为准,不能只读旧状态文件。 + +## 8. 端口管理 + +agent 必须实现并发安全的端口分配器: + +```text +reserve(instance_id, generation, count) -> port_set +commit(instance_id, generation, port_set) +release(instance_id, generation) +list_used() +``` + +要求: + +- 分配时持有互斥锁。 +- 在锁内检查 agent 内存状态和系统监听端口。 +- 启动失败、健康检查失败、DELETE 成功后必须释放端口。 +- 如果 runtime 需要多个端口,必须一次性分配端口组,避免部分成功。 +- 同一 Pod 内端口不能冲突;不同 Pod 可以复用相同端口,因为路由使用 `pod_ip + port`。 + +## 9. Workspace 与资源隔离 + +workspace 固定格式: + +```text +/workspaces/{runtime}/user-{user_id}/instance-{instance_id} +``` + +agent 必须: + +- 对 workspace root 和请求路径执行 realpath。 +- 拒绝 `..`、绝对路径逃逸、符号链接逃逸、跨用户、跨实例路径。 +- 创建 `${workspace}/home`,并设置 `HOME=${workspace}/home`。 +- owner 设置为请求中的 `uid/gid`,当前默认 `200000 + instance_id`。 +- token、provider、配置文件权限建议 `0600`。 +- 删除 gateway 时不删除 workspace。 + +资源限制要求: + +- 每个 gateway 使用独立进程组。 +- 业务进程最终以实例 UID/GID 运行。 +- CPU/Memory 使用 cgroup 限制。 +- Disk 优先使用 filesystem quota;不支持时至少周期扫描并在超限时上报错误。 +- 资源限制无法启用时必须在 report 或错误消息中体现,不要静默忽略。 + +## 10. Runtime 配置和 LLM 注入 + +ClawManager 会通过 `POST /v1/gateways` 的 `environment` 字段下发实例级 LLM 环境变量。agent 只能转发白名单变量: + +```text +CLAWMANAGER_LLM_BASE_URL +CLAWMANAGER_LLM_API_KEY +CLAWMANAGER_LLM_MODEL +CLAWMANAGER_LLM_PROVIDER +CLAWMANAGER_INSTANCE_TOKEN +OPENAI_BASE_URL +OPENAI_API_BASE +OPENAI_API_KEY +OPENAI_MODEL +``` + +要求: + +- LLM base URL 使用 ClawManager 内部 Service DNS。 +- API key 使用当前实例 token。 +- 不要求用户在 runtime 页面手工填写 OpenAI key。 +- 不打印 token/API key。 +- 如果缺少 LLM key,不能上报 gateway `running`;应上报 `error` 并说明缺少实例 LLM token。 + +runtime 配置必须写在实例 workspace 内,例如: + +| Runtime | 推荐路径 | +| --- | --- | +| OpenClaw | `{workspace}/home/.openclaw/openclaw.json` | +| Hermes | `{workspace}/home/.hermes/hermes.json` 或 Hermes 原生配置路径 | + +JSON merge 规则: + +- 对象字段递归 merge。 +- 保留用户已有未知字段。 +- 平台必须控制的字段可以覆盖,但要在代码中显式列出。 +- 数组 append 后去重,例如 `allowedOrigins`。 +- 写入前先生成临时文件,再原子 rename。 + +## 11. 代理和浏览器访问 + +浏览器访问链路必须是: + +```text +Browser HTTPS -> ClawManager HTTPS -> ClawManager backend proxy -> Runtime Pod HTTP gateway +``` + +agent 写 runtime 配置时必须保证: + +- gateway 支持 `/api/v1/instances/{instance_id}/proxy` base path。 +- allowed origin 使用 ClawManager 内部 origin,例如 `http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001`。 +- trusted proxies 来自 `CLAWMANAGER_TRUSTED_PROXY_CIDRS` 或等价 env。 +- 不使用外部地址作为 runtime 内部 trusted origin。 +- 通过 ClawManager proxy 访问时,不应要求用户手工粘贴 gateway token。 + +## 12. 时间同步 + +agent 必须处理 clock skew: + +- 注册和 heartbeat 响应如果带 `server_time`,记录本地时间和服务端时间偏移。 +- 偏移超过 30 秒时记录 warning,并在 health 中暴露 `clock_skew_warning`。 +- 偏移超过 120 秒时不应上报 `ready` 或 gateway `running`。 +- 所有签名、token 时间和上报时间使用 UTC。 +- 不在容器里修改系统时间,时间同步由 Kubernetes 节点 NTP/chrony 负责。 + +常见症状包括 `device signature expired`、JWT/WebSocket 鉴权异常、签名 URL 过期和 gateway 实际状态与 ClawManager 展示不一致。 + +## 13. Instance Agent v1 兼容协议 + +当前后端仍提供 `/api/v1/agent/*`,用于实例级状态、命令和 skill 管理。托管 runtime 如果需要复用这套能力,应由具体实例的 gateway/agent 逻辑使用实例级环境变量。 + +ClawManager 会为支持托管集成的实例注入: + +| 变量 | 说明 | +| --- | --- | +| `CLAWMANAGER_AGENT_ENABLED` | `true` 时启用实例 agent | +| `CLAWMANAGER_AGENT_BASE_URL` | ClawManager API base URL,不带 `/api/v1/agent` 后缀 | +| `CLAWMANAGER_AGENT_BOOTSTRAP_TOKEN` | 首次注册使用 | +| `CLAWMANAGER_AGENT_INSTANCE_ID` | 当前实例 ID | +| `CLAWMANAGER_AGENT_PROTOCOL_VERSION` | 当前为 `v1` | +| `CLAWMANAGER_AGENT_PERSISTENT_DIR` | 当前实例持久化目录 | +| `CLAWMANAGER_AGENT_DISK_LIMIT_BYTES` | 实例磁盘配额 | + +### 13.1 Register + +```http +POST {base}/api/v1/agent/register +Authorization: Bearer ${CLAWMANAGER_AGENT_BOOTSTRAP_TOKEN} +``` + +payload: + +```json +{ + "instance_id": 123, + "agent_id": "openclaw-123-main", + "agent_version": "0.1.0", + "protocol_version": "v1", + "capabilities": [ + "runtime.status", + "runtime.health", + "metrics.report", + "skills.inventory", + "skills.upload", + "commands.poll", + "llm.gateway" + ], + "host_info": { + "runtime_type": "openclaw", + "runtime_name": "OpenClaw", + "image": "ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest", + "persistent_dir": "/config" + } +} +``` + +响应 `data` 包含: + +- `session_token` +- `session_expires_at` +- `heartbeat_interval_seconds`,当前默认 `15` +- `command_poll_interval_seconds`,当前默认 `5` +- `server_time` + +session token 每次成功 heartbeat 会续期 24 小时。收到 `401` 时删除本地 session,并用 bootstrap token 重新注册。 + +### 13.2 Heartbeat + +```http +POST {base}/api/v1/agent/heartbeat +Authorization: Bearer ${session_token} +``` + +payload: + +```json +{ + "agent_id": "openclaw-123-main", + "timestamp": "2026-06-08T09:00:15Z", + "openclaw_status": "running", + "current_config_revision_id": null, + "summary": { + "runtime_type": "openclaw", + "runtime_status": "running", + "runtime_pid": 245, + "openclaw_pid": 245, + "skill_count": 8, + "disk_used_bytes": 2147483648, + "disk_limit_bytes": 10737418240 + } +} +``` + +兼容要求: + +- `openclaw_status`、`openclaw_pid`、`openclaw_version` 是历史字段名;非 OpenClaw runtime 也需要填自己的主进程状态、PID 和版本,直到协议升级。 +- 后端 45 秒内收到 heartbeat 展示 online,45 到 120 秒展示 stale,超过 120 秒展示 offline。 +- heartbeat 响应里的 `has_pending_command=true` 时,应立即拉取命令。 + +### 13.3 State Report + +```http +POST {base}/api/v1/agent/state/report +Authorization: Bearer ${session_token} +``` + +payload: + +```json +{ + "agent_id": "openclaw-123-main", + "reported_at": "2026-06-08T09:00:20Z", + "runtime": { + "openclaw_status": "running", + "openclaw_pid": 245, + "openclaw_version": "openclaw-2026.5.4", + "current_config_revision_id": null + }, + "system_info": { + "runtime_type": "openclaw", + "cpu": { + "cores": 2, + "load": { + "1m": 0.64, + "5m": 0.52, + "15m": 0.4 + } + }, + "memory": { + "mem_total_bytes": 4294967296, + "mem_available_bytes": 2147483648 + }, + "disk": { + "root_total_bytes": 10737418240, + "root_free_bytes": 8589934592 + }, + "network": { + "interfaces": [ + { + "name": "eth0", + "status": "up", + "addresses": ["10.42.0.12"], + "rx_bytes": 123456789, + "tx_bytes": 98765432 + } + ] + } + }, + "health": { + "runtime_process": "ok", + "agent": "ok", + "metrics_collector": "ok" + } +} +``` + +network 的 `rx_bytes` / `tx_bytes` 必须是单调递增 counter。 + +### 13.4 Commands + +拉取: + +```http +GET {base}/api/v1/agent/commands/next +Authorization: Bearer ${session_token} +``` + +开始: + +```http +POST {base}/api/v1/agent/commands/{id}/start +Authorization: Bearer ${session_token} +``` + +完成: + +```http +POST {base}/api/v1/agent/commands/{id}/finish +Authorization: Bearer ${session_token} +``` + +finish 只允许: + +- `succeeded` +- `failed` + +当前支持的命令类型: + +```text +start_openclaw +stop_openclaw +restart_openclaw +collect_system_info +apply_config_revision +reload_config +health_check +install_skill +update_skill +uninstall_skill +remove_skill +disable_skill +quarantine_skill +handle_skill_risk +sync_skill_inventory +refresh_skill_inventory +collect_skill_package +``` + +未知命令必须 finish 为 `failed`,`error_message` 写明 `unsupported command type: `。 + +### 13.5 Skills + +inventory: + +```http +POST {base}/api/v1/agent/skills/inventory +Authorization: Bearer ${session_token} +``` + +upload: + +```http +POST {base}/api/v1/agent/skills/upload +Authorization: Bearer ${session_token} +Content-Type: multipart/form-data +``` + +上传表单字段: + +- `file` +- `agent_id` +- `skill_id` +- `skill_version` +- `identifier` +- `content_md5` +- `source` + +要求: + +- `identifier` 和 `content_md5` 必须稳定。 +- `mode=full` 的 inventory 代表全量结果,平台会用它对齐实例 skill 状态。 +- `content_md5` 按目录内容指纹计算,不是 zip 文件 MD5;算法见 `docs/skill-content-md5-spec.md`。 +- 打包和解压必须防止路径逃逸。 + +## 14. 安全规范 + +- 控制接口只接受正确 `X-ClawManager-Control-Token`。 +- 上报接口只使用 `X-ClawManager-Agent-Token`。 +- 实例级接口使用 Bearer bootstrap/session token。 +- 日志不得打印 token、API key、完整 Authorization header、channel secret。 +- 启动进程使用 argv 数组,不使用 shell 拼接: + +```text +execve(binary, ["openclaw", "gateway", "run", "--port", "20017"], env) +``` + +- 不允许用户环境变量覆盖 agent 自己的 control/report token。 +- 不允许 `allowedOrigins=["*"]` 作为生产配置。 +- 所有用户输入路径都要边界校验。 +- 命令执行必须有超时,超过 `timeout_seconds` 主动终止并 finish failed。 + +## 15. 测试要求 + +单元测试至少覆盖: + +- Runtime type 只接受 `openclaw` / `hermes`。 +- create gateway 幂等。 +- 更高 generation 替换旧 generation。 +- 端口并发分配无冲突。 +- 端口被系统占用时跳过。 +- 端口耗尽返回 `409 no free port`。 +- workspace path 越权被拒绝。 +- JSON merge 保留用户字段,数组去重。 +- LLM env 白名单转发。 +- 缺少 LLM token 时 gateway 不报 `running`。 +- trusted proxy 和 allowed origin 配置生成。 +- clock skew 阈值判断。 +- redirect 响应不被当成成功。 +- DELETE gateway id path escape/decode。 + +集成测试至少覆盖: + +- Runtime Pod 注册为 `ready`,capacity/used_slots/draining 正确。 +- heartbeat 更新 Pod 状态。 +- metrics 上报后管理端 Runtime Pods 页面能看到指标。 +- 创建 100 个 gateway 端口唯一,第 101 个触发扩容或 no capacity。 +- 重复 create 同一 `instance_id + generation` 不重复启动。 +- gateway report 旧 generation 不覆盖新 generation。 +- ClawManager 通过 proxy 能访问 gateway 页面和 WebSocket。 +- 通过 ClawManager proxy 访问时不要求用户手工粘贴 gateway token。 +- Pod drain 后拒绝新 create,已有 gateway 持续上报,DELETE 后释放端口。 +- Pod 重启后不误报旧 gateway running。 +- instance agent register/heartbeat/state/commands/skills 全链路可用。 +- 日志中没有 token 和 API key。 + +## 16. Do / Don't + +| Do | Don't | +| --- | --- | +| 使用 Kubernetes Service DNS 访问 ClawManager backend | 写死外部 NodePort、`172.16.1.12` 或浏览器 HTTPS 入口 | +| `POST /v1/gateways` 快速返回 `starting` | 在 HTTP handler 中阻塞等待前台进程退出 | +| 以 `instance_id + generation` 幂等 | 重试时重复启动多个 gateway | +| 以真实进程和健康检查决定 `running` | 只读旧状态文件就上报 `running` | +| JSON merge runtime 配置 | 覆盖用户整个配置文件 | +| 只转发白名单 env | 把用户请求里的全部 env 原样传给进程 | +| trusted proxy 模式下通过 ClawManager 自动进入 | 要求用户手工粘贴 gateway token | +| 检测并上报 clock skew | 忽略时间导致签名和 WebSocket 鉴权问题反复出现 | +| DELETE gateway 不删除 workspace | 清理进程时顺手删除用户数据 | + +## 17. 代码位置 + +后端关键实现: + +- Runtime Pod 上报接口:`backend/internal/handlers/runtime_agent_handler.go` +- ClawManager 调 agent 客户端:`backend/internal/services/runtime_agent_client.go` +- Runtime 调度器:`backend/internal/services/runtime_scheduler.go` +- Runtime Deployment 构建:`backend/internal/services/k8s/runtime_deployment_service.go` +- Runtime 常量与路径:`backend/internal/services/runtime_capacity.go` +- 实例 agent handler:`backend/internal/handlers/agent_handler.go` +- 实例 agent service:`backend/internal/services/instance_agent_service.go` +- 实例命令 service:`backend/internal/services/instance_command_service.go` +- 实例 runtime 状态 service:`backend/internal/services/instance_runtime_status_service.go` + +相关文档: + +- `docs/clawmanager-agent-v2-contract.md` +- `docs/runtime-agent-integration-guide.md` +- `docs/hermes-lite-pro-agent-development.md` +- `docs/hermes-runtime-agent-development.md` +- `docs/skill-content-md5-spec.md` diff --git a/docs/hermes-lite-pro-agent-development.md b/docs/hermes-lite-pro-agent-development.md new file mode 100644 index 0000000..82ef047 --- /dev/null +++ b/docs/hermes-lite-pro-agent-development.md @@ -0,0 +1,227 @@ +# Hermes Lite / Pro Agent 开发说明 + +本文写给 Hermes agent 侧开发者,用来区分 ClawManager 中 Hermes Lite 和 Hermes Pro 两种形态。先读本文,再分别参考: + +- Lite: `docs/agent-runtime-development-spec.md` 和 `docs/clawmanager-agent-v2-contract.md` +- Pro: `docs/hermes-runtime-agent-development.md` 和 `docs/agent-control-plane.md` + +一句话原则: + +- **Hermes Lite 是 Runtime Pod Agent V2**:一个共享 `hermes-runtime` Pod 内只有一个 pod-level agent,它负责创建、删除和上报多个 Hermes gateway 子进程。 +- **Hermes Pro 是 Instance Agent Control Plane**:每个 Hermes Pro 实例有自己的桌面容器和 instance-level agent,它只负责这个实例自己的状态、命令、skill 和 metrics。 + +不要只根据镜像名或 `hermes` 这个 runtime 类型判断开发目标。Hermes Lite 和 Hermes Pro 都叫 Hermes,但 agent 协议、进程模型、目录和端口完全不同。 + +## 1. 模式对照 + +| 项目 | Hermes Lite | Hermes Pro | +| --- | --- | --- | +| ClawManager mode | `lite` | `pro` | +| Runtime backend | `gateway` | `desktop` | +| Kubernetes 资源 | 共享 Hermes runtime Deployment/Pod | 每个实例一个专属 Deployment + Service | +| agent 定位 | Pod 级 gateway 管理器 | 实例级状态和命令 agent | +| agent 入站端口 | 必须监听 `0.0.0.0:19090` | 不需要实现 runtime-pod 控制端口 | +| ClawManager 调 agent | `GET /v1/health`、`POST /v1/gateways`、`DELETE /v1/gateways/{id}`、`POST /v1/drain` | 不适用 | +| agent 上报 ClawManager | `/api/v1/runtime-agent/*` | `/api/v1/agent/*` | +| 用户实例进程 | 每个 Lite 实例是一个 gateway 子进程 | 整个桌面容器就是实例 | +| 用户访问端口 | agent 分配 `20000-20099` 中的 gateway 端口 | Webtop/KasmVNC `3001` | +| workspace | `/workspaces/hermes/user-{user_id}/instance-{instance_id}` | `/config/.hermes` | +| 典型实现参考 | 按 OpenClaw Lite 的 Runtime Pod Agent V2 做 | 按当前 Hermes Webtop guide 做 | + +## 2. 如何判断当前该跑哪种 agent + +建议 Hermes 镜像里可以放同一个 `hermes-agent` 二进制,但必须拆成两个清晰入口: + +```bash +hermes-agent runtime-pod # Lite +hermes-agent instance # Pro +``` + +也可以自动检测环境变量: + +| 检测条件 | 应进入的模式 | +| --- | --- | +| `CLAWMANAGER_RUNTIME_TYPE=hermes` 且存在 `CLAWMANAGER_AGENT_PORT` 或 `RUNTIME_AGENT_PUBLIC_PORT` | Lite runtime-pod agent | +| `CLAWMANAGER_AGENT_ENABLED=true` 且存在 `CLAWMANAGER_AGENT_INSTANCE_ID` 和 `CLAWMANAGER_AGENT_BOOTSTRAP_TOKEN` | Pro instance agent | + +如果两组变量同时存在,优先按明确的启动参数决定;没有启动参数时建议拒绝启动并打印清晰错误,避免一个进程同时承担两套协议。 + +## 3. Hermes Lite 应该怎么开发 + +Hermes Lite 要仿照当前 OpenClaw Lite 做。差异只有 runtime 名称、Hermes gateway 启动命令、Hermes 配置文件位置和健康检查细节。 + +Lite agent 是共享 Pod 内的控制进程,不是某个用户实例内部的 agent。它启动后必须: + +- 监听 `0.0.0.0:19090`。 +- 校验 `X-ClawManager-Control-Token: ${CLAWMANAGER_AGENT_CONTROL_TOKEN}`。 +- 实现 `GET /v1/health`、`POST /v1/gateways`、`DELETE /v1/gateways/{gateway_id}`、`POST /v1/drain`。 +- 向 `${CLAWMANAGER_BACKEND_URL}/api/v1/runtime-agent/*` 注册、heartbeat、metrics、gateway 状态和 skill inventory。 +- 管理本 Pod 内多个 Hermes gateway 子进程,端口默认从 `20000-20099` 分配。 +- 按 `instance_id + generation` 做幂等创建,重复请求不能启动多个进程。 +- `POST /v1/gateways` 必须快速返回 `starting`,后台继续启动和健康检查。 +- gateway 健康后再上报 `running`;缺少 LLM token、workspace 越权、端口冲突或 Hermes 配置失败时上报 `error`。 +- 每个实例使用独立 workspace,不能共用一个全局 `/config/.hermes`。 + +Lite runtime Pod 会收到这些环境变量: + +| 变量 | 用途 | +| --- | --- | +| `CLAWMANAGER_RUNTIME_TYPE=hermes` | runtime 类型,注册和上报都用 `hermes` | +| `CLAWMANAGER_BACKEND_URL` | ClawManager backend 内部地址 | +| `CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME` | 当前 runtime Deployment 名 | +| `CLAWMANAGER_RUNTIME_IMAGE_REF` | 当前 runtime 镜像 | +| `CLAWMANAGER_AGENT_PORT=19090` | agent 控制端口 | +| `CLAWMANAGER_AGENT_CONTROL_TOKEN` | ClawManager 调 agent 控制接口使用 | +| `CLAWMANAGER_AGENT_REPORT_TOKEN` | agent 上报 ClawManager 使用 | +| `RUNTIME_WORKSPACE_ROOT=/workspaces` | workspace 根目录 | +| `RUNTIME_AGENT_LISTEN_ADDR=0.0.0.0:19090` | agent 监听地址 | +| `RUNTIME_AGENT_PUBLIC_PORT=19090` | agent 对 ClawManager 暴露的端口 | +| `RUNTIME_GATEWAY_PORT_START` / `RUNTIME_GATEWAY_PORT_END` | gateway 端口范围 | +| `POD_NAME` / `POD_NAMESPACE` / `POD_IP` / `NODE_NAME` | Pod 身份和调度信息 | +| `CLAWMANAGER_TRUSTED_PROXY_CIDRS` | ClawManager 反代来源网段,用于 Hermes trusted proxy | + +ClawManager 创建 Lite 实例时会调用 runtime-pod agent: + +```http +POST /v1/gateways +X-ClawManager-Control-Token: ${CLAWMANAGER_AGENT_CONTROL_TOKEN} +Content-Type: application/json +``` + +```json +{ + "instance_id": 123, + "user_id": 45, + "generation": 7, + "agent_type": "hermes", + "workspace_path": "/workspaces/hermes/user-45/instance-123", + "gateway_port_start": 20000, + "gateway_port_end": 20099, + "uid": 200123, + "gid": 200123, + "environment": { + "CLAWMANAGER_LLM_BASE_URL": "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001/api/v1/gateway/llm", + "CLAWMANAGER_LLM_API_KEY": "instance-token", + "CLAWMANAGER_LLM_MODEL": "[{\"id\":\"gpt-4.1\"}]", + "CLAWMANAGER_INSTANCE_TOKEN": "instance-token", + "OPENAI_BASE_URL": "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001/api/v1/gateway/llm", + "OPENAI_API_KEY": "instance-token" + } +} +``` + +Lite agent 的建议处理流程: + +1. 校验控制 token、`agent_type=hermes`、`workspace_path` 必须在 `/workspaces/hermes/` 下。 +2. 用 `instance_id + generation` 查本地状态;如果已经创建过,返回同一个 gateway 元数据。 +3. 创建 workspace,并把 owner 设置为请求里的 `uid/gid`。 +4. 分配未占用 gateway 端口,记录端口、pid、workspace、generation。 +5. 把请求里的 `environment` 白名单变量传给 Hermes gateway 子进程。 +6. 写入 Hermes gateway 需要的 LLM、workspace、trusted proxy 和 instance token 配置。 +7. 后台启动 Hermes gateway,例如 `hermes gateway run --port ${port}`,实际命令由 Hermes 项目确定。 +8. 立即返回 `status=starting`。 +9. 后台轮询健康检查,成功后上报 `/api/v1/runtime-agent/gateways/report` 为 `running`。 +10. 删除时停止对应进程组并释放端口,但不要删除 workspace。 + +Lite 下浏览器访问链路是: + +```text +Browser -> ClawManager -> Runtime Pod HTTP gateway +``` + +ClawManager 会向 Lite runtime gateway 代理请求注入实例 token。Hermes gateway 不应该要求用户手工粘贴 gateway token;它需要信任来自 ClawManager 的代理来源,并正确处理 `Authorization`、`X-Forwarded-*` 和内部 base URL。 + +## 4. Hermes Pro 应该怎么开发 + +Hermes Pro 继续按现有 Hermes Webtop 文档实现。Pro 是专属桌面实例,不需要实现 Lite 的 `/v1/gateways` 协议,也不需要在一个 Pod 里管理多个用户实例。 + +Pro image 的固定约定: + +- 基于 Webtop/KasmVNC。 +- 对外服务端口是 `3001`。 +- 持久化目录是 `/config/.hermes`。 +- ClawManager 会把 `SUBFOLDER` 设置为 `/api/v1/instances/{instance_id}/proxy/`。 +- agent 作为 s6 longrun 或等价守护进程运行在这个实例容器里。 + +Pro instance agent 会收到这些环境变量: + +| 变量 | 用途 | +| --- | --- | +| `CLAWMANAGER_AGENT_ENABLED=true` | 启用实例 agent | +| `CLAWMANAGER_AGENT_BASE_URL` | ClawManager API base URL,不带 `/api/v1/agent` 后缀 | +| `CLAWMANAGER_AGENT_BOOTSTRAP_TOKEN` | 首次注册使用的一次性 token | +| `CLAWMANAGER_AGENT_INSTANCE_ID` | 当前实例 ID | +| `CLAWMANAGER_AGENT_PROTOCOL_VERSION=v1` | instance agent 协议版本 | +| `CLAWMANAGER_AGENT_PERSISTENT_DIR=/config/.hermes` | Hermes 持久化目录 | +| `CLAWMANAGER_AGENT_DISK_LIMIT_BYTES` | 实例磁盘配额 | + +Pro agent 的职责: + +- 读取 `CLAWMANAGER_AGENT_*` 变量。 +- 向 `${CLAWMANAGER_AGENT_BASE_URL}/api/v1/agent/register` 注册。 +- 使用 bootstrap 后返回的 session token 持续 heartbeat。 +- 上报 runtime state、health、system metrics、config revision 和 skill inventory。 +- 轮询并执行 ClawManager 下发的 instance command。 +- 管理 `/config/.hermes/skills` 里的 skill 下载、安装、上传和 inventory。 +- 所有状态都只代表这个 Pro 实例自身。 + +Pro agent 不应该实现: + +- `POST /v1/gateways` +- gateway 端口池 +- 多实例 workspace 隔离 +- `/api/v1/runtime-agent/*` runtime pod 上报 +- Lite 的 drain 和 gateway 迁移逻辑 + +## 5. OpenClaw 对照开发法 + +如果你已经看过 OpenClaw 的实现,可以这样映射: + +| OpenClaw 概念 | Hermes Lite 应替换为 | +| --- | --- | +| `CLAWMANAGER_RUNTIME_TYPE=openclaw` | `CLAWMANAGER_RUNTIME_TYPE=hermes` | +| `agent_type=openclaw` | `agent_type=hermes` | +| `/workspaces/openclaw/user-{uid}/instance-{id}` | `/workspaces/hermes/user-{uid}/instance-{id}` | +| OpenClaw gateway 启动命令 | Hermes gateway 启动命令 | +| OpenClaw workspace 配置文件 | Hermes workspace 内的实例级配置文件 | +| OpenClaw 健康检查 | Hermes gateway 健康检查 | + +不要把 OpenClaw Pro/Desktop 的 agent 逻辑直接套到 Hermes Lite。Lite 的关键不是“启动一个桌面”,而是“在共享 Pod 中稳定管理多个 gateway 子进程”。 + +## 6. 验收清单 + +Hermes Lite 验收: + +- runtime pod agent 可以注册为 `runtime_type=hermes`。 +- `GET /v1/health` 返回 ready 状态和真实容量。 +- 创建 100 个 Hermes gateway 时端口唯一,状态最终变为 `running`。 +- 重复 `POST /v1/gateways` 不会启动重复进程。 +- 删除 gateway 会停止进程并释放端口,不会删除 workspace。 +- `POST /v1/drain` 后拒绝新 create,已有 gateway 持续上报。 +- Pod 重启后不会把不存在的旧 gateway 误报为 `running`。 +- 缺少 LLM token 时 gateway 不报告 `running`,错误信息明确。 +- 通过 ClawManager proxy 访问 Hermes Lite 不需要用户手工粘贴 token。 +- 每个 Lite 实例的 skill inventory、配置和 workspace 互相隔离。 + +Hermes Pro 验收: + +- Pro 实例的 Webtop 端口 `3001` 可通过 ClawManager proxy 访问。 +- `/config/.hermes` 持久化,实例重启后数据仍在。 +- instance agent 能注册、heartbeat、上报 state 和 metrics。 +- instance command 能 poll、start、finish 并返回失败原因。 +- `/config/.hermes/skills` 的 skill 下载、安装、上传和 inventory 可用。 +- Pro agent 不监听或实现 `/v1/gateways`。 + +## 7. 常见错误 + +| 错误 | 正确做法 | +| --- | --- | +| 只实现 Pro instance agent,然后期望 Lite 可用 | Lite 必须实现 Runtime Pod Agent V2 | +| Lite agent 监听 `3001` | Lite agent 监听 `19090`,gateway 子进程监听 `20000-20099` | +| Lite 所有实例共用 `/config/.hermes` | Lite 每个实例使用自己的 `/workspaces/hermes/user-{uid}/instance-{id}` | +| `POST /v1/gateways` 阻塞到 Hermes gateway 完全启动 | 快速返回 `starting`,后台健康检查后上报 `running` | +| 重试 create 时启动多个 gateway | 按 `instance_id + generation` 幂等 | +| 把外部 HTTPS/NodePort 写进 runtime 内部配置 | runtime 内部用 ClawManager backend service DNS | +| 要求用户粘贴 gateway token | 使用 ClawManager 注入的 instance token 和 trusted proxy | +| Lite 上报 `/api/v1/agent/*` 当作生命周期依据 | Lite 生命周期必须上报 `/api/v1/runtime-agent/*` | +| Pro 实现 `/v1/gateways` 和端口池 | Pro 只实现 instance-level `/api/v1/agent/*` | diff --git a/docs/hermes-runtime-agent-development.md b/docs/hermes-runtime-agent-development.md index 1fc396a..25ce32c 100644 --- a/docs/hermes-runtime-agent-development.md +++ b/docs/hermes-runtime-agent-development.md @@ -2,6 +2,8 @@ This guide is for Hermes image and agent developers. It explains how to build a ClawManager-managed Hermes runtime image on top of Webtop, and how the embedded Hermes agent should integrate with ClawManager Agent Control Plane so Hermes can behave like OpenClaw: report live runtime status, report health and system metrics, sync skill inventory, upload skill packages, and poll runtime commands. +> Mode note: this document describes the Hermes Pro Webtop/instance-agent path. For Hermes Lite, implement Runtime Pod Agent V2 instead. See [Hermes Lite / Pro Agent 开发说明](./hermes-lite-pro-agent-development.md). + ## Goals A Hermes image must satisfy two layers of requirements: diff --git a/docs/lite-mode-development-progress-2026-06-11.md b/docs/lite-mode-development-progress-2026-06-11.md new file mode 100644 index 0000000..859e8eb --- /dev/null +++ b/docs/lite-mode-development-progress-2026-06-11.md @@ -0,0 +1,98 @@ +# Lite 模式与 Share Link 开发进度汇报 + +> 汇报时间:2026-06-11 下午 + +## 技术实现概览 + +本次开发的核心是把实例形态从原来的运行时类型,抽象成用户可理解的 `Lite / Pro` 两种模式。 + +- `Lite`:使用共享 Runtime Pod,在 Pod 内为每个实例创建独立 gateway 进程,适合轻量、低资源成本的使用场景。 +- `Pro`:使用独立的 Kubernetes Deployment / Service / PVC,适合需要完整桌面运行环境和独立资源的场景。 + +后端以 `instance_mode` 作为产品层模式,内部再映射到不同 runtime backend: + +- `lite` -> `gateway` +- `pro` -> `desktop` + +这样前端、普通实例创建、Team 成员创建、管理端展示和外部访问都统一使用 Lite / Pro 语义;底层调度、代理、资源控制仍然按 gateway / desktop 执行。 + +Share Link 能力基于实例外部访问表实现,生成 `/s//` 形式的短链接。短链 code 只在创建时返回,数据库只保存 hash;访问时由 ClawManager 解析短链,再复用现有实例代理链路转发到对应 Lite gateway 或 Pro service。当前支持普通分享链接和密码访问,并支持固定时间、自定义时间和永久有效几类过期策略。 + +## Lite / Pro 架构图 + +```mermaid +flowchart LR + user["用户 / 外部访问者"] --> web["ClawManager Web"] + user --> shortLink["Share Link
/s/<code>/"] + + web --> api["ClawManager Backend API"] + shortLink --> publicAccess["短链解析与鉴权
Share Link / Password"] + publicAccess --> proxy["统一实例代理"] + api --> instanceService["Instance Service
instance_mode: lite / pro"] + instanceService --> mode{"实例模式"} + + mode -->|Lite| liteScheduler["Lite Scheduler
runtime_type = gateway"] + mode -->|Pro| proLifecycle["Pro Lifecycle
runtime_type = desktop"] + + liteScheduler --> runtimePod["共享 Runtime Pod"] + runtimePod --> runtimeAgent["Runtime Pod Agent V2"] + runtimeAgent --> gatewayA["Gateway 进程
Lite 实例 A"] + runtimeAgent --> gatewayB["Gateway 进程
Lite 实例 B"] + runtimeAgent --> gatewayN["Gateway 进程
Lite 实例 N"] + + proLifecycle --> deployment["专属 Deployment"] + deployment --> service["专属 Service"] + deployment --> pvc["专属 PVC"] + service --> desktop["Desktop Runtime
Pro 实例"] + + proxy -->|Lite| gatewayA + proxy -->|Lite| gatewayB + proxy -->|Pro| service + + team["Team 创建"] --> teamService["Team Service"] + teamService --> teamMemberMode{"成员实例模式"} + teamMemberMode -->|Lite 成员| liteScheduler + teamMemberMode -->|Pro 成员| proLifecycle + + aiGateway["AI Gateway / 审计 / 费用 / 风险"] --> attribution["实例归因
instance_id / instance_mode / runtime_type"] + gatewayA --> attribution + gatewayB --> attribution + desktop --> attribution +``` + +## 目标 + +- 新增 Lite 实例模式,让用户在创建普通实例时可以选择 Lite / Pro。 +- 保留 Pro 模式的完整桌面实例能力,同时让 Lite 模式复用共享运行时,降低资源占用和启动成本。 +- 将 Lite / Pro 模式贯通到 Team 场景,让 Team 成员实例也可以按角色选择 Lite 或 Pro。 +- 增加 Share Link 能力,支持将实例访问入口分享给外部用户。 +- 建立自动化测试覆盖,保证普通实例、Team 实例和分享访问能力可以持续回归。 + +## 进展 + +- 普通实例创建已经支持 Lite / Pro 模式选择。 +- Lite 实例创建时会进入共享 gateway runtime 链路,Pro 实例创建时会进入独立 desktop runtime 链路。 +- 前端创建页、实例详情页、实例列表和管理端已完成 Lite / Pro 的展示和交互区分。 +- Lite 模式下已简化资源配置流程,不再要求用户配置专属 CPU、内存、存储和 GPU;Pro 模式继续保留专属资源配置能力。 +- Team 创建流程已经支持为成员选择 Lite / Pro 实例模式。 +- Team 成员实例已能按选择的模式创建,并保留后续接入 Team 协作链路所需的基础配置。 +- Share Link 已完成基础能力,包括短链接分享、密码访问、过期时间选择和复制入口。 +- Share Link 访问已统一走 ClawManager 代理,能够兼容 Lite gateway 和 Pro service 两种后端形态。 +- 相关改动已部署到 172.16.1.12 环境,并完成一轮自动化回归。 +- 当前自动化测试已覆盖普通实例 Lite / Pro 模式、Share Link 基础行为、管理端模式展示以及 Team Lite 创建链路。 +- 联调中发现,部分 Lite runtime / agent 能力仍需补齐,主要集中在 Team 任务消费和 Hermes Lite 会话稳定性上。 + +## 计划 + +- 继续推进 agent / runtime 侧补齐 Lite 模式能力,重点包括 Team 任务接收、执行状态回写和会话稳定性。 +- 在 agent / runtime 修复后,补充完整的 Team Lite 任务执行 e2e,覆盖从建队、成员实例创建、任务下发到状态回写的闭环。 +- 继续完善 Hermes Lite 的联调验证,确保 Lite 模式下聊天、访问和实例生命周期稳定。 +- 对 Share Link 增加更完整的回归覆盖,包括短链访问、密码访问、过期策略、禁用/重新生成等场景。 +- 在 172.16.1.12 环境继续做端到端回归,确认普通实例、Team 实例和 Share Link 在 Lite / Pro 两种模式下都能稳定运行。 +- 最后整理验收说明,明确 Lite / Pro 的支持范围、已验证场景和仍需 agent 侧配合的事项。 + +## 简短汇报口径 + +这次开发不是单独做 Team Lite,而是把实例模式整体升级为 Lite / Pro 两种形态。Lite 走共享 runtime gateway,Pro 走独立 desktop runtime,对用户层统一暴露 Lite / Pro,对后端仍按不同 runtime backend 调度。 + +目前普通实例创建、实例展示、管理端识别、Team 成员创建以及 Share Link 分享能力都已经完成基础支持,并已在 172.16.1.12 做了一轮自动化回归。后续重点是继续和 agent / runtime 侧联调,补齐 Lite 模式下 Team 任务执行闭环和 Hermes Lite 会话稳定性,然后补充更完整的端到端验收测试。 diff --git a/docs/openclaw-windows-node-exec.md b/docs/openclaw-windows-node-exec.md new file mode 100644 index 0000000..5cee925 --- /dev/null +++ b/docs/openclaw-windows-node-exec.md @@ -0,0 +1,233 @@ +# OpenClaw Gateway + Windows Node Exec Configuration + +本文说明一个常见 ClawManager 使用场景: + +```text +用户浏览器 + -> ClawManager Web / OpenClaw WebChat + -> 服务端 OpenClaw 实例里的 Gateway + -> 本机 Windows 上运行的 OpenClaw node + -> 在 Windows 本机执行 system.run / system.which +``` + +目标是:用户访问 ClawManager 创建出的 OpenClaw 实例 Web UI,在聊天中让 Agent 操作个人 Windows 电脑上的命令或自动化任务。 + +## 1. 基本判断 + +这个场景可行,但要区分两类授权: + +| 授权类型 | 作用 | 是否每次变化 | +| --- | --- | --- | +| Device pairing | 允许 Windows node 接入服务端 Gateway | 通常只需要一次 | +| Exec approval | 允许某条 `system.run` 在 Windows node 上执行 | ask 模式下每条命令都会生成新的 requestId | + +如果错误是: + +```json +{ + "status": "error", + "tool": "exec", + "error": "UNAVAILABLE: SYSTEM_RUN_DENIED: approval required" +} +``` + +说明链路已经打到 Windows node,问题不是网络,而是 `system.run` 被 Windows node 的 exec approval 策略拦住。 + +## 2. 前置条件 + +服务端 OpenClaw 实例里的 Gateway 必须满足: + +- Gateway WebSocket 端口能被 Windows 电脑访问,例如 `192.168.30.133:31889`。 +- Gateway 不能只监听容器内 `127.0.0.1`。 +- 如果使用明文 `ws://`,建议只在内网、VPN、Tailscale 或 SSH tunnel 内使用。 +- Windows node 需要拿到 Gateway 的认证 token 或 password。 + +Windows 本机需要满足: + +- 已安装 OpenClaw CLI。 +- 能访问服务端 Gateway 地址和端口。 +- Node 进程能持久运行,建议调通后改为 service 模式。 + +## 3. 启动 Windows Node + +在服务端 OpenClaw 实例里查看 Gateway token: + +```bash +openclaw config get gateway.auth.token +``` + +在 Windows PowerShell 中设置 token 并启动 node: + +```powershell +$env:OPENCLAW_GATEWAY_TOKEN="" +openclaw node run --host 192.168.30.133 --port 31889 --display-name "windows-node" +``` + +如果要后台常驻: + +```powershell +$env:OPENCLAW_GATEWAY_TOKEN="" +openclaw node install --host 192.168.30.133 --port 31889 --display-name "windows-node" --force +openclaw node restart +openclaw node status +``` + +如果 Gateway 使用 TLS,再加: + +```powershell +openclaw node run --host 192.168.30.133 --port 31889 --tls --display-name "windows-node" +``` + +## 4. 在 Gateway 侧完成 Node Pairing + +进入服务端 OpenClaw 实例终端,查看待审批设备: + +```bash +openclaw devices list +``` + +找到 role 为 `node` 的请求,批准最新的 `requestId`: + +```bash +openclaw devices approve +``` + +确认 node 已连接并 paired: + +```bash +openclaw nodes status +openclaw nodes describe --node "windows-node" +``` + +如果 `devices list` 每次看到新的 pairing `requestId`,通常说明 node 身份或认证信息变化了。优先检查: + +- Windows 上是否反复使用了不同的 `--node-id`。 +- Windows 上 `~/.openclaw/node.json` 是否被删除或重建。 +- Gateway token/password 是否变更。 +- Node 是否确实使用同一个 `OPENCLAW_GATEWAY_TOKEN` 启动。 + +## 5. 将 Exec 指向 Windows Node + +在服务端 OpenClaw 实例里配置默认 exec host: + +```bash +openclaw config set tools.exec.host node +openclaw config set tools.exec.node "windows-node" +``` + +如果有多个 Agent,也可以按 Agent 配置: + +```bash +openclaw config get agents.list +openclaw config set agents.list[0].tools.exec.node "windows-node" +``` + +## 6. Exec Approval 配置方案 + +### 方案 A:稳定自动化,免审批执行 + +仅建议在完全可信的内网或 VPN 环境中使用。 + +服务端 Gateway 请求策略: + +```bash +openclaw config set tools.exec.security full +openclaw config set tools.exec.ask off +openclaw gateway restart +``` + +同时把 Windows node 本地 approvals 策略也打开: + +```bash +openclaw approvals set --node "windows-node" --stdin <<'EOF' +{ + "version": 1, + "defaults": { + "security": "full", + "ask": "off", + "askFallback": "full" + } +} +EOF +``` + +注意:只改 `tools.exec.*` 不够。OpenClaw 会取 Gateway 请求策略和 node 本地 `~/.openclaw/exec-approvals.json` 中更严格的一方。如果 node 本地仍是 `ask: "always"` 或 `askFallback: "deny"`,还是会继续提示授权。 + +### 方案 B:更安全,使用 allowlist + +服务端 Gateway 请求策略: + +```bash +openclaw config set tools.exec.security allowlist +openclaw config set tools.exec.ask off +openclaw gateway restart +``` + +给 Windows node 添加允许执行的命令: + +```bash +openclaw approvals allowlist add --agent "*" --node "windows-node" "powershell.exe" +openclaw approvals allowlist add --agent "*" --node "windows-node" "pwsh.exe" +openclaw approvals allowlist add --agent "*" --node "windows-node" "cmd.exe" +``` + +更推荐只放行具体脚本或具体工具,例如: + +```bash +openclaw approvals allowlist add --agent "*" --node "windows-node" "C:\\Tools\\automation\\daily-task.ps1" +openclaw approvals allowlist add --agent "*" --node "windows-node" "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe" +``` + +Windows 上如果命令被包装成 `cmd.exe /c ...`,allowlist 模式下仍可能需要审批。这是 OpenClaw 的安全行为。若需要完全无提示自动化,使用方案 A。 + +## 7. 验证 + +在服务端 OpenClaw 实例里执行: + +```bash +openclaw nodes status +openclaw nodes describe --node "windows-node" +openclaw approvals get --node "windows-node" +``` + +然后在 WebChat 里让 Agent 执行一个简单命令: + +```text +请在 node 上执行 hostname,并告诉我输出。 +``` + +如果仍然报 `SYSTEM_RUN_DENIED`: + +1. 确认 `openclaw nodes status` 中 node 是 connected/paired。 +2. 确认 `openclaw nodes describe --node "windows-node"` 中包含 `system.run`。 +3. 确认 `openclaw approvals get --node "windows-node"` 的 effective policy 是预期值。 +4. 如果 effective policy 仍然是 ask 或 deny,说明 node 本地 approvals 文件没有被更新。 +5. 如果是 allowlist miss,说明命令没有命中 allowlist,改成 full 或补 allowlist。 + +## 8. requestId 每次不同怎么处理 + +如果 requestId 来自 `devices list`: + +- 这是 node pairing 请求。 +- 应该只需要批准一次。 +- 如果每次都变,说明 node 身份被重置或认证信息变化。 + +如果 requestId 来自 exec approval: + +- 这是命令执行审批。 +- 每条新命令都有自己的 requestId,变化是正常现象。 +- 要避免反复授权,需要配置 `tools.exec.ask off`,并同步配置 node 本地 approvals。 + +## 9. 安全建议 + +- 不要把 Gateway WebSocket 端口直接暴露公网。 +- 生产环境优先使用 VPN、Tailscale、WireGuard 或 SSH tunnel。 +- 如果只是做固定自动化,优先使用 allowlist 放行具体脚本。 +- `security full + ask off` 相当于允许 WebChat 控制 Windows 本机命令,只有在完全可信环境使用。 +- 定期检查 Windows node 的 `~/.openclaw/exec-approvals.json`。 +- 不再使用时,及时停掉 node: + +```powershell +openclaw node stop +openclaw node uninstall +``` diff --git a/docs/superpowers/plans/2026-06-01-clawmanager-v2-runtime-pool.md b/docs/superpowers/plans/2026-06-01-clawmanager-v2-runtime-pool.md new file mode 100644 index 0000000..2ff15ef --- /dev/null +++ b/docs/superpowers/plans/2026-06-01-clawmanager-v2-runtime-pool.md @@ -0,0 +1,3080 @@ +# ClawManager V2 Runtime Pool Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build ClawManager V2 so OpenClaw and Hermes instances run as gateway processes inside shared runtime Pods, with workspace isolation, fast failover, admin runtime observability, and a simpler user UI. + +**Architecture:** The backend remains the source of truth in MySQL, uses Redis for hot locks/events/cache, and runs an embedded leader-elected runtime scheduler across multiple backend replicas. Runtime Pods are managed by Kubernetes Deployments and `clawmanager-agent`; each runtime Pod hosts up to 100 gateways and exposes gateway traffic by Pod IP plus assigned port. Workspaces live under a shared `/workspaces` mount and are accessed through a dedicated backend file manager API, not through the frame. + +**Tech Stack:** Go 1.26, Gin, MySQL via `upper/db`, Redis RESP client, Kubernetes client-go, React 19, Vite, TypeScript, Tailwind CSS, Kubernetes YAML. + +--- + +## Scope Check + +This release touches several subsystems, but they are tightly ordered: schema first, scheduler second, lifecycle/proxy/files third, UI and manifests last. Keep the work in this single master plan, but commit after each task so any subsystem can be reviewed independently. + +## File Structure + +Backend data and migrations: +- Modify: `backend/internal/db/migrations.go` - existing embedded migration runner remains unchanged unless ordering assumptions fail in tests. +- Create: `backend/internal/db/migrations/023_add_runtime_pool_v2.sql` - runtime Pod, gateway binding, rollout, and workspace audit schema. +- Modify: `backend/internal/db/migrations_test.go` - verify migration file ordering and SQL splitting still handles migration 023. +- Modify: `backend/internal/models/instance.go` - add workspace fields and V2 runtime metadata. +- Create: `backend/internal/models/runtime_pod.go` - runtime Pod state and metrics model. +- Create: `backend/internal/models/instance_runtime_binding.go` - active gateway binding model. +- Create: `backend/internal/models/runtime_rollout.go` - rollout state model for gray upgrades. +- Create: `backend/internal/models/workspace_file_audit.go` - upload/download/write audit model. +- Modify: `backend/internal/repository/instance_repository.go` - V2 lifecycle selectors and status update helpers. +- Create: `backend/internal/repository/runtime_pod_repository.go` - runtime Pod CRUD and capacity claims. +- Create: `backend/internal/repository/instance_runtime_binding_repository.go` - gateway binding CRUD and generation-safe updates. +- Create: `backend/internal/repository/runtime_rollout_repository.go` - rollout CRUD and selectors. +- Create: `backend/internal/repository/workspace_file_audit_repository.go` - append-only file audit writes. + +Backend runtime scheduling and agent integration: +- Create: `docs/clawmanager-agent-v2-contract.md` - exact HTTP dual-channel contract required from `clawmanager-agent`. +- Create: `backend/internal/services/redis_client.go` - platform Redis client reusing the existing RESP approach. +- Create: `backend/internal/services/runtime_agent_client.go` - backend-to-agent command client. +- Create: `backend/internal/services/runtime_scheduler.go` - leader loop, assignment, failover, and gray-drain orchestration. +- Create: `backend/internal/services/runtime_capacity.go` - capacity calculations and runtime type normalization. +- Create: `backend/internal/services/runtime_events.go` - Redis-backed runtime event fanout for all backend replicas. +- Create: `backend/internal/services/runtime_leader.go` - Kubernetes Lease leader election wrapper. +- Create: `backend/internal/services/k8s/runtime_deployment_service.go` - create, scale, and inspect OpenClaw/Hermes runtime Deployments. +- Modify: `backend/internal/services/instance_service.go` - route V2 create/start/stop/delete through scheduler state; keep legacy behavior for old instance types. +- Modify: `backend/internal/services/instance_proxy_service.go` - proxy V2 instances to binding Pod IP plus gateway port; keep legacy service proxy fallback. +- Modify: `backend/internal/services/sync_service.go` - stop assuming every running instance owns a Pod. +- Modify: `backend/cmd/server/main.go` - wire repositories, services, scheduler start, routes, and websocket runtime event bridge. + +Backend workspace file APIs: +- Create: `backend/internal/services/workspace_path_guard.go` - path clean, realpath, and symlink escape prevention. +- Create: `backend/internal/services/workspace_file_service.go` - list, preview, download, upload, mkdir, rename, delete. +- Create: `backend/internal/handlers/workspace_file_handler.go` - user-facing workspace file endpoints. +- Modify: `backend/internal/handlers/instance_handler.go` - attach workspace routes and simplify status response for V2. +- Modify: `backend/internal/utils/response.go` - add V2 file validation errors to client-safe bad request mapping. + +Backend admin APIs: +- Create: `backend/internal/handlers/runtime_pool_handler.go` - admin runtime Pod status, metrics, drain, and rollout endpoints. +- Modify: `backend/internal/services/websocket_service.go` - add admin runtime event subscriptions with user-role checks. +- Modify: `backend/internal/handlers/websocket_handler.go` if it exists in this repository; otherwise wire websocket changes in the existing handler file discovered during implementation. + +Frontend user and admin experience: +- Modify: `frontend/package.json` and lock file - add `lucide-react` if icons are not already available. +- Modify: `frontend/src/index.css` - neutral, simple design tokens; remove heavy gradients and oversized radii. +- Modify: `frontend/src/components/UserLayout.tsx` - simpler user shell. +- Modify: `frontend/src/components/AdminLayout.tsx` - simpler admin shell and add Runtime Pods navigation. +- Modify: `frontend/src/types/instance.ts` - V2 instance availability and workspace types. +- Create: `frontend/src/types/runtimePool.ts` - admin runtime Pod, gateway, metrics, rollout types. +- Create: `frontend/src/services/workspaceService.ts` - workspace API client. +- Create: `frontend/src/services/runtimePoolService.ts` - admin runtime API client. +- Modify: `frontend/src/hooks/useWebSocket.ts` - add admin runtime event hook. +- Modify: `frontend/src/services/instanceService.ts` - V2 status fields and workspace entry points. +- Modify: `frontend/src/pages/instances/CreateInstancePage.tsx` - only allow creating OpenClaw and Hermes for V2. +- Modify: `frontend/src/pages/instances/InstanceListPage.tsx` - hide Pod/service/resource details from ordinary users. +- Modify: `frontend/src/pages/instances/InstanceDetailPage.tsx` - show availability, service frame, workspace file manager, and simple controls. +- Create: `frontend/src/components/WorkspaceFileManager.tsx` - independent file list, preview, upload, download, rename, delete. +- Create: `frontend/src/components/InstanceServiceFrame.tsx` - OpenClaw/Hermes iframe wrapper with availability state. +- Create: `frontend/src/pages/admin/RuntimePodsPage.tsx` - admin-only runtime Pod metrics and gateway usage. +- Modify: `frontend/src/pages/admin/SystemSettingsPage.tsx` - add OpenClaw/Hermes runtime image and gray rollout controls. +- Modify: `frontend/src/router/index.tsx` - route `/admin/runtime-pods`. + +Deployment: +- Modify: `deployments/k8s/clawmanager.yaml` - one-YAML install for Redis, workspace store, backend replicas, runtime Deployments, RBAC, env. +- Modify: `deployments/k3s/clawmanager.yaml` - same defaults adapted to k3s/local path storage. +- Modify: `backend/deployments/k8s/clawreef-incluster.yaml` if this file is still shipped as an install path; otherwise leave it unchanged and document that `deployments/k8s/clawmanager.yaml` is canonical. + +## Agent Contract Summary + +The `clawmanager-agent` changes are documented but implemented outside this repository. The backend implementation must assume these endpoints and payloads: + +Agent-to-backend channel: + +```http +POST /api/v1/runtime-agent/register +POST /api/v1/runtime-agent/heartbeat +POST /api/v1/runtime-agent/gateways/report +POST /api/v1/runtime-agent/skills/report +POST /api/v1/runtime-agent/metrics/report +``` + +Backend-to-agent channel: + +```http +GET http://{pod_ip}:19090/v1/health +GET http://{pod_ip}:19090/v1/metrics +POST http://{pod_ip}:19090/v1/gateways +DELETE http://{pod_ip}:19090/v1/gateways/{gateway_id} +POST http://{pod_ip}:19090/v1/gateways/{gateway_id}/health +POST http://{pod_ip}:19090/v1/drain +``` + +Gateway create request: + +```json +{ + "instance_id": 123, + "user_id": 45, + "agent_type": "openclaw", + "workspace_path": "/workspaces/openclaw/user-45/instance-123", + "port_range": { "start": 20000, "end": 20099 }, + "uid": 200123, + "gid": 200123, + "cpu_cores": 2, + "memory_mb": 4096, + "disk_quota_mb": 20480, + "generation": 7 +} +``` + +Gateway create response: + +```json +{ + "gateway_id": "gw-123-7", + "port": 20017, + "pid": 8842, + "status": "running" +} +``` + +The agent must allocate a free port inside `20000-20099`, enforce one Linux UID/GID per instance using `200000 + instance_id`, apply cgroup CPU/memory limits, enforce workspace quota, reject symlink escapes inside workspaces, report gateway health, and survive backend replica changes. During gray upgrade, `POST /v1/drain` must reject new gateways and continue reporting existing gateway health until ClawManager migrates them away. + +## Task 1: Schema, Models, And Repository Interfaces + +**Files:** +- Create: `backend/internal/db/migrations/023_add_runtime_pool_v2.sql` +- Modify: `backend/internal/db/migrations_test.go` +- Modify: `backend/internal/models/instance.go` +- Create: `backend/internal/models/runtime_pod.go` +- Create: `backend/internal/models/instance_runtime_binding.go` +- Create: `backend/internal/models/runtime_rollout.go` +- Create: `backend/internal/models/workspace_file_audit.go` +- Modify: `backend/internal/repository/instance_repository.go` +- Create: `backend/internal/repository/runtime_pod_repository.go` +- Create: `backend/internal/repository/instance_runtime_binding_repository.go` +- Create: `backend/internal/repository/runtime_rollout_repository.go` +- Create: `backend/internal/repository/workspace_file_audit_repository.go` + +- [ ] **Step 1: Add failing migration order test** + +Append this test to `backend/internal/db/migrations_test.go`: + +```go +func TestMigration023IsEmbedded(t *testing.T) { + files, err := migrationFiles.ReadDir("migrations") + if err != nil { + t.Fatalf("read embedded migrations: %v", err) + } + + found := false + for _, file := range files { + if file.Name() == "023_add_runtime_pool_v2.sql" { + found = true + break + } + } + if !found { + t.Fatalf("migration 023_add_runtime_pool_v2.sql is not embedded") + } +} +``` + +- [ ] **Step 2: Run migration test and verify it fails** + +Run: + +```powershell +go test ./backend/internal/db -run TestMigration023IsEmbedded -count=1 +``` + +Expected: fail with `migration 023_add_runtime_pool_v2.sql is not embedded`. + +- [ ] **Step 3: Create migration 023** + +Create `backend/internal/db/migrations/023_add_runtime_pool_v2.sql` with: + +```sql +ALTER TABLE instances + ADD COLUMN workspace_path VARCHAR(1024) NULL AFTER mount_path, + ADD COLUMN workspace_usage_bytes BIGINT NOT NULL DEFAULT 0 AFTER workspace_path, + ADD COLUMN runtime_generation INT NOT NULL DEFAULT 1 AFTER workspace_usage_bytes, + ADD COLUMN runtime_error_message TEXT NULL AFTER runtime_generation; + +CREATE TABLE runtime_pods ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + runtime_type VARCHAR(32) NOT NULL, + namespace VARCHAR(128) NOT NULL, + pod_name VARCHAR(255) NOT NULL, + pod_uid VARCHAR(128) NULL, + pod_ip VARCHAR(64) NULL, + node_name VARCHAR(255) NULL, + deployment_name VARCHAR(255) NOT NULL, + image_ref VARCHAR(512) NOT NULL, + agent_endpoint VARCHAR(255) NULL, + state VARCHAR(32) NOT NULL DEFAULT 'pending', + capacity INT NOT NULL DEFAULT 100, + used_slots INT NOT NULL DEFAULT 0, + draining TINYINT(1) NOT NULL DEFAULT 0, + cpu_millis_used BIGINT NOT NULL DEFAULT 0, + memory_bytes_used BIGINT NOT NULL DEFAULT 0, + disk_bytes_used BIGINT NOT NULL DEFAULT 0, + network_rx_bytes BIGINT NOT NULL DEFAULT 0, + network_tx_bytes BIGINT NOT NULL DEFAULT 0, + metrics_json JSON NULL, + last_seen_at DATETIME NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY uk_runtime_pod_namespace_name (namespace, pod_name), + KEY idx_runtime_pods_schedulable (runtime_type, state, draining, used_slots, capacity), + KEY idx_runtime_pods_last_seen (last_seen_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE instance_runtime_bindings ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + instance_id INT NOT NULL, + runtime_pod_id BIGINT NOT NULL, + runtime_type VARCHAR(32) NOT NULL, + gateway_id VARCHAR(128) NOT NULL, + gateway_port INT NOT NULL, + gateway_pid INT NULL, + workspace_path VARCHAR(1024) NOT NULL, + state VARCHAR(32) NOT NULL DEFAULT 'creating', + generation INT NOT NULL DEFAULT 1, + last_health_at DATETIME NULL, + error_message TEXT NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY uk_instance_runtime_binding_instance (instance_id), + UNIQUE KEY uk_instance_runtime_binding_gateway (runtime_pod_id, gateway_port), + KEY idx_instance_runtime_binding_pod_state (runtime_pod_id, state), + CONSTRAINT fk_instance_runtime_binding_instance + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE, + CONSTRAINT fk_instance_runtime_binding_pod + FOREIGN KEY (runtime_pod_id) REFERENCES runtime_pods(id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE runtime_rollouts ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + runtime_type VARCHAR(32) NOT NULL, + target_image_ref VARCHAR(512) NOT NULL, + status VARCHAR(32) NOT NULL DEFAULT 'pending', + batch_size INT NOT NULL DEFAULT 1, + max_unavailable INT NOT NULL DEFAULT 1, + started_by INT NULL, + started_at DATETIME NULL, + finished_at DATETIME NULL, + error_message TEXT NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + KEY idx_runtime_rollouts_type_status (runtime_type, status) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE workspace_file_audits ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + instance_id INT NOT NULL, + user_id INT NOT NULL, + action VARCHAR(32) NOT NULL, + relative_path VARCHAR(1024) NOT NULL, + bytes BIGINT NOT NULL DEFAULT 0, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + KEY idx_workspace_file_audits_instance_time (instance_id, created_at), + KEY idx_workspace_file_audits_user_time (user_id, created_at), + CONSTRAINT fk_workspace_file_audits_instance + FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +``` + +- [ ] **Step 4: Extend instance model** + +Add fields to `backend/internal/models/instance.go` after `MountPath` and after `StoppedAt`: + +```go + WorkspacePath *string `db:"workspace_path" json:"workspace_path,omitempty"` + WorkspaceUsageBytes int64 `db:"workspace_usage_bytes" json:"workspace_usage_bytes"` + RuntimeGeneration int `db:"runtime_generation" json:"runtime_generation"` + RuntimeErrorMessage *string `db:"runtime_error_message" json:"runtime_error_message,omitempty"` +``` + +- [ ] **Step 5: Add runtime models** + +Create `backend/internal/models/runtime_pod.go`: + +```go +package models + +import "time" + +type RuntimePod struct { + ID int64 `db:"id,primarykey,autoincrement" json:"id"` + RuntimeType string `db:"runtime_type" json:"runtime_type"` + Namespace string `db:"namespace" json:"namespace"` + PodName string `db:"pod_name" json:"pod_name"` + PodUID *string `db:"pod_uid" json:"pod_uid,omitempty"` + PodIP *string `db:"pod_ip" json:"pod_ip,omitempty"` + NodeName *string `db:"node_name" json:"node_name,omitempty"` + DeploymentName string `db:"deployment_name" json:"deployment_name"` + ImageRef string `db:"image_ref" json:"image_ref"` + AgentEndpoint *string `db:"agent_endpoint" json:"agent_endpoint,omitempty"` + State string `db:"state" json:"state"` + Capacity int `db:"capacity" json:"capacity"` + UsedSlots int `db:"used_slots" json:"used_slots"` + Draining bool `db:"draining" json:"draining"` + CPUMillisUsed int64 `db:"cpu_millis_used" json:"cpu_millis_used"` + MemoryBytesUsed int64 `db:"memory_bytes_used" json:"memory_bytes_used"` + DiskBytesUsed int64 `db:"disk_bytes_used" json:"disk_bytes_used"` + NetworkRXBytes int64 `db:"network_rx_bytes" json:"network_rx_bytes"` + NetworkTXBytes int64 `db:"network_tx_bytes" json:"network_tx_bytes"` + MetricsJSON *string `db:"metrics_json" json:"metrics_json,omitempty"` + LastSeenAt *time.Time `db:"last_seen_at" json:"last_seen_at,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (RuntimePod) TableName() string { + return "runtime_pods" +} +``` + +Create `backend/internal/models/instance_runtime_binding.go`: + +```go +package models + +import "time" + +type InstanceRuntimeBinding struct { + ID int64 `db:"id,primarykey,autoincrement" json:"id"` + InstanceID int `db:"instance_id" json:"instance_id"` + RuntimePodID int64 `db:"runtime_pod_id" json:"runtime_pod_id"` + RuntimeType string `db:"runtime_type" json:"runtime_type"` + GatewayID string `db:"gateway_id" json:"gateway_id"` + GatewayPort int `db:"gateway_port" json:"gateway_port"` + GatewayPID *int `db:"gateway_pid" json:"gateway_pid,omitempty"` + WorkspacePath string `db:"workspace_path" json:"workspace_path"` + State string `db:"state" json:"state"` + Generation int `db:"generation" json:"generation"` + LastHealthAt *time.Time `db:"last_health_at" json:"last_health_at,omitempty"` + ErrorMessage *string `db:"error_message" json:"error_message,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (InstanceRuntimeBinding) TableName() string { + return "instance_runtime_bindings" +} +``` + +Create `backend/internal/models/runtime_rollout.go`: + +```go +package models + +import "time" + +type RuntimeRollout struct { + ID int64 `db:"id,primarykey,autoincrement" json:"id"` + RuntimeType string `db:"runtime_type" json:"runtime_type"` + TargetImageRef string `db:"target_image_ref" json:"target_image_ref"` + Status string `db:"status" json:"status"` + BatchSize int `db:"batch_size" json:"batch_size"` + MaxUnavailable int `db:"max_unavailable" json:"max_unavailable"` + StartedBy *int `db:"started_by" json:"started_by,omitempty"` + StartedAt *time.Time `db:"started_at" json:"started_at,omitempty"` + FinishedAt *time.Time `db:"finished_at" json:"finished_at,omitempty"` + ErrorMessage *string `db:"error_message" json:"error_message,omitempty"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` +} + +func (RuntimeRollout) TableName() string { + return "runtime_rollouts" +} +``` + +Create `backend/internal/models/workspace_file_audit.go`: + +```go +package models + +import "time" + +type WorkspaceFileAudit struct { + ID int64 `db:"id,primarykey,autoincrement" json:"id"` + InstanceID int `db:"instance_id" json:"instance_id"` + UserID int `db:"user_id" json:"user_id"` + Action string `db:"action" json:"action"` + RelativePath string `db:"relative_path" json:"relative_path"` + Bytes int64 `db:"bytes" json:"bytes"` + CreatedAt time.Time `db:"created_at" json:"created_at"` +} + +func (WorkspaceFileAudit) TableName() string { + return "workspace_file_audits" +} +``` + +- [ ] **Step 6: Add repository interfaces and implementations** + +Implement interfaces with these method signatures. Use `upper/db` collections for normal CRUD and raw SQL for capacity claims requiring atomic increments. + +`backend/internal/repository/runtime_pod_repository.go`: + +```go +package repository + +import ( + "context" + "fmt" + "time" + + "clawreef/internal/models" + "github.com/upper/db/v4" +) + +type RuntimePodRepository interface { + UpsertFromAgent(ctx context.Context, pod *models.RuntimePod) error + GetByID(ctx context.Context, id int64) (*models.RuntimePod, error) + GetByNamespaceName(ctx context.Context, namespace, podName string) (*models.RuntimePod, error) + List(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) + ListSchedulable(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) + TryClaimSlot(ctx context.Context, podID int64) (bool, error) + ReleaseSlot(ctx context.Context, podID int64) error + MarkState(ctx context.Context, podID int64, state string, draining bool) error + MarkUnseenUnhealthy(ctx context.Context, cutoff time.Time) error + UpdateMetrics(ctx context.Context, podID int64, metrics RuntimePodMetricsUpdate) error +} + +type RuntimePodMetricsUpdate struct { + CPUMillisUsed int64 + MemoryBytesUsed int64 + DiskBytesUsed int64 + NetworkRXBytes int64 + NetworkTXBytes int64 + MetricsJSON string + LastSeenAt time.Time +} + +type runtimePodRepository struct { + sess db.Session +} + +func NewRuntimePodRepository(sess db.Session) RuntimePodRepository { + return &runtimePodRepository{sess: sess} +} + +func (r *runtimePodRepository) collection() db.Collection { + return r.sess.Collection("runtime_pods") +} +``` + +Then add each method in the same file. The `TryClaimSlot` implementation must use this SQL shape: + +```go +func (r *runtimePodRepository) TryClaimSlot(ctx context.Context, podID int64) (bool, error) { + res, err := r.sess.SQL(). + ExecContext(ctx, `UPDATE runtime_pods + SET used_slots = used_slots + 1 + WHERE id = ? AND state = 'ready' AND draining = 0 AND used_slots < capacity`, podID) + if err != nil { + return false, err + } + affected, err := res.RowsAffected() + if err != nil { + return false, err + } + return affected == 1, nil +} +``` + +`backend/internal/repository/instance_runtime_binding_repository.go` must include: + +```go +type InstanceRuntimeBindingRepository interface { + Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error + GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) + GetRunningByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) + ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error) + ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error) + UpdateRunning(ctx context.Context, instanceID int, generation int, gatewayID string, port int, pid *int) error + UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error + DeleteByInstanceID(ctx context.Context, instanceID int) error +} +``` + +Extend `backend/internal/repository/instance_repository.go` with: + +```go + GetV2DesiredRunning(ctx context.Context, limit int) ([]models.Instance, error) + GetV2Creating(ctx context.Context, limit int) ([]models.Instance, error) + UpdateRuntimeState(ctx context.Context, id int, status string, generation int, message *string) error + SetWorkspacePath(ctx context.Context, id int, workspacePath string) error + UpdateWorkspaceUsage(ctx context.Context, id int, usageBytes int64) error +``` + +- [ ] **Step 7: Run backend tests for schema compilation** + +Run: + +```powershell +go test ./backend/internal/db ./backend/internal/models ./backend/internal/repository -count=1 +``` + +Expected: pass, or report packages with no test files and no compile errors. + +- [ ] **Step 8: Commit schema and repository layer** + +Run: + +```powershell +git add backend/internal/db backend/internal/models backend/internal/repository +git commit -m "feat: add runtime pool schema" +``` + +## Task 2: Agent Contract Documentation And HTTP Client + +**Files:** +- Create: `docs/clawmanager-agent-v2-contract.md` +- Create: `backend/internal/services/runtime_agent_client.go` +- Create: `backend/internal/services/runtime_agent_client_test.go` + +- [ ] **Step 1: Write the agent contract document** + +Create `docs/clawmanager-agent-v2-contract.md` with these sections: + +```markdown +# clawmanager-agent V2 Contract + +## Purpose + +The agent runs inside each shared OpenClaw or Hermes runtime Pod. It manages gateway subprocesses, ports, cgroups, Linux users, workspace quota, health checks, skill/status reporting, and metrics. + +## Agent-To-Backend Reports + +All report requests use header `X-ClawManager-Agent-Token`. + +### POST /api/v1/runtime-agent/register + +Request body: + +```json +{ + "runtime_type": "openclaw", + "namespace": "clawmanager-system", + "pod_name": "openclaw-runtime-6f77f8b8c7-abcde", + "pod_uid": "pod-uid", + "pod_ip": "10.42.0.31", + "node_name": "node-a", + "deployment_name": "openclaw-runtime", + "image_ref": "ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest", + "agent_endpoint": "http://10.42.0.31:19090", + "capacity": 100 +} +``` + +### POST /api/v1/runtime-agent/heartbeat + +Request body: + +```json +{ + "pod_name": "openclaw-runtime-6f77f8b8c7-abcde", + "runtime_type": "openclaw", + "state": "ready", + "used_slots": 37, + "draining": false +} +``` + +### POST /api/v1/runtime-agent/metrics/report + +Request body: + +```json +{ + "pod_name": "openclaw-runtime-6f77f8b8c7-abcde", + "cpu_millis_used": 13600, + "memory_bytes_used": 42949672960, + "disk_bytes_used": 214748364800, + "network_rx_bytes": 9223372, + "network_tx_bytes": 19223372, + "gateways": [ + { + "instance_id": 123, + "gateway_id": "gw-123-7", + "port": 20017, + "state": "running", + "last_health_at": "2026-06-01T10:00:00Z" + } + ] +} +``` + +## Backend-To-Agent Commands + +All command requests use header `X-ClawManager-Control-Token`. + +### POST /v1/gateways + +The agent creates a gateway subprocess and returns the bound port. If no port is free in the requested range, return HTTP 409. + +### DELETE /v1/gateways/{gateway_id} + +The agent stops the gateway and releases cgroup, process, and port resources. + +### POST /v1/drain + +The agent marks the Pod as draining and rejects new gateway creation. Existing gateways continue until ClawManager migrates them. + +## Isolation Requirements + +Every gateway runs as UID/GID `200000 + instance_id`. The agent rejects workspace paths outside `/workspaces/{runtime}/user-{user_id}/instance-{instance_id}` after resolving symlinks. CPU and memory limits are enforced with cgroups. Workspace quota is enforced before writes. +``` + +- [ ] **Step 2: Write failing client tests** + +Create `backend/internal/services/runtime_agent_client_test.go`: + +```go +package services + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestRuntimeAgentClientCreateGateway(t *testing.T) { + var gotToken string + var gotReq RuntimeAgentCreateGatewayRequest + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/gateways" { + t.Fatalf("unexpected path %s", r.URL.Path) + } + gotToken = r.Header.Get("X-ClawManager-Control-Token") + if err := json.NewDecoder(r.Body).Decode(&gotReq); err != nil { + t.Fatalf("decode request: %v", err) + } + _ = json.NewEncoder(w).Encode(RuntimeAgentCreateGatewayResponse{ + GatewayID: "gw-7-3", + Port: 20017, + PID: intPtr(8842), + Status: "running", + }) + })) + defer server.Close() + + client := NewRuntimeAgentClient("secret") + resp, err := client.CreateGateway(context.Background(), server.URL, RuntimeAgentCreateGatewayRequest{ + InstanceID: 7, + UserID: 8, + AgentType: "openclaw", + WorkspacePath: "/workspaces/openclaw/user-8/instance-7", + PortRange: RuntimeAgentPortRange{Start: 20000, End: 20099}, + UID: 200007, + GID: 200007, + CPUCores: 2, + MemoryMB: 4096, + DiskQuotaMB: 20480, + Generation: 3, + }) + if err != nil { + t.Fatalf("CreateGateway returned error: %v", err) + } + if gotToken != "secret" { + t.Fatalf("unexpected token %q", gotToken) + } + if gotReq.InstanceID != 7 || gotReq.PortRange.Start != 20000 { + t.Fatalf("unexpected request %#v", gotReq) + } + if resp.GatewayID != "gw-7-3" || resp.Port != 20017 { + t.Fatalf("unexpected response %#v", resp) + } +} + +func TestRuntimeAgentClientConflict(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "no free port", http.StatusConflict) + })) + defer server.Close() + + client := NewRuntimeAgentClient("secret") + _, err := client.CreateGateway(context.Background(), server.URL, RuntimeAgentCreateGatewayRequest{}) + if err == nil || err.Error() != "runtime agent conflict: no free port\n" { + t.Fatalf("unexpected error %v", err) + } +} + +func intPtr(v int) *int { + return &v +} +``` + +- [ ] **Step 3: Run client tests and verify they fail** + +Run: + +```powershell +go test ./backend/internal/services -run RuntimeAgentClient -count=1 +``` + +Expected: fail because `RuntimeAgentCreateGatewayRequest` and `NewRuntimeAgentClient` do not exist. + +- [ ] **Step 4: Implement runtime agent client** + +Create `backend/internal/services/runtime_agent_client.go`: + +```go +package services + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +type RuntimeAgentClient interface { + Health(ctx context.Context, endpoint string) error + CreateGateway(ctx context.Context, endpoint string, req RuntimeAgentCreateGatewayRequest) (*RuntimeAgentCreateGatewayResponse, error) + DeleteGateway(ctx context.Context, endpoint, gatewayID string) error + Drain(ctx context.Context, endpoint string) error +} + +type RuntimeAgentPortRange struct { + Start int `json:"start"` + End int `json:"end"` +} + +type RuntimeAgentCreateGatewayRequest struct { + InstanceID int `json:"instance_id"` + UserID int `json:"user_id"` + AgentType string `json:"agent_type"` + WorkspacePath string `json:"workspace_path"` + PortRange RuntimeAgentPortRange `json:"port_range"` + UID int `json:"uid"` + GID int `json:"gid"` + CPUCores float64 `json:"cpu_cores"` + MemoryMB int `json:"memory_mb"` + DiskQuotaMB int `json:"disk_quota_mb"` + Generation int `json:"generation"` +} + +type RuntimeAgentCreateGatewayResponse struct { + GatewayID string `json:"gateway_id"` + Port int `json:"port"` + PID *int `json:"pid,omitempty"` + Status string `json:"status"` +} + +type runtimeAgentHTTPClient struct { + controlToken string + httpClient *http.Client +} + +func NewRuntimeAgentClient(controlToken string) RuntimeAgentClient { + return &runtimeAgentHTTPClient{ + controlToken: controlToken, + httpClient: &http.Client{ + Timeout: 5 * time.Second, + }, + } +} + +func (c *runtimeAgentHTTPClient) Health(ctx context.Context, endpoint string) error { + return c.do(ctx, http.MethodGet, endpoint, "/v1/health", nil, nil) +} + +func (c *runtimeAgentHTTPClient) CreateGateway(ctx context.Context, endpoint string, req RuntimeAgentCreateGatewayRequest) (*RuntimeAgentCreateGatewayResponse, error) { + var resp RuntimeAgentCreateGatewayResponse + if err := c.do(ctx, http.MethodPost, endpoint, "/v1/gateways", req, &resp); err != nil { + return nil, err + } + return &resp, nil +} + +func (c *runtimeAgentHTTPClient) DeleteGateway(ctx context.Context, endpoint, gatewayID string) error { + return c.do(ctx, http.MethodDelete, endpoint, "/v1/gateways/"+gatewayID, nil, nil) +} + +func (c *runtimeAgentHTTPClient) Drain(ctx context.Context, endpoint string) error { + return c.do(ctx, http.MethodPost, endpoint, "/v1/drain", map[string]bool{"draining": true}, nil) +} + +func (c *runtimeAgentHTTPClient) do(ctx context.Context, method, endpoint, path string, body any, out any) error { + endpoint = strings.TrimRight(endpoint, "/") + var reader io.Reader + if body != nil { + payload, err := json.Marshal(body) + if err != nil { + return err + } + reader = bytes.NewReader(payload) + } + req, err := http.NewRequestWithContext(ctx, method, endpoint+path, reader) + if err != nil { + return err + } + req.Header.Set("X-ClawManager-Control-Token", c.controlToken) + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + resp, err := c.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + if resp.StatusCode == http.StatusConflict { + return fmt.Errorf("runtime agent conflict: %s", string(msg)) + } + return fmt.Errorf("runtime agent status %d: %s", resp.StatusCode, string(msg)) + } + if out == nil { + return nil + } + return json.NewDecoder(resp.Body).Decode(out) +} +``` + +- [ ] **Step 5: Run client tests** + +Run: + +```powershell +go test ./backend/internal/services -run RuntimeAgentClient -count=1 +``` + +Expected: pass. + +- [ ] **Step 6: Commit agent contract and client** + +Run: + +```powershell +git add docs/clawmanager-agent-v2-contract.md backend/internal/services/runtime_agent_client.go backend/internal/services/runtime_agent_client_test.go +git commit -m "feat: add runtime agent contract" +``` + +## Task 3: Runtime Capacity, Kubernetes Runtime Deployments, And Leader Election + +**Files:** +- Create: `backend/internal/services/runtime_capacity.go` +- Create: `backend/internal/services/runtime_capacity_test.go` +- Create: `backend/internal/services/runtime_leader.go` +- Create: `backend/internal/services/k8s/runtime_deployment_service.go` +- Create: `backend/internal/services/k8s/runtime_deployment_service_test.go` +- Modify: `backend/internal/config/config.go` + +- [ ] **Step 1: Write runtime capacity tests** + +Create `backend/internal/services/runtime_capacity_test.go`: + +```go +package services + +import "testing" + +func TestNormalizeV2RuntimeType(t *testing.T) { + tests := map[string]string{ + "openclaw": "openclaw", + "hermes": "hermes", + "webtop": "", + "ubuntu": "", + } + for input, want := range tests { + got, ok := NormalizeV2RuntimeType(input) + if want == "" { + if ok { + t.Fatalf("expected %s to be rejected", input) + } + continue + } + if !ok || got != want { + t.Fatalf("NormalizeV2RuntimeType(%q) = %q, %v", input, got, ok) + } + } +} + +func TestRuntimeWorkspacePath(t *testing.T) { + got := RuntimeWorkspacePath("openclaw", 45, 123) + want := "/workspaces/openclaw/user-45/instance-123" + if got != want { + t.Fatalf("workspace path mismatch: want %s got %s", want, got) + } +} + +func TestRuntimeLinuxID(t *testing.T) { + got := RuntimeLinuxID(123) + if got != 200123 { + t.Fatalf("runtime linux id mismatch: %d", got) + } +} +``` + +- [ ] **Step 2: Implement capacity helpers** + +Create `backend/internal/services/runtime_capacity.go`: + +```go +package services + +import "fmt" + +const ( + RuntimeTypeOpenClaw = "openclaw" + RuntimeTypeHermes = "hermes" + + RuntimeGatewayPortStart = 20000 + RuntimeGatewayPortEnd = 20099 + RuntimePodCapacity = 100 + RuntimeLinuxIDBase = 200000 +) + +func NormalizeV2RuntimeType(instanceType string) (string, bool) { + switch instanceType { + case RuntimeTypeOpenClaw: + return RuntimeTypeOpenClaw, true + case RuntimeTypeHermes: + return RuntimeTypeHermes, true + default: + return "", false + } +} + +func RuntimeWorkspacePath(runtimeType string, userID int, instanceID int) string { + return fmt.Sprintf("/workspaces/%s/user-%d/instance-%d", runtimeType, userID, instanceID) +} + +func RuntimeLinuxID(instanceID int) int { + return RuntimeLinuxIDBase + instanceID +} +``` + +- [ ] **Step 3: Run capacity tests** + +Run: + +```powershell +go test ./backend/internal/services -run 'NormalizeV2RuntimeType|RuntimeWorkspacePath|RuntimeLinuxID' -count=1 +``` + +Expected: pass. + +- [ ] **Step 4: Add runtime config fields** + +Modify `backend/internal/config/config.go` by adding fields to the main config struct: + +```go + Runtime RuntimeConfig +``` + +Add this struct and env parsing: + +```go +type RuntimeConfig struct { + Namespace string + WorkspaceRoot string + AgentControlToken string + AgentReportToken string + BackendReplicaID string + RedisURL string + SchedulerEnabled bool + HeartbeatTimeout time.Duration + SchedulerTick time.Duration + OpenClawImage string + HermesImage string + MaxGatewaysPerPod int + GatewayPortStart int + GatewayPortEnd int +} +``` + +Parse with defaults: + +```go +Runtime: RuntimeConfig{ + Namespace: getEnv("RUNTIME_NAMESPACE", getEnv("K8S_NAMESPACE", "clawmanager-system")), + WorkspaceRoot: getEnv("RUNTIME_WORKSPACE_ROOT", "/workspaces"), + AgentControlToken: getEnv("RUNTIME_AGENT_CONTROL_TOKEN", ""), + AgentReportToken: getEnv("RUNTIME_AGENT_REPORT_TOKEN", ""), + BackendReplicaID: getEnv("HOSTNAME", "clawmanager-backend-local"), + RedisURL: getEnv("PLATFORM_REDIS_URL", getEnv("TEAM_REDIS_URL", "")), + SchedulerEnabled: getEnvBool("RUNTIME_SCHEDULER_ENABLED", true), + HeartbeatTimeout: getEnvDuration("RUNTIME_HEARTBEAT_TIMEOUT", 10*time.Second), + SchedulerTick: getEnvDuration("RUNTIME_SCHEDULER_TICK", 2*time.Second), + OpenClawImage: getEnv("OPENCLAW_RUNTIME_IMAGE", "ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest"), + HermesImage: getEnv("HERMES_RUNTIME_IMAGE", "ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest"), + MaxGatewaysPerPod: getEnvInt("RUNTIME_MAX_GATEWAYS_PER_POD", RuntimePodCapacity), + GatewayPortStart: getEnvInt("RUNTIME_GATEWAY_PORT_START", RuntimeGatewayPortStart), + GatewayPortEnd: getEnvInt("RUNTIME_GATEWAY_PORT_END", RuntimeGatewayPortEnd), +}, +``` + +If `getEnvDuration`, `getEnvBool`, or `getEnvInt` is missing, add them in `config.go` using `strconv` and `time.ParseDuration`. + +- [ ] **Step 5: Add Kubernetes runtime deployment service test** + +Create `backend/internal/services/k8s/runtime_deployment_service_test.go` with a unit test that constructs the Deployment and checks the shared workspace mount: + +```go +package k8s + +import "testing" + +func TestBuildRuntimeDeploymentUsesSharedWorkspaceAndAgentPort(t *testing.T) { + dep := BuildRuntimeDeployment(RuntimeDeploymentSpec{ + Name: "openclaw-runtime", + Namespace: "clawmanager-system", + RuntimeType: "openclaw", + Image: "example/openclaw:latest", + Replicas: 2, + WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local", + WorkspaceNFSPath: "/exports/workspaces", + AgentControlToken: "control", + AgentReportToken: "report", + }) + + if dep.Name != "openclaw-runtime" || *dep.Spec.Replicas != 2 { + t.Fatalf("unexpected deployment identity %#v", dep) + } + container := dep.Spec.Template.Spec.Containers[0] + if container.Ports[0].ContainerPort != 19090 { + t.Fatalf("agent port missing: %#v", container.Ports) + } + foundWorkspace := false + for _, mount := range container.VolumeMounts { + if mount.Name == "workspaces" && mount.MountPath == "/workspaces" { + foundWorkspace = true + } + } + if !foundWorkspace { + t.Fatalf("workspace mount missing: %#v", container.VolumeMounts) + } +} +``` + +- [ ] **Step 6: Implement Kubernetes runtime deployment service** + +Create `backend/internal/services/k8s/runtime_deployment_service.go` with: + +```go +package k8s + +import ( + "context" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/client-go/kubernetes" +) + +type RuntimeDeploymentSpec struct { + Name string + Namespace string + RuntimeType string + Image string + Replicas int32 + WorkspaceNFSServer string + WorkspaceNFSPath string + AgentControlToken string + AgentReportToken string +} + +type RuntimeDeploymentService interface { + Ensure(ctx context.Context, spec RuntimeDeploymentSpec) error + Scale(ctx context.Context, namespace, name string, replicas int32) error +} + +type runtimeDeploymentService struct { + client kubernetes.Interface +} + +func NewRuntimeDeploymentService(client kubernetes.Interface) RuntimeDeploymentService { + return &runtimeDeploymentService{client: client} +} + +func int32Ptr(v int32) *int32 { + return &v +} + +func BuildRuntimeDeployment(spec RuntimeDeploymentSpec) *appsv1.Deployment { + labels := map[string]string{ + "app": spec.Name, + "clawmanager.io/runtime-type": spec.RuntimeType, + } + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: spec.Name, + Namespace: spec.Namespace, + Labels: labels, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: int32Ptr(spec.Replicas), + Selector: &metav1.LabelSelector{MatchLabels: labels}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "runtime", + Image: spec.Image, + Ports: []corev1.ContainerPort{{ + Name: "agent", + ContainerPort: 19090, + }}, + Env: []corev1.EnvVar{ + {Name: "CLAWMANAGER_RUNTIME_TYPE", Value: spec.RuntimeType}, + {Name: "CLAWMANAGER_AGENT_PORT", Value: "19090"}, + {Name: "CLAWMANAGER_GATEWAY_PORT_START", Value: "20000"}, + {Name: "CLAWMANAGER_GATEWAY_PORT_END", Value: "20099"}, + {Name: "CLAWMANAGER_AGENT_CONTROL_TOKEN", Value: spec.AgentControlToken}, + {Name: "CLAWMANAGER_AGENT_REPORT_TOKEN", Value: spec.AgentReportToken}, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("1Gi"), + }, + }, + VolumeMounts: []corev1.VolumeMount{{ + Name: "workspaces", + MountPath: "/workspaces", + }}, + }}, + Volumes: []corev1.Volume{{ + Name: "workspaces", + VolumeSource: corev1.VolumeSource{ + NFS: &corev1.NFSVolumeSource{ + Server: spec.WorkspaceNFSServer, + Path: spec.WorkspaceNFSPath, + }, + }, + }}, + }, + }, + }, + } +} +``` + +Add `Ensure` and `Scale` using AppsV1 Deployments `Get`, `Create`, `Update`, and `UpdateScale`. + +- [ ] **Step 7: Implement Lease leader service** + +Create `backend/internal/services/runtime_leader.go`: + +```go +package services + +import ( + "context" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +type RuntimeLeaderService interface { + IsLeader(ctx context.Context) bool +} + +type runtimeLeaderService struct { + client kubernetes.Interface + namespace string + name string + holder string + ttl time.Duration +} + +func NewRuntimeLeaderService(client kubernetes.Interface, namespace, holder string) RuntimeLeaderService { + return &runtimeLeaderService{ + client: client, + namespace: namespace, + name: "clawmanager-runtime-scheduler", + holder: holder, + ttl: 15 * time.Second, + } +} + +func (s *runtimeLeaderService) IsLeader(ctx context.Context) bool { + now := metav1.Now() + leaseClient := s.client.CoordinationV1().Leases(s.namespace) + lease, err := leaseClient.Get(ctx, s.name, metav1.GetOptions{}) + if err != nil { + lease = &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{Name: s.name, Namespace: s.namespace}, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: &s.holder, + RenewTime: &now, + LeaseDurationSeconds: int32Ptr32(int32(s.ttl.Seconds())), + }, + } + _, createErr := leaseClient.Create(ctx, lease, metav1.CreateOptions{}) + return createErr == nil + } + if lease.Spec.HolderIdentity != nil && *lease.Spec.HolderIdentity != s.holder && lease.Spec.RenewTime != nil { + expires := lease.Spec.RenewTime.Time.Add(s.ttl) + if time.Now().Before(expires) { + return false + } + } + lease.Spec.HolderIdentity = &s.holder + lease.Spec.RenewTime = &now + _, err = leaseClient.Update(ctx, lease, metav1.UpdateOptions{}) + return err == nil +} + +func int32Ptr32(v int32) *int32 { + return &v +} +``` + +- [ ] **Step 8: Run runtime deployment and leader tests** + +Run: + +```powershell +go test ./backend/internal/services ./backend/internal/services/k8s -run 'Runtime|Leader' -count=1 +``` + +Expected: pass. + +- [ ] **Step 9: Commit runtime capacity and Kubernetes services** + +Run: + +```powershell +git add backend/internal/config backend/internal/services backend/internal/services/k8s +git commit -m "feat: add runtime pool infrastructure services" +``` + +## Task 4: Runtime Scheduler And Redis Event Fanout + +**Files:** +- Create: `backend/internal/services/redis_client.go` +- Create: `backend/internal/services/runtime_events.go` +- Create: `backend/internal/services/runtime_scheduler.go` +- Create: `backend/internal/services/runtime_scheduler_test.go` +- Modify: `backend/cmd/server/main.go` + +- [ ] **Step 1: Extract a platform Redis client** + +Create `backend/internal/services/redis_client.go` by reusing the existing RESP parsing pattern from `team_redis.go`. The exported interface must be: + +```go +type PlatformRedisClient interface { + SetNX(ctx context.Context, key, value string, ttl time.Duration) (bool, error) + Del(ctx context.Context, key string) error + XAdd(ctx context.Context, key string, fields map[string]string) (string, error) + XRead(ctx context.Context, key, lastID string, block time.Duration) ([]redisStreamMessage, error) +} +``` + +Use the same URL parsing behavior as `newRedisBus`, and add: + +```go +func NewPlatformRedisClient(rawURL string) (PlatformRedisClient, error) { + return newRedisBus(rawURL) +} +``` + +Extend `redisBus` in `team_redis.go` only if needed so the same type satisfies `SetNX` and `Del`: + +```go +func (b *redisBus) SetNX(ctx context.Context, key, value string, ttl time.Duration) (bool, error) { + reply, err := b.do(ctx, "SET", key, value, "NX", "PX", fmt.Sprintf("%d", ttl.Milliseconds())) + if err != nil { + return false, err + } + return reply == "OK", nil +} + +func (b *redisBus) Del(ctx context.Context, key string) error { + _, err := b.do(ctx, "DEL", key) + return err +} +``` + +- [ ] **Step 2: Add runtime event service** + +Create `backend/internal/services/runtime_events.go`: + +```go +package services + +import ( + "context" + "encoding/json" + "time" +) + +const runtimeEventStreamKey = "clawmanager:runtime-events" + +type RuntimeEvent struct { + Type string `json:"type"` + Payload json.RawMessage `json:"payload"` + CreatedAt time.Time `json:"created_at"` +} + +type RuntimeEventService interface { + Publish(ctx context.Context, eventType string, payload any) error + Read(ctx context.Context, lastID string, block time.Duration) ([]redisStreamMessage, error) +} + +type runtimeEventService struct { + redis PlatformRedisClient +} + +func NewRuntimeEventService(redis PlatformRedisClient) RuntimeEventService { + return &runtimeEventService{redis: redis} +} + +func (s *runtimeEventService) Publish(ctx context.Context, eventType string, payload any) error { + raw, err := json.Marshal(payload) + if err != nil { + return err + } + event := RuntimeEvent{Type: eventType, Payload: raw, CreatedAt: time.Now().UTC()} + eventRaw, err := json.Marshal(event) + if err != nil { + return err + } + _, err = s.redis.XAdd(ctx, runtimeEventStreamKey, map[string]string{"event": string(eventRaw)}) + return err +} + +func (s *runtimeEventService) Read(ctx context.Context, lastID string, block time.Duration) ([]redisStreamMessage, error) { + return s.redis.XRead(ctx, runtimeEventStreamKey, lastID, block) +} +``` + +- [ ] **Step 3: Write scheduler assignment test with fakes** + +Create `backend/internal/services/runtime_scheduler_test.go` with fake repositories and this test: + +```go +func TestRuntimeSchedulerAssignsCreatingInstanceToReadyPod(t *testing.T) { + instance := models.Instance{ID: 123, UserID: 45, Type: "openclaw", CPUCores: 2, MemoryGB: 4, DiskGB: 20, RuntimeGeneration: 1} + pod := models.RuntimePod{ID: 9, RuntimeType: "openclaw", PodIP: stringPtr("10.42.0.31"), AgentEndpoint: stringPtr("http://10.42.0.31:19090"), State: "ready", Capacity: 100, UsedSlots: 0} + s := newRuntimeSchedulerForTest(instance, pod) + + if err := s.assignInstance(context.Background(), instance); err != nil { + t.Fatalf("assignInstance returned error: %v", err) + } + if s.fakeAgent.createGatewayCalls != 1 { + t.Fatalf("expected one create gateway call, got %d", s.fakeAgent.createGatewayCalls) + } + binding := s.fakeBindings.byInstance[123] + if binding.GatewayPort != 20017 || binding.RuntimePodID != 9 { + t.Fatalf("unexpected binding %#v", binding) + } +} +``` + +The fake agent must return: + +```go +&RuntimeAgentCreateGatewayResponse{ + GatewayID: "gw-123-1", + Port: 20017, + PID: intPtr(8842), + Status: "running", +} +``` + +- [ ] **Step 4: Implement scheduler** + +Create `backend/internal/services/runtime_scheduler.go` with: + +```go +package services + +import ( + "context" + "fmt" + "log" + "time" + + "clawreef/internal/models" + "clawreef/internal/repository" + k8ssvc "clawreef/internal/services/k8s" +) + +type RuntimeScheduler struct { + instanceRepo repository.InstanceRepository + podRepo repository.RuntimePodRepository + bindingRepo repository.InstanceRuntimeBindingRepository + rolloutRepo repository.RuntimeRolloutRepository + agentClient RuntimeAgentClient + events RuntimeEventService + leader RuntimeLeaderService + deployments k8ssvc.RuntimeDeploymentService + tick time.Duration +} + +func NewRuntimeScheduler( + instanceRepo repository.InstanceRepository, + podRepo repository.RuntimePodRepository, + bindingRepo repository.InstanceRuntimeBindingRepository, + rolloutRepo repository.RuntimeRolloutRepository, + agentClient RuntimeAgentClient, + events RuntimeEventService, + leader RuntimeLeaderService, + deployments k8ssvc.RuntimeDeploymentService, + tick time.Duration, +) *RuntimeScheduler { + return &RuntimeScheduler{ + instanceRepo: instanceRepo, + podRepo: podRepo, + bindingRepo: bindingRepo, + rolloutRepo: rolloutRepo, + agentClient: agentClient, + events: events, + leader: leader, + deployments: deployments, + tick: tick, + } +} + +func (s *RuntimeScheduler) Start(ctx context.Context) { + go func() { + ticker := time.NewTicker(s.tick) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if !s.leader.IsLeader(ctx) { + continue + } + if err := s.reconcile(ctx); err != nil { + log.Printf("runtime scheduler reconcile failed: %v", err) + } + } + } + }() +} + +func (s *RuntimeScheduler) reconcile(ctx context.Context) error { + creating, err := s.instanceRepo.GetV2Creating(ctx, 100) + if err != nil { + return err + } + for _, instance := range creating { + if err := s.assignInstance(ctx, instance); err != nil { + msg := err.Error() + _ = s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "error", instance.RuntimeGeneration, &msg) + } + } + running, err := s.instanceRepo.GetV2DesiredRunning(ctx, 200) + if err != nil { + return err + } + for _, instance := range running { + if _, err := s.bindingRepo.GetRunningByInstanceID(ctx, instance.ID); err != nil { + if assignErr := s.assignInstance(ctx, instance); assignErr != nil { + msg := assignErr.Error() + _ = s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "error", instance.RuntimeGeneration, &msg) + } + } + } + return nil +} + +func (s *RuntimeScheduler) assignInstance(ctx context.Context, instance models.Instance) error { + runtimeType, ok := NormalizeV2RuntimeType(instance.Type) + if !ok { + return fmt.Errorf("unsupported V2 runtime type %s", instance.Type) + } + pods, err := s.podRepo.ListSchedulable(ctx, runtimeType) + if err != nil { + return err + } + for _, pod := range pods { + if pod.AgentEndpoint == nil { + continue + } + claimed, err := s.podRepo.TryClaimSlot(ctx, pod.ID) + if err != nil { + return err + } + if !claimed { + continue + } + if err := s.createGatewayOnPod(ctx, instance, pod); err != nil { + _ = s.podRepo.ReleaseSlot(ctx, pod.ID) + continue + } + return nil + } + return fmt.Errorf("no schedulable runtime pod for %s", runtimeType) +} + +func (s *RuntimeScheduler) createGatewayOnPod(ctx context.Context, instance models.Instance, pod models.RuntimePod) error { + workspacePath := RuntimeWorkspacePath(pod.RuntimeType, instance.UserID, instance.ID) + linuxID := RuntimeLinuxID(instance.ID) + resp, err := s.agentClient.CreateGateway(ctx, *pod.AgentEndpoint, RuntimeAgentCreateGatewayRequest{ + InstanceID: instance.ID, + UserID: instance.UserID, + AgentType: pod.RuntimeType, + WorkspacePath: workspacePath, + PortRange: RuntimeAgentPortRange{Start: RuntimeGatewayPortStart, End: RuntimeGatewayPortEnd}, + UID: linuxID, + GID: linuxID, + CPUCores: instance.CPUCores, + MemoryMB: instance.MemoryGB * 1024, + DiskQuotaMB: instance.DiskGB * 1024, + Generation: instance.RuntimeGeneration, + }) + if err != nil { + return err + } + binding := &models.InstanceRuntimeBinding{ + InstanceID: instance.ID, + RuntimePodID: pod.ID, + RuntimeType: pod.RuntimeType, + GatewayID: resp.GatewayID, + GatewayPort: resp.Port, + GatewayPID: resp.PID, + WorkspacePath: workspacePath, + State: "running", + Generation: instance.RuntimeGeneration, + } + if err := s.bindingRepo.Create(ctx, binding); err != nil { + return err + } + if err := s.instanceRepo.SetWorkspacePath(ctx, instance.ID, workspacePath); err != nil { + return err + } + return s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "running", instance.RuntimeGeneration, nil) +} +``` + +Add failover and rollout in the same file with these public methods: + +```go +func (s *RuntimeScheduler) DrainPod(ctx context.Context, podID int64) error +func (s *RuntimeScheduler) FailoverPod(ctx context.Context, podID int64, reason string) error +func (s *RuntimeScheduler) StartRollout(ctx context.Context, rolloutID int64) error +``` + +`FailoverPod` must mark the Pod `unhealthy`, list its bindings, delete each binding, release the old slot, increment instance generation, set instance status `creating`, and let the next scheduler tick recreate the gateway on another Pod. + +- [ ] **Step 5: Wire scheduler in `main.go`** + +In `backend/cmd/server/main.go`, create the new repositories after the existing repositories: + +```go +runtimePodRepo := repository.NewRuntimePodRepository(sess) +bindingRepo := repository.NewInstanceRuntimeBindingRepository(sess) +rolloutRepo := repository.NewRuntimeRolloutRepository(sess) +``` + +Create Redis and runtime services: + +```go +platformRedis, err := services.NewPlatformRedisClient(cfg.Runtime.RedisURL) +if err != nil { + log.Printf("platform redis disabled: %v", err) +} +runtimeEvents := services.NewRuntimeEventService(platformRedis) +agentClient := services.NewRuntimeAgentClient(cfg.Runtime.AgentControlToken) +leader := services.NewRuntimeLeaderService(k8sClient, cfg.Runtime.Namespace, cfg.Runtime.BackendReplicaID) +runtimeDeployments := k8s.NewRuntimeDeploymentService(k8sClient) +runtimeScheduler := services.NewRuntimeScheduler(instanceRepo, runtimePodRepo, bindingRepo, rolloutRepo, agentClient, runtimeEvents, leader, runtimeDeployments, cfg.Runtime.SchedulerTick) +if cfg.Runtime.SchedulerEnabled { + runtimeScheduler.Start(ctx) +} +``` + +If `platformRedis` can be nil in local development, make `NewRuntimeEventService` accept nil and return a no-op service that never panics. + +- [ ] **Step 6: Run scheduler tests** + +Run: + +```powershell +go test ./backend/internal/services -run RuntimeScheduler -count=1 +``` + +Expected: pass. + +- [ ] **Step 7: Commit scheduler** + +Run: + +```powershell +git add backend/internal/services backend/cmd/server/main.go +git commit -m "feat: add runtime scheduler" +``` + +## Task 5: V2 Instance Lifecycle And Pod-IP Proxy + +**Files:** +- Modify: `backend/internal/services/instance_service.go` +- Modify: `backend/internal/services/instance_proxy_service.go` +- Modify: `backend/internal/services/sync_service.go` +- Modify: `backend/internal/handlers/instance_handler.go` +- Modify: `backend/cmd/server/main.go` +- Create: `backend/internal/services/instance_proxy_service_v2_test.go` + +- [ ] **Step 1: Add proxy test for binding target** + +Create `backend/internal/services/instance_proxy_service_v2_test.go` with a test that builds an instance, a binding, and a runtime Pod: + +```go +func TestInstanceProxyServiceUsesRuntimeBindingForV2(t *testing.T) { + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/apps/openclaw" { + t.Fatalf("unexpected path %s", r.URL.Path) + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + })) + defer target.Close() + + podIP, port := splitHostPortForTest(t, target.URL) + instance := models.Instance{ID: 123, UserID: 45, Type: "openclaw", Status: "running"} + binding := models.InstanceRuntimeBinding{InstanceID: 123, RuntimePodID: 9, GatewayPort: port, State: "running"} + pod := models.RuntimePod{ID: 9, PodIP: &podIP, State: "ready"} + + service := newProxyServiceForV2Test(instance, binding, pod) + req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/123/proxy/apps/openclaw", nil) + rec := httptest.NewRecorder() + + service.Proxy(rec, req, 123, "/apps/openclaw") + if rec.Code != http.StatusOK || rec.Body.String() != "ok" { + t.Fatalf("unexpected proxy response %d %q", rec.Code, rec.Body.String()) + } +} +``` + +- [ ] **Step 2: Modify instance creation for V2** + +In `backend/internal/services/instance_service.go`, keep legacy behavior for non-V2 types and add this branch near the start of create: + +```go +runtimeType, isV2 := NormalizeV2RuntimeType(req.Type) +if isV2 { + instance := &models.Instance{ + UserID: userID, + Name: strings.TrimSpace(req.Name), + Description: normalizeStringPtr(req.Description), + Type: runtimeType, + RuntimeType: "gateway", + Status: "creating", + CPUCores: req.CPUCores, + MemoryGB: req.MemoryGB, + DiskGB: req.DiskGB, + StorageClass: strings.TrimSpace(req.StorageClass), + MountPath: "/workspaces", + RuntimeGeneration: 1, + } + if err := s.instanceRepo.Create(ctx, instance); err != nil { + return nil, err + } + workspacePath := RuntimeWorkspacePath(runtimeType, userID, instance.ID) + if err := os.MkdirAll(workspacePath, 0750); err != nil { + return nil, err + } + if err := s.instanceRepo.SetWorkspacePath(ctx, instance.ID, workspacePath); err != nil { + return nil, err + } + instance.WorkspacePath = &workspacePath + return instance, nil +} +``` + +Keep validation so new creation accepts only `openclaw` and `hermes`; old records with `ubuntu`, `webtop`, `debian`, `centos`, or `custom` remain readable and manageable through legacy paths. + +- [ ] **Step 3: Modify start, stop, restart, delete** + +For V2 instances in `instance_service.go`: + +```go +func (s *instanceService) Start(ctx context.Context, userID int, instanceID int) error { + instance, err := s.mustOwnInstance(ctx, userID, instanceID) + if err != nil { + return err + } + if _, ok := NormalizeV2RuntimeType(instance.Type); ok { + return s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "creating", instance.RuntimeGeneration+1, nil) + } + return s.startLegacy(ctx, instance) +} + +func (s *instanceService) Stop(ctx context.Context, userID int, instanceID int) error { + instance, err := s.mustOwnInstance(ctx, userID, instanceID) + if err != nil { + return err + } + if _, ok := NormalizeV2RuntimeType(instance.Type); ok { + binding, err := s.bindingRepo.GetByInstanceID(ctx, instance.ID) + if err == nil { + pod, podErr := s.runtimePodRepo.GetByID(ctx, binding.RuntimePodID) + if podErr == nil && pod.AgentEndpoint != nil { + _ = s.agentClient.DeleteGateway(ctx, *pod.AgentEndpoint, binding.GatewayID) + } + _ = s.bindingRepo.DeleteByInstanceID(ctx, instance.ID) + _ = s.runtimePodRepo.ReleaseSlot(ctx, binding.RuntimePodID) + } + return s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "stopped", instance.RuntimeGeneration, nil) + } + return s.stopLegacy(ctx, instance) +} +``` + +Use the existing legacy code by moving old logic into private helpers `startLegacy`, `stopLegacy`, and `deleteLegacy`. + +- [ ] **Step 4: Modify status for ordinary users** + +In `backend/internal/handlers/instance_handler.go`, make `/instances/:id/status` return user-safe availability for V2: + +```json +{ + "instance_status": { + "instance_id": 123, + "status": "running", + "availability": "available", + "agent_type": "openclaw", + "workspace_usage_bytes": 123456 + } +} +``` + +Do not include Pod name, namespace, Pod IP, service name, port, capacity, node, or runtime scheduling fields for normal users. + +- [ ] **Step 5: Modify proxy for V2** + +In `backend/internal/services/instance_proxy_service.go`, resolve target for V2: + +```go +func (s *instanceProxyService) resolveV2Target(ctx context.Context, instanceID int) (*url.URL, error) { + binding, err := s.bindingRepo.GetRunningByInstanceID(ctx, instanceID) + if err != nil { + return nil, fmt.Errorf("instance gateway is not available") + } + pod, err := s.runtimePodRepo.GetByID(ctx, binding.RuntimePodID) + if err != nil { + return nil, err + } + if pod.PodIP == nil || *pod.PodIP == "" { + return nil, fmt.Errorf("runtime pod ip is not available") + } + return &url.URL{Scheme: "http", Host: net.JoinHostPort(*pod.PodIP, strconv.Itoa(binding.GatewayPort))}, nil +} +``` + +Use this target when `NormalizeV2RuntimeType(instance.Type)` succeeds, otherwise execute the existing Service-based proxy path. + +- [ ] **Step 6: Run lifecycle and proxy tests** + +Run: + +```powershell +go test ./backend/internal/services -run 'Instance|Proxy|RuntimeScheduler' -count=1 +``` + +Expected: pass. + +- [ ] **Step 7: Commit lifecycle and proxy** + +Run: + +```powershell +git add backend/internal/services backend/internal/handlers backend/cmd/server/main.go +git commit -m "feat: route V2 instances through runtime gateways" +``` + +## Task 6: Workspace File Manager Backend + +**Files:** +- Create: `backend/internal/services/workspace_path_guard.go` +- Create: `backend/internal/services/workspace_path_guard_test.go` +- Create: `backend/internal/services/workspace_file_service.go` +- Create: `backend/internal/services/workspace_file_service_test.go` +- Create: `backend/internal/handlers/workspace_file_handler.go` +- Modify: `backend/internal/utils/response.go` +- Modify: `backend/cmd/server/main.go` + +- [ ] **Step 1: Write path guard tests** + +Create `backend/internal/services/workspace_path_guard_test.go`: + +```go +package services + +import ( + "os" + "path/filepath" + "testing" +) + +func TestResolveWorkspacePathRejectsTraversal(t *testing.T) { + root := t.TempDir() + _, err := ResolveWorkspacePath(root, "../outside.txt") + if err == nil || err.Error() != "workspace path escapes instance workspace" { + t.Fatalf("unexpected error %v", err) + } +} + +func TestResolveWorkspacePathRejectsSymlinkEscape(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + if err := os.Symlink(outside, filepath.Join(root, "escape")); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + _, err := ResolveWorkspacePath(root, "escape/file.txt") + if err == nil || err.Error() != "workspace path escapes instance workspace" { + t.Fatalf("unexpected error %v", err) + } +} + +func TestResolveWorkspacePathAllowsNestedFile(t *testing.T) { + root := t.TempDir() + got, err := ResolveWorkspacePath(root, "a/b.txt") + if err != nil { + t.Fatalf("ResolveWorkspacePath returned error: %v", err) + } + want := filepath.Join(root, "a", "b.txt") + if got != want { + t.Fatalf("want %s got %s", want, got) + } +} +``` + +- [ ] **Step 2: Implement path guard** + +Create `backend/internal/services/workspace_path_guard.go`: + +```go +package services + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +func ResolveWorkspacePath(workspaceRoot, relativePath string) (string, error) { + workspaceRoot = filepath.Clean(workspaceRoot) + cleanRelative := filepath.Clean("/" + strings.TrimSpace(relativePath)) + cleanRelative = strings.TrimPrefix(cleanRelative, string(filepath.Separator)) + fullPath := filepath.Join(workspaceRoot, cleanRelative) + + rootReal, err := filepath.EvalSymlinks(workspaceRoot) + if err != nil { + return "", err + } + parent := fullPath + if info, statErr := os.Lstat(fullPath); statErr == nil && !info.IsDir() { + parent = filepath.Dir(fullPath) + } + if _, statErr := os.Lstat(parent); statErr == nil { + realParent, evalErr := filepath.EvalSymlinks(parent) + if evalErr != nil { + return "", evalErr + } + if !isPathInside(rootReal, realParent) { + return "", fmt.Errorf("workspace path escapes instance workspace") + } + } + if !isPathInside(workspaceRoot, fullPath) { + return "", fmt.Errorf("workspace path escapes instance workspace") + } + return fullPath, nil +} + +func isPathInside(root, candidate string) bool { + rel, err := filepath.Rel(root, candidate) + if err != nil { + return false + } + return rel == "." || (!strings.HasPrefix(rel, ".."+string(filepath.Separator)) && rel != "..") +} +``` + +- [ ] **Step 3: Write workspace service tests** + +Create `backend/internal/services/workspace_file_service_test.go`: + +```go +func TestWorkspaceFileServicePreviewTextLimit(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "small.txt"), []byte("hello"), 0640); err != nil { + t.Fatal(err) + } + service := NewWorkspaceFileService(fakeAuditRepo{}) + preview, err := service.Preview(context.Background(), WorkspaceFileScope{InstanceID: 1, UserID: 2, WorkspacePath: root}, "small.txt") + if err != nil { + t.Fatalf("Preview returned error: %v", err) + } + if preview.Kind != "text" || preview.Text != "hello" { + t.Fatalf("unexpected preview %#v", preview) + } +} + +func TestWorkspaceFileServiceRejectsLargeTextPreview(t *testing.T) { + root := t.TempDir() + large := bytes.Repeat([]byte("a"), 1024*1024+1) + if err := os.WriteFile(filepath.Join(root, "large.log"), large, 0640); err != nil { + t.Fatal(err) + } + service := NewWorkspaceFileService(fakeAuditRepo{}) + _, err := service.Preview(context.Background(), WorkspaceFileScope{InstanceID: 1, UserID: 2, WorkspacePath: root}, "large.log") + if err == nil || err.Error() != "text preview exceeds 1 MiB" { + t.Fatalf("unexpected error %v", err) + } +} +``` + +- [ ] **Step 4: Implement workspace file service** + +Create `backend/internal/services/workspace_file_service.go` with these exported types: + +```go +type WorkspaceFileScope struct { + InstanceID int + UserID int + WorkspacePath string +} + +type WorkspaceEntry struct { + Name string `json:"name"` + Path string `json:"path"` + IsDir bool `json:"is_dir"` + Size int64 `json:"size"` + ModifiedAt time.Time `json:"modified_at"` + Previewable bool `json:"previewable"` + Downloadable bool `json:"downloadable"` +} + +type WorkspacePreview struct { + Kind string `json:"kind"` + ContentType string `json:"content_type"` + Text string `json:"text,omitempty"` + URL string `json:"url,omitempty"` +} + +type WorkspaceFileService interface { + List(ctx context.Context, scope WorkspaceFileScope, relativePath string) ([]WorkspaceEntry, error) + Preview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspacePreview, error) + OpenDownload(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) + Upload(ctx context.Context, scope WorkspaceFileScope, relativeDir string, filename string, reader io.Reader, size int64) error + Mkdir(ctx context.Context, scope WorkspaceFileScope, relativePath string) error + Rename(ctx context.Context, scope WorkspaceFileScope, oldPath, newPath string) error + Delete(ctx context.Context, scope WorkspaceFileScope, relativePath string) error +} +``` + +Rules to implement: +- Text preview extensions: `.txt`, `.md`, `.json`, `.yaml`, `.yml`, `.log`, `.py`, `.js`, `.ts`, `.go`, `.sh`; max 1 MiB. +- Image preview extensions: `.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, `.svg`; max 10 MiB. +- PDF preview uses iframe URL and content type `application/pdf`. +- Unknown binary files are download-only. +- Upload max is read from env `WORKSPACE_UPLOAD_MAX_BYTES`, default `524288000`. +- Audit upload, mkdir, rename, delete, and download. Do not audit preview. + +- [ ] **Step 5: Implement handler routes** + +Create `backend/internal/handlers/workspace_file_handler.go` with endpoints: + +```go +GET /api/v1/instances/:id/workspace/files +GET /api/v1/instances/:id/workspace/preview +GET /api/v1/instances/:id/workspace/download +POST /api/v1/instances/:id/workspace/upload +POST /api/v1/instances/:id/workspace/folders +PATCH /api/v1/instances/:id/workspace/entries +DELETE /api/v1/instances/:id/workspace/entries +``` + +Each handler must load the instance, confirm ownership from auth context, require `WorkspacePath != nil`, and create: + +```go +scope := services.WorkspaceFileScope{ + InstanceID: instance.ID, + UserID: userID, + WorkspacePath: *instance.WorkspacePath, +} +``` + +- [ ] **Step 6: Wire workspace routes** + +In `backend/cmd/server/main.go`: + +```go +workspaceAuditRepo := repository.NewWorkspaceFileAuditRepository(sess) +workspaceFileService := services.NewWorkspaceFileService(workspaceAuditRepo) +workspaceFileHandler := handlers.NewWorkspaceFileHandler(instanceService, workspaceFileService) + +instances := api.Group("/instances") +instances.GET("/:id/workspace/files", workspaceFileHandler.List) +instances.GET("/:id/workspace/preview", workspaceFileHandler.Preview) +instances.GET("/:id/workspace/download", workspaceFileHandler.Download) +instances.POST("/:id/workspace/upload", workspaceFileHandler.Upload) +instances.POST("/:id/workspace/folders", workspaceFileHandler.Mkdir) +instances.PATCH("/:id/workspace/entries", workspaceFileHandler.Rename) +instances.DELETE("/:id/workspace/entries", workspaceFileHandler.Delete) +``` + +- [ ] **Step 7: Run workspace backend tests** + +Run: + +```powershell +go test ./backend/internal/services ./backend/internal/handlers -run 'Workspace|ResolveWorkspacePath' -count=1 +``` + +Expected: pass. + +- [ ] **Step 8: Commit workspace backend** + +Run: + +```powershell +git add backend/internal/services backend/internal/handlers backend/internal/repository backend/internal/utils backend/cmd/server/main.go +git commit -m "feat: add workspace file APIs" +``` + +## Task 7: Runtime Agent Report And Admin APIs + +**Files:** +- Create: `backend/internal/handlers/runtime_agent_handler.go` +- Create: `backend/internal/handlers/runtime_pool_handler.go` +- Create: `backend/internal/handlers/runtime_pool_handler_test.go` +- Modify: `backend/internal/services/websocket_service.go` +- Modify: `backend/cmd/server/main.go` + +- [ ] **Step 1: Implement runtime agent report handler** + +Create `backend/internal/handlers/runtime_agent_handler.go` with: + +```go +type RuntimeAgentHandler struct { + cfg config.RuntimeConfig + podRepo repository.RuntimePodRepository + bindingRepo repository.InstanceRuntimeBindingRepository + events services.RuntimeEventService +} + +func (h *RuntimeAgentHandler) requireAgentToken(c *gin.Context) bool { + if h.cfg.AgentReportToken == "" || c.GetHeader("X-ClawManager-Agent-Token") != h.cfg.AgentReportToken { + utils.Unauthorized(c, "invalid runtime agent token") + return false + } + return true +} +``` + +Add handlers: +- `Register` upserts `runtime_pods`. +- `Heartbeat` updates state, used slots, draining, and last seen. +- `ReportMetrics` updates metrics and binding health, then publishes `runtime_pod_metrics` to Redis events. +- `ReportGateways` reconciles binding state from agent. +- `ReportSkills` stores skill reports using existing skill service hooks if available; if no hook exists, accept the request and publish a `runtime_agent_skills_reported` event. + +- [ ] **Step 2: Implement admin runtime handler** + +Create `backend/internal/handlers/runtime_pool_handler.go`: + +```go +type RuntimePoolHandler struct { + podRepo repository.RuntimePodRepository + bindingRepo repository.InstanceRuntimeBindingRepository + rolloutRepo repository.RuntimeRolloutRepository + scheduler *services.RuntimeScheduler +} + +func (h *RuntimePoolHandler) ListPods(c *gin.Context) +func (h *RuntimePoolHandler) GetPodGateways(c *gin.Context) +func (h *RuntimePoolHandler) DrainPod(c *gin.Context) +func (h *RuntimePoolHandler) StartRollout(c *gin.Context) +``` + +`ListPods` returns: + +```json +{ + "pods": [ + { + "id": 9, + "runtime_type": "openclaw", + "pod_name": "openclaw-runtime-abcde", + "pod_ip": "10.42.0.31", + "node_name": "node-a", + "state": "ready", + "used_slots": 37, + "capacity": 100, + "draining": false, + "cpu_millis_used": 13600, + "memory_bytes_used": 42949672960, + "disk_bytes_used": 214748364800, + "network_rx_bytes": 9223372, + "network_tx_bytes": 19223372, + "last_seen_at": "2026-06-01T10:00:00Z" + } + ] +} +``` + +- [ ] **Step 3: Add admin handler tests** + +Create `backend/internal/handlers/runtime_pool_handler_test.go` and verify non-admin access is rejected and admin access returns metrics. Use existing auth middleware test helpers if present; otherwise call the handler directly with `gin.CreateTestContext`. + +- [ ] **Step 4: Extend websocket service** + +Modify `backend/internal/services/websocket_service.go`: + +```go +type WebSocketTopic string + +const ( + WebSocketTopicUser WebSocketTopic = "user" + WebSocketTopicRuntimeAdmin WebSocketTopic = "runtime_admin" +) +``` + +Add subscription filtering: + +```go +type Client struct { + UserID int + Role string + Topic WebSocketTopic + Send chan []byte +} + +func (h *Hub) BroadcastRuntimeAdmin(message []byte) { + h.mu.RLock() + defer h.mu.RUnlock() + for client := range h.clients { + if client.Role == "admin" && client.Topic == WebSocketTopicRuntimeAdmin { + select { + case client.Send <- message: + default: + } + } + } +} +``` + +Start one goroutine per backend replica to `XREAD` `clawmanager:runtime-events` and broadcast runtime admin messages to local websocket clients. + +- [ ] **Step 5: Wire routes** + +In `backend/cmd/server/main.go`: + +```go +runtimeAgentHandler := handlers.NewRuntimeAgentHandler(cfg.Runtime, runtimePodRepo, bindingRepo, runtimeEvents) +api.POST("/runtime-agent/register", runtimeAgentHandler.Register) +api.POST("/runtime-agent/heartbeat", runtimeAgentHandler.Heartbeat) +api.POST("/runtime-agent/gateways/report", runtimeAgentHandler.ReportGateways) +api.POST("/runtime-agent/skills/report", runtimeAgentHandler.ReportSkills) +api.POST("/runtime-agent/metrics/report", runtimeAgentHandler.ReportMetrics) + +runtimePoolHandler := handlers.NewRuntimePoolHandler(runtimePodRepo, bindingRepo, rolloutRepo, runtimeScheduler) +admin.GET("/runtime-pods", runtimePoolHandler.ListPods) +admin.GET("/runtime-pods/:id/gateways", runtimePoolHandler.GetPodGateways) +admin.POST("/runtime-pods/:id/drain", runtimePoolHandler.DrainPod) +admin.POST("/runtime-rollouts", runtimePoolHandler.StartRollout) +``` + +- [ ] **Step 6: Run admin API tests** + +Run: + +```powershell +go test ./backend/internal/handlers ./backend/internal/services -run 'RuntimePool|RuntimeAgent|WebSocket' -count=1 +``` + +Expected: pass. + +- [ ] **Step 7: Commit admin runtime APIs** + +Run: + +```powershell +git add backend/internal/handlers backend/internal/services backend/cmd/server/main.go +git commit -m "feat: add runtime admin APIs" +``` + +## Task 8: Frontend Simple UI Foundation + +**Files:** +- Modify: `frontend/package.json` +- Modify: package lock file if present after install. +- Modify: `frontend/src/index.css` +- Modify: `frontend/src/components/UserLayout.tsx` +- Modify: `frontend/src/components/AdminLayout.tsx` +- Modify: `frontend/src/App.css` if it still injects decorative gradients. + +- [ ] **Step 1: Add icon dependency** + +Run: + +```powershell +cd frontend +npm install lucide-react +cd .. +``` + +Expected: `frontend/package.json` and the lock file include `lucide-react`. + +- [ ] **Step 2: Replace global visual tokens** + +Modify `frontend/src/index.css` so the design uses a neutral, simple base: + +```css +:root { + font-family: + Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", + sans-serif; + color: #111827; + background: #f7f8fa; + font-synthesis: none; + text-rendering: optimizeLegibility; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +body { + margin: 0; + min-width: 320px; + min-height: 100vh; + background: #f7f8fa; +} + +button, +input, +select, +textarea { + font: inherit; +} + +.cm-surface { + background: #ffffff; + border: 1px solid #e5e7eb; + border-radius: 8px; +} + +.cm-button { + display: inline-flex; + align-items: center; + justify-content: center; + gap: 6px; + min-height: 36px; + border-radius: 6px; + border: 1px solid #d1d5db; + background: #ffffff; + color: #111827; +} + +.cm-button-primary { + border-color: #2563eb; + background: #2563eb; + color: #ffffff; +} +``` + +Remove decorative gradient or orb classes from `App.css`. + +- [ ] **Step 3: Simplify user layout** + +Modify `frontend/src/components/UserLayout.tsx`: +- Use a plain top bar and left nav. +- Keep border radius at `rounded-lg` or smaller. +- Use `lucide-react` icons for nav and command buttons. +- Remove runtime details from ordinary user navigation labels. +- Keep language switcher and account actions. + +Use this layout shape: + +```tsx +
+ +
+
+ ... +
+
+ +
+
+
+``` + +- [ ] **Step 4: Simplify admin layout** + +Modify `frontend/src/components/AdminLayout.tsx` with the same visual system and add a nav entry: + +```tsx +{ + to: "/admin/runtime-pods", + label: "Runtime Pods", + icon: Server, +} +``` + +Do not expose admin-only runtime metrics in `UserLayout`. + +- [ ] **Step 5: Run frontend lint** + +Run: + +```powershell +cd frontend +npm run lint +cd .. +``` + +Expected: pass. + +- [ ] **Step 6: Commit UI foundation** + +Run: + +```powershell +git add frontend/package.json frontend/package-lock.json frontend/src/index.css frontend/src/App.css frontend/src/components/UserLayout.tsx frontend/src/components/AdminLayout.tsx +git commit -m "style: simplify ClawManager UI shell" +``` + +## Task 9: Frontend Workspace Manager And V2 User Pages + +**Files:** +- Modify: `frontend/src/types/instance.ts` +- Create: `frontend/src/types/workspace.ts` +- Create: `frontend/src/services/workspaceService.ts` +- Create: `frontend/src/components/WorkspaceFileManager.tsx` +- Create: `frontend/src/components/InstanceServiceFrame.tsx` +- Modify: `frontend/src/pages/instances/CreateInstancePage.tsx` +- Modify: `frontend/src/pages/instances/InstanceListPage.tsx` +- Modify: `frontend/src/pages/instances/InstanceDetailPage.tsx` +- Modify: `frontend/src/services/instanceService.ts` + +- [ ] **Step 1: Add TypeScript types** + +Create `frontend/src/types/workspace.ts`: + +```ts +export interface WorkspaceEntry { + name: string; + path: string; + is_dir: boolean; + size: number; + modified_at: string; + previewable: boolean; + downloadable: boolean; +} + +export interface WorkspacePreview { + kind: "text" | "image" | "pdf" | "download"; + content_type: string; + text?: string; + url?: string; +} +``` + +Modify `frontend/src/types/instance.ts`: + +```ts +export type V2InstanceType = "openclaw" | "hermes"; +export type InstanceAvailability = "available" | "starting" | "unavailable"; + +export interface InstanceStatus { + instance_id: number; + status: string; + availability?: InstanceAvailability; + agent_type?: V2InstanceType; + workspace_usage_bytes?: number; + created_at?: string; + started_at?: string; +} +``` + +Keep legacy optional Pod fields only if existing pages still need them for admin legacy screens. + +- [ ] **Step 2: Add workspace service** + +Create `frontend/src/services/workspaceService.ts`: + +```ts +import api from "./api"; +import type { WorkspaceEntry, WorkspacePreview } from "../types/workspace"; + +export const workspaceService = { + async list(instanceId: number, path = ""): Promise { + const response = await api.get(`/instances/${instanceId}/workspace/files`, { + params: { path }, + }); + return response.data.data.entries; + }, + + async preview(instanceId: number, path: string): Promise { + const response = await api.get(`/instances/${instanceId}/workspace/preview`, { + params: { path }, + }); + return response.data.data.preview; + }, + + downloadUrl(instanceId: number, path: string): string { + return `/api/v1/instances/${instanceId}/workspace/download?path=${encodeURIComponent(path)}`; + }, + + async upload(instanceId: number, path: string, file: File): Promise { + const formData = new FormData(); + formData.append("file", file); + await api.post(`/instances/${instanceId}/workspace/upload`, formData, { + params: { path }, + headers: { "Content-Type": "multipart/form-data" }, + }); + }, + + async mkdir(instanceId: number, path: string): Promise { + await api.post(`/instances/${instanceId}/workspace/folders`, { path }); + }, + + async rename(instanceId: number, oldPath: string, newPath: string): Promise { + await api.patch(`/instances/${instanceId}/workspace/entries`, { + old_path: oldPath, + new_path: newPath, + }); + }, + + async remove(instanceId: number, path: string): Promise { + await api.delete(`/instances/${instanceId}/workspace/entries`, { + params: { path }, + }); + }, +}; +``` + +- [ ] **Step 3: Build independent file manager** + +Create `frontend/src/components/WorkspaceFileManager.tsx`. It must: +- Fetch entries with React Query. +- Show breadcrumb path. +- Upload via hidden file input. +- Provide icon buttons for preview, download, rename, delete, and new folder. +- Show text preview in a compact preformatted pane. +- Show image/PDF preview in a modal or right panel. +- Never render an arbitrary path input that can bypass the breadcrumb. + +Core state: + +```tsx +const [currentPath, setCurrentPath] = useState(""); +const [previewPath, setPreviewPath] = useState(null); +const entriesQuery = useQuery({ + queryKey: ["workspace", instanceId, currentPath], + queryFn: () => workspaceService.list(instanceId, currentPath), +}); +``` + +- [ ] **Step 4: Build service frame** + +Create `frontend/src/components/InstanceServiceFrame.tsx`: + +```tsx +interface InstanceServiceFrameProps { + instanceId: number; + availability: "available" | "starting" | "unavailable"; +} + +export function InstanceServiceFrame({ instanceId, availability }: InstanceServiceFrameProps) { + if (availability === "starting") { + return
Starting
; + } + if (availability === "unavailable") { + return
Unavailable
; + } + return ( +