diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 9d0efe5..391c77f 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -8,6 +8,7 @@ on:
- 'feature/**'
- 'fix/**'
pull_request:
+ workflow_dispatch:
permissions:
contents: read
@@ -106,6 +107,7 @@ jobs:
uses: helm/kind-action@v1.12.0
with:
cluster_name: clawmanager-ci
+ ignore_failed_clean: true
- name: Load image into kind
run: kind load docker-image clawmanager:ci --name clawmanager-ci
@@ -157,6 +159,12 @@ jobs:
count=1,
)
+ text = re.sub(
+ r"(\n\s+- name: workspaces\s+)nfs:\s+server: workspace-store\.clawmanager-system\.svc\.cluster\.local\s+path: /exports/workspaces",
+ r"\1emptyDir: {}",
+ text,
+ )
+
path.write_text(text, encoding="utf-8")
PY
diff --git a/.gitignore b/.gitignore
index ad3f021..40af172 100644
--- a/.gitignore
+++ b/.gitignore
@@ -59,6 +59,9 @@ Thumbs.db
.codex-logs/
+.cache/
+.VSCodeCounter/
+.superpowers/
dev_docs/
devplan.md
AGENTS.md
diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go
index d0892ed..8abb0de 100644
--- a/backend/cmd/server/main.go
+++ b/backend/cmd/server/main.go
@@ -6,6 +6,7 @@ import (
"net/http"
"os"
"os/signal"
+ "strings"
"syscall"
"time"
@@ -14,6 +15,7 @@ import (
"clawreef/internal/db"
"clawreef/internal/handlers"
"clawreef/internal/middleware"
+ "clawreef/internal/models"
"clawreef/internal/repository"
"clawreef/internal/services"
"clawreef/internal/services/k8s"
@@ -65,9 +67,14 @@ func main() {
instanceDesiredStateRepo := repository.NewInstanceDesiredStateRepository(database)
instanceCommandRepo := repository.NewInstanceCommandRepository(database)
instanceConfigRevisionRepo := repository.NewInstanceConfigRevisionRepository(database)
+ runtimePodRepo := repository.NewRuntimePodRepository(database)
+ bindingRepo := repository.NewInstanceRuntimeBindingRepository(database)
+ rolloutRepo := repository.NewRuntimeRolloutRepository(database)
+ workspaceFileAuditRepo := repository.NewWorkspaceFileAuditRepository(database)
teamRepo := repository.NewTeamRepository(database)
skillRepo := repository.NewSkillRepository(database)
securityScanRepo := repository.NewSecurityScanRepository(database)
+ instanceExternalAccessRepo := repository.NewInstanceExternalAccessRepository(database)
if repaired, repairErr := services.RepairSeededAdminPassword(userRepo); repairErr != nil {
log.Printf("Warning: failed to repair seeded admin password: %v", repairErr)
@@ -98,26 +105,53 @@ func main() {
aiObservabilityService := services.NewAIObservabilityService(modelInvocationRepo, auditEventRepo, costRecordRepo, riskHitRepo, chatMessageRepo, llmModelRepo, instanceRepo, userRepo)
clusterResourceService := services.NewClusterResourceService(instanceRepo)
services.SetRuntimeImageSettingsProvider(systemImageSettingService)
+ services.SetOpenClawTransferRuntimeRepositories(instanceRepo, bindingRepo, runtimePodRepo)
+ runtimeAgentClient := services.NewRuntimeAgentClient(cfg.Runtime.AgentControlToken)
instanceService := services.NewInstanceService(
instanceRepo,
quotaRepo,
llmModelRepo,
openClawConfigService,
services.WithPrivilegedInstancePods(cfg.Kubernetes.Runtime.Pod.Privileged),
+ services.WithV2RuntimeLifecycle(runtimePodRepo, bindingRepo, runtimeAgentClient, cfg.Runtime.WorkspaceRoot),
)
instanceAgentService := services.NewInstanceAgentService(instanceRepo, instanceAgentRepo, instanceDesiredStateRepo, instanceRuntimeStatusRepo, instanceCommandRepo)
instanceRuntimeStatusService := services.NewInstanceRuntimeStatusService(instanceRuntimeStatusRepo, instanceAgentRepo, instanceDesiredStateRepo)
instanceCommandService := services.NewInstanceCommandService(instanceCommandRepo, instanceRuntimeStatusRepo, instanceDesiredStateRepo)
instanceConfigRevisionService := services.NewInstanceConfigRevisionService(instanceConfigRevisionRepo)
- teamService := services.NewTeamService(teamRepo, instanceService)
+ teamService := services.NewTeamService(teamRepo, instanceService, services.WithTeamRuntimeWorkspaceRoot(cfg.Runtime.WorkspaceRoot))
+ var platformRedis services.PlatformRedisClient
+ if redisURL := strings.TrimSpace(cfg.Runtime.RedisURL); redisURL != "" {
+ var redisErr error
+ platformRedis, redisErr = services.NewPlatformRedisClient(redisURL)
+ if redisErr != nil {
+ log.Printf("platform redis disabled: %v", redisErr)
+ }
+ } else {
+ log.Printf("platform redis disabled: redis url is empty")
+ }
+ runtimeEvents := services.NewRuntimeEventService(platformRedis)
+ workspaceFileService := services.NewWorkspaceFileService(workspaceFileAuditRepo)
+ runtimeWorkspaceFileService := services.NewRuntimeWorkspaceFileService(workspaceFileAuditRepo)
skillService := services.NewSkillService(skillRepo, instanceRepo, instanceCommandService, objectStorageService, skillScannerClient)
securityScanService := services.NewSecurityScanService(securityScanRepo, skillRepo, objectStorageService, skillScannerClient)
+ externalAccessService := services.NewInstanceExternalAccessService(instanceExternalAccessRepo)
aiGatewayService := aigateway.NewService(llmModelRepo, modelInvocationService, auditEventService, costRecordService, riskDetectionService, riskHitService, chatSessionService, chatMessageService)
// Initialize handlers
authHandler := handlers.NewAuthHandler(authService)
userHandler := handlers.NewUserHandler(userService, quotaService)
- instanceHandler := handlers.NewInstanceHandler(instanceService, instanceAgentService, instanceRuntimeStatusService, instanceCommandService, instanceConfigRevisionService, openClawConfigService, skillService)
+ instanceHandler := handlers.NewInstanceHandler(
+ instanceService,
+ instanceAgentService,
+ instanceRuntimeStatusService,
+ instanceCommandService,
+ instanceConfigRevisionService,
+ openClawConfigService,
+ skillService,
+ externalAccessService,
+ services.WithInstanceProxyRuntimeRepositories(instanceRepo, runtimePodRepo, bindingRepo),
+ )
systemSettingsHandler := handlers.NewSystemSettingsHandler(systemImageSettingService)
llmModelHandler := handlers.NewLLMModelHandler(llmModelService)
aiGatewayHandler := handlers.NewAIGatewayHandler(aiGatewayService)
@@ -130,15 +164,65 @@ func main() {
securityHandler := handlers.NewSecurityHandler(securityScanService)
agentHandler := handlers.NewAgentHandler(instanceAgentService, instanceCommandService, instanceRuntimeStatusService, instanceConfigRevisionService, skillService)
teamHandler := handlers.NewTeamHandler(teamService)
+ workspaceFileHandler := handlers.NewWorkspaceFileHandler(instanceService, workspaceFileService, runtimeWorkspaceFileService)
+ runtimeAgentHandler := handlers.NewRuntimeAgentHandler(cfg.Runtime, runtimePodRepo, bindingRepo, runtimeEvents)
// Initialize WebSocket hub and handler
wsHub := services.GetHub()
wsHandler := handlers.NewWebSocketHandler(wsHub)
+ var runtimeAdminEventBridgeCancel context.CancelFunc
+ if platformRedis != nil {
+ var bridgeCtx context.Context
+ bridgeCtx, runtimeAdminEventBridgeCancel = context.WithCancel(context.Background())
+ services.StartRuntimeAdminEventBridge(bridgeCtx, runtimeEvents, wsHub)
+ }
// Start sync service to keep instance status in sync with K8s
syncService := services.NewSyncService(instanceRepo, instanceRuntimeStatusService)
syncService.Start()
teamService.Start()
+ var runtimeSchedulerCancel context.CancelFunc
+ var runtimeScheduler *services.RuntimeScheduler
+ if cfg.Runtime.SchedulerEnabled {
+ k8sClient := k8s.GetClient()
+ if k8sClient == nil || k8sClient.Clientset == nil {
+ log.Printf("runtime scheduler disabled: k8s client is unavailable")
+ } else {
+ leader := services.NewRuntimeLeaderService(k8sClient.Clientset, cfg.Runtime.Namespace, cfg.Runtime.BackendReplicaID)
+ runtimeDeployments := k8s.NewRuntimeDeploymentService(k8sClient.Clientset)
+ runtimeSchedulerOptions := []services.RuntimeSchedulerOption{
+ services.WithRuntimeSchedulerWorkspaceRoot(cfg.Runtime.WorkspaceRoot),
+ services.WithRuntimeSchedulerNamespace(cfg.Runtime.Namespace),
+ services.WithRuntimeSchedulerGatewayPortRange(cfg.Runtime.GatewayPortStart, cfg.Runtime.GatewayPortEnd),
+ services.WithRuntimeSchedulerHeartbeatTimeout(cfg.Runtime.HeartbeatTimeout),
+ services.WithRuntimeSchedulerMaxGatewaysPerPod(cfg.Runtime.MaxGatewaysPerPod),
+ }
+ if gatewayEnvProvider, ok := instanceService.(interface {
+ BuildGatewayEnv(*models.Instance) (map[string]string, error)
+ }); ok {
+ runtimeSchedulerOptions = append(runtimeSchedulerOptions, services.WithRuntimeSchedulerGatewayEnvBuilder(gatewayEnvProvider.BuildGatewayEnv))
+ }
+ runtimeScheduler = services.NewRuntimeScheduler(
+ instanceRepo,
+ runtimePodRepo,
+ bindingRepo,
+ rolloutRepo,
+ runtimeAgentClient,
+ runtimeEvents,
+ leader,
+ runtimeDeployments,
+ cfg.Runtime.SchedulerTick,
+ runtimeSchedulerOptions...,
+ )
+ var schedulerCtx context.Context
+ schedulerCtx, runtimeSchedulerCancel = context.WithCancel(context.Background())
+ runtimeScheduler.Start(schedulerCtx)
+ log.Printf("runtime scheduler started")
+ }
+ } else {
+ log.Printf("runtime scheduler disabled by configuration")
+ }
+ runtimePoolHandler := handlers.NewRuntimePoolHandler(runtimePodRepo, bindingRepo, rolloutRepo, runtimeScheduler, runtimeEvents)
// Setup router
r := gin.Default()
@@ -150,8 +234,20 @@ func main() {
r.NoMethod(egressProxyHandler.Handle)
// Routes
+ r.Any("/s/:code", instanceHandler.OpenShortExternalAccess)
+ r.Any("/s/:code/*path", instanceHandler.OpenShortExternalAccess)
+
api := r.Group("/api/v1")
{
+ runtimeAgent := api.Group("/runtime-agent")
+ {
+ runtimeAgent.POST("/register", runtimeAgentHandler.Register)
+ runtimeAgent.POST("/heartbeat", runtimeAgentHandler.Heartbeat)
+ runtimeAgent.POST("/metrics/report", runtimeAgentHandler.ReportMetrics)
+ runtimeAgent.POST("/gateways/report", runtimeAgentHandler.ReportGateways)
+ runtimeAgent.POST("/skills/report", runtimeAgentHandler.ReportSkills)
+ }
+
// Auth routes
auth := api.Group("/auth")
{
@@ -212,6 +308,17 @@ func main() {
instances.POST("/:id/openclaw/import", instanceHandler.ImportOpenClaw)
instances.GET("/:id/hermes/export", instanceHandler.ExportHermes)
instances.POST("/:id/hermes/import", instanceHandler.ImportHermes)
+ instances.GET("/:id/external-access", instanceHandler.GetExternalAccess)
+ instances.POST("/:id/external-access/share-link", instanceHandler.EnableShareLink)
+ instances.POST("/:id/external-access/password", instanceHandler.CreateExternalAccessPassword)
+ instances.DELETE("/:id/external-access", instanceHandler.DisableExternalAccess)
+ instances.GET("/:id/workspace/files", workspaceFileHandler.List)
+ instances.GET("/:id/workspace/preview", workspaceFileHandler.Preview)
+ instances.GET("/:id/workspace/download", workspaceFileHandler.Download)
+ instances.POST("/:id/workspace/upload", workspaceFileHandler.Upload)
+ instances.POST("/:id/workspace/folders", workspaceFileHandler.Mkdir)
+ instances.PATCH("/:id/workspace/entries", workspaceFileHandler.Rename)
+ instances.DELETE("/:id/workspace/entries", workspaceFileHandler.Delete)
instances.GET("/:id/skills", skillHandler.ListInstanceSkills)
instances.POST("/:id/skills", skillHandler.AttachSkillToInstance)
instances.DELETE("/:id/skills/:skillId", skillHandler.RemoveSkillFromInstance)
@@ -229,6 +336,17 @@ func main() {
adminInstances.GET("", instanceHandler.ListAllInstances)
}
+ adminRuntime := api.Group("/admin")
+ adminRuntime.Use(middleware.Auth())
+ adminRuntime.Use(middleware.SetUserInfo(userRepo))
+ adminRuntime.Use(middleware.NewAdminAuth(userRepo))
+ {
+ adminRuntime.GET("/runtime-pods", runtimePoolHandler.ListPods)
+ adminRuntime.GET("/runtime-pods/:id/gateways", runtimePoolHandler.GetPodGateways)
+ adminRuntime.POST("/runtime-pods/:id/drain", runtimePoolHandler.DrainPod)
+ adminRuntime.POST("/runtime-rollouts", runtimePoolHandler.StartRollout)
+ }
+
teams := api.Group("/teams")
teams.Use(middleware.Auth())
teams.Use(middleware.SetUserInfo(userRepo))
@@ -360,7 +478,7 @@ func main() {
}
gatewayLLM := api.Group("/gateway/llm")
- gatewayLLM.Use(middleware.GatewayAuth(instanceRepo))
+ gatewayLLM.Use(middleware.GatewayAuth(instanceRepo, bindingRepo))
{
gatewayLLM.GET("/models", aiGatewayHandler.ListModels)
gatewayLLM.POST("/chat/completions", aiGatewayHandler.ChatCompletions)
@@ -423,6 +541,12 @@ func main() {
}
// Stop background services
+ if runtimeSchedulerCancel != nil {
+ runtimeSchedulerCancel()
+ }
+ if runtimeAdminEventBridgeCancel != nil {
+ runtimeAdminEventBridgeCancel()
+ }
syncService.Stop()
teamService.Stop()
wsHub.Stop()
diff --git a/backend/deployments/k8s/clawreef-incluster.yaml b/backend/deployments/k8s/clawreef-incluster.yaml
index f306284..96c2c0e 100644
--- a/backend/deployments/k8s/clawreef-incluster.yaml
+++ b/backend/deployments/k8s/clawreef-incluster.yaml
@@ -413,8 +413,8 @@ data:
SELECT
'openclaw',
'shell',
- 'OpenClaw Shell',
- 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-shell:latest',
+ 'OpenClaw Lite',
+ 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest',
TRUE
WHERE NOT EXISTS (
SELECT 1
@@ -449,6 +449,39 @@ data:
updated_at = CURRENT_TIMESTAMP
WHERE type IN ('openclaw', 'ubuntu', 'webtop', 'hermes')
AND mount_path IN ('/data', '/home/user/data', '/config/.hermes');
+ 031_add_hermes_lite_runtime_image.sql: |
+ USE clawreef;
+ UPDATE system_image_settings
+ SET
+ image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest',
+ display_name = 'Hermes Lite',
+ is_enabled = TRUE
+ WHERE instance_type = 'hermes'
+ AND runtime_type = 'shell'
+ AND image IN (
+ 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-shell:latest',
+ 'registry.example.com/your-custom-shell-image:latest'
+ );
+
+ INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled)
+ SELECT
+ 'hermes',
+ 'shell',
+ 'Hermes Lite',
+ 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest',
+ TRUE
+ WHERE NOT EXISTS (
+ SELECT 1
+ FROM system_image_settings
+ WHERE instance_type = 'hermes'
+ AND runtime_type = 'shell'
+ )
+ AND NOT EXISTS (
+ SELECT 1
+ FROM system_image_settings
+ WHERE instance_type = 'hermes'
+ AND is_enabled = FALSE
+ );
---
apiVersion: v1
kind: PersistentVolumeClaim
diff --git a/backend/internal/aigateway/service.go b/backend/internal/aigateway/service.go
index bb515db..4fc84b1 100644
--- a/backend/internal/aigateway/service.go
+++ b/backend/internal/aigateway/service.go
@@ -72,6 +72,10 @@ type ChatCompletionRequest struct {
User *string `json:"user,omitempty"`
SessionID *string `json:"session_id,omitempty"`
InstanceID *int `json:"instance_id,omitempty"`
+ InstanceMode *string `json:"instance_mode,omitempty"`
+ RuntimeType *string `json:"runtime_type,omitempty"`
+ GatewayID *string `json:"gateway_id,omitempty"`
+ RuntimePodID *int64 `json:"runtime_pod_id,omitempty"`
TraceID *string `json:"trace_id,omitempty"`
RequestID *string `json:"request_id,omitempty"`
}
@@ -357,6 +361,10 @@ func (s *service) ChatCompletions(ctx context.Context, userID int, req ChatCompl
RequestID: prepared.requestIDPtr,
UserID: prepared.userIDPtr,
InstanceID: prepared.req.InstanceID,
+ InstanceMode: runtimeAttributionString(prepared.req.InstanceMode),
+ RuntimeType: runtimeAttributionString(prepared.req.RuntimeType),
+ GatewayID: runtimeAttributionString(prepared.req.GatewayID),
+ RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID),
EventType: "gateway.request.blocked",
TrafficClass: models.TrafficClassLLM,
Severity: models.AuditSeverityWarn,
@@ -388,6 +396,10 @@ func (s *service) StreamChatCompletions(ctx context.Context, userID int, req Cha
RequestID: prepared.requestIDPtr,
UserID: prepared.userIDPtr,
InstanceID: prepared.req.InstanceID,
+ InstanceMode: runtimeAttributionString(prepared.req.InstanceMode),
+ RuntimeType: runtimeAttributionString(prepared.req.RuntimeType),
+ GatewayID: runtimeAttributionString(prepared.req.GatewayID),
+ RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID),
EventType: "gateway.request.blocked",
TrafficClass: models.TrafficClassLLM,
Severity: models.AuditSeverityWarn,
@@ -449,6 +461,10 @@ func (s *service) prepareChatRequest(userID int, req ChatCompletionRequest) (*pr
RequestID: prepared.requestIDPtr,
UserID: prepared.userIDPtr,
InstanceID: prepared.req.InstanceID,
+ InstanceMode: runtimeAttributionString(prepared.req.InstanceMode),
+ RuntimeType: runtimeAttributionString(prepared.req.RuntimeType),
+ GatewayID: runtimeAttributionString(prepared.req.GatewayID),
+ RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID),
EventType: "gateway.request.received",
TrafficClass: models.TrafficClassLLM,
Severity: models.AuditSeverityInfo,
@@ -465,6 +481,10 @@ func (s *service) prepareChatRequest(userID int, req ChatCompletionRequest) (*pr
RequestID: prepared.requestIDPtr,
UserID: prepared.userIDPtr,
InstanceID: prepared.req.InstanceID,
+ InstanceMode: runtimeAttributionString(prepared.req.InstanceMode),
+ RuntimeType: runtimeAttributionString(prepared.req.RuntimeType),
+ GatewayID: runtimeAttributionString(prepared.req.GatewayID),
+ RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID),
EventType: "gateway.risk.detected",
TrafficClass: models.TrafficClassLLM,
Severity: models.AuditSeverityWarn,
@@ -477,7 +497,7 @@ func (s *service) prepareChatRequest(userID int, req ChatCompletionRequest) (*pr
resolvedModel, riskAction, resolveErr := s.resolveTargetModel(prepared.selectedModel, riskAnalysis)
if resolveErr != nil {
blockedInvocationID := s.recordBlockedInvocation(prepared.traceID, prepared.requestID, prepared.req, prepared.userID, prepared.selectedModel, resolveErr.Error())
- if err := s.riskHitService.RecordHits(prepared.traceID, prepared.req.SessionID, prepared.requestIDPtr, prepared.userIDPtr, prepared.req.InstanceID, blockedInvocationID, riskAction, riskAnalysis.Hits); err != nil {
+ if err := s.riskHitService.RecordHits(prepared.traceID, prepared.req.SessionID, prepared.requestIDPtr, prepared.userIDPtr, prepared.req.InstanceID, blockedInvocationID, riskHitAttribution(prepared.req), riskAction, riskAnalysis.Hits); err != nil {
logPersistenceError("record blocked risk hits", prepared.traceID, err)
}
if err := s.auditEventService.RecordEvent(&models.AuditEvent{
@@ -486,6 +506,10 @@ func (s *service) prepareChatRequest(userID int, req ChatCompletionRequest) (*pr
RequestID: prepared.requestIDPtr,
UserID: prepared.userIDPtr,
InstanceID: prepared.req.InstanceID,
+ InstanceMode: runtimeAttributionString(prepared.req.InstanceMode),
+ RuntimeType: runtimeAttributionString(prepared.req.RuntimeType),
+ GatewayID: runtimeAttributionString(prepared.req.GatewayID),
+ RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID),
InvocationID: blockedInvocationID,
EventType: "gateway.request.blocked",
TrafficClass: models.TrafficClassLLM,
@@ -498,7 +522,7 @@ func (s *service) prepareChatRequest(userID int, req ChatCompletionRequest) (*pr
}
if riskAnalysis.IsSensitive {
- if err := s.riskHitService.RecordHits(prepared.traceID, prepared.req.SessionID, prepared.requestIDPtr, prepared.userIDPtr, prepared.req.InstanceID, nil, riskAction, riskAnalysis.Hits); err != nil {
+ if err := s.riskHitService.RecordHits(prepared.traceID, prepared.req.SessionID, prepared.requestIDPtr, prepared.userIDPtr, prepared.req.InstanceID, nil, riskHitAttribution(prepared.req), riskAction, riskAnalysis.Hits); err != nil {
logPersistenceError("record risk hits", prepared.traceID, err)
}
if riskAction == models.RiskActionRouteSecureModel && resolvedModel != nil && resolvedModel.ID != prepared.selectedModel.ID {
@@ -508,6 +532,10 @@ func (s *service) prepareChatRequest(userID int, req ChatCompletionRequest) (*pr
RequestID: prepared.requestIDPtr,
UserID: prepared.userIDPtr,
InstanceID: prepared.req.InstanceID,
+ InstanceMode: runtimeAttributionString(prepared.req.InstanceMode),
+ RuntimeType: runtimeAttributionString(prepared.req.RuntimeType),
+ GatewayID: runtimeAttributionString(prepared.req.GatewayID),
+ RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID),
EventType: "gateway.request.rerouted",
TrafficClass: models.TrafficClassLLM,
Severity: models.AuditSeverityWarn,
@@ -655,6 +683,10 @@ func (s *service) recordFailure(traceID, requestID string, req ChatCompletionReq
RequestID: requestID,
UserID: intPtr(userID),
InstanceID: req.InstanceID,
+ InstanceMode: runtimeAttributionString(req.InstanceMode),
+ RuntimeType: runtimeAttributionString(req.RuntimeType),
+ GatewayID: runtimeAttributionString(req.GatewayID),
+ RuntimePodID: runtimeAttributionInt64(req.RuntimePodID),
ModelID: intPtr(model.ID),
ProviderType: model.ProviderType,
RequestedModel: req.Model,
@@ -679,6 +711,10 @@ func (s *service) recordFailure(traceID, requestID string, req ChatCompletionReq
RequestID: stringPtr(requestID),
UserID: intPtr(userID),
InstanceID: req.InstanceID,
+ InstanceMode: runtimeAttributionString(req.InstanceMode),
+ RuntimeType: runtimeAttributionString(req.RuntimeType),
+ GatewayID: runtimeAttributionString(req.GatewayID),
+ RuntimePodID: runtimeAttributionInt64(req.RuntimePodID),
InvocationID: intPtr(invocation.ID),
EventType: "gateway.request.failed",
TrafficClass: models.TrafficClassLLM,
@@ -703,6 +739,10 @@ func (s *service) recordSuccess(prepared *preparedChatRequest, providerRequestBo
RequestID: prepared.requestID,
UserID: prepared.userIDPtr,
InstanceID: prepared.req.InstanceID,
+ InstanceMode: runtimeAttributionString(prepared.req.InstanceMode),
+ RuntimeType: runtimeAttributionString(prepared.req.RuntimeType),
+ GatewayID: runtimeAttributionString(prepared.req.GatewayID),
+ RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID),
ModelID: intPtr(prepared.resolvedModel.ID),
ProviderType: prepared.resolvedModel.ProviderType,
RequestedModel: prepared.req.Model,
@@ -750,6 +790,10 @@ func (s *service) recordSuccess(prepared *preparedChatRequest, providerRequestBo
RequestID: prepared.requestIDPtr,
UserID: prepared.userIDPtr,
InstanceID: prepared.req.InstanceID,
+ InstanceMode: runtimeAttributionString(prepared.req.InstanceMode),
+ RuntimeType: runtimeAttributionString(prepared.req.RuntimeType),
+ GatewayID: runtimeAttributionString(prepared.req.GatewayID),
+ RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID),
InvocationID: intPtr(invocation.ID),
ModelID: intPtr(prepared.resolvedModel.ID),
ProviderType: prepared.resolvedModel.ProviderType,
@@ -772,6 +816,10 @@ func (s *service) recordSuccess(prepared *preparedChatRequest, providerRequestBo
RequestID: prepared.requestIDPtr,
UserID: prepared.userIDPtr,
InstanceID: prepared.req.InstanceID,
+ InstanceMode: runtimeAttributionString(prepared.req.InstanceMode),
+ RuntimeType: runtimeAttributionString(prepared.req.RuntimeType),
+ GatewayID: runtimeAttributionString(prepared.req.GatewayID),
+ RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID),
InvocationID: intPtr(invocation.ID),
EventType: "gateway.request.completed",
TrafficClass: models.TrafficClassLLM,
@@ -787,6 +835,10 @@ func (s *service) recordSuccess(prepared *preparedChatRequest, providerRequestBo
RequestID: prepared.requestIDPtr,
UserID: prepared.userIDPtr,
InstanceID: prepared.req.InstanceID,
+ InstanceMode: runtimeAttributionString(prepared.req.InstanceMode),
+ RuntimeType: runtimeAttributionString(prepared.req.RuntimeType),
+ GatewayID: runtimeAttributionString(prepared.req.GatewayID),
+ RuntimePodID: runtimeAttributionInt64(prepared.req.RuntimePodID),
InvocationID: intPtr(invocation.ID),
EventType: "gateway.usage.estimated",
TrafficClass: models.TrafficClassLLM,
@@ -1312,6 +1364,10 @@ func buildOpenAICompatibleRequestBody(req ChatCompletionRequest, model *models.L
}
delete(payload, "session_id")
delete(payload, "instance_id")
+ delete(payload, "instance_mode")
+ delete(payload, "runtime_type")
+ delete(payload, "gateway_id")
+ delete(payload, "runtime_pod_id")
delete(payload, "trace_id")
delete(payload, "request_id")
modelPayload, err := json.Marshal(model.ProviderModelName)
@@ -1847,6 +1903,10 @@ func (s *service) recordBlockedInvocation(traceID, requestID string, req ChatCom
RequestID: requestID,
UserID: intPtr(userID),
InstanceID: req.InstanceID,
+ InstanceMode: runtimeAttributionString(req.InstanceMode),
+ RuntimeType: runtimeAttributionString(req.RuntimeType),
+ GatewayID: runtimeAttributionString(req.GatewayID),
+ RuntimePodID: runtimeAttributionInt64(req.RuntimePodID),
ModelID: intPtr(model.ID),
ProviderType: model.ProviderType,
RequestedModel: req.Model,
@@ -2042,6 +2102,68 @@ func intPtr(value int) *int {
return &value
}
+func runtimeAttributionString(value *string) *string {
+ if value == nil {
+ return nil
+ }
+ trimmed := strings.TrimSpace(*value)
+ if trimmed == "" {
+ return nil
+ }
+ return &trimmed
+}
+
+func runtimeAttributionInt64(value *int64) *int64 {
+ if value == nil || *value <= 0 {
+ return nil
+ }
+ copied := *value
+ return &copied
+}
+
+func applyInvocationRuntimeAttribution(invocation *models.ModelInvocation, req ChatCompletionRequest) {
+ if invocation == nil {
+ return
+ }
+ invocation.InstanceMode = runtimeAttributionString(req.InstanceMode)
+ invocation.RuntimeType = runtimeAttributionString(req.RuntimeType)
+ invocation.GatewayID = runtimeAttributionString(req.GatewayID)
+ invocation.RuntimePodID = runtimeAttributionInt64(req.RuntimePodID)
+}
+
+func applyAuditRuntimeAttribution(event *models.AuditEvent, req ChatCompletionRequest) {
+ if event == nil {
+ return
+ }
+ event.InstanceMode = runtimeAttributionString(req.InstanceMode)
+ event.RuntimeType = runtimeAttributionString(req.RuntimeType)
+ event.GatewayID = runtimeAttributionString(req.GatewayID)
+ event.RuntimePodID = runtimeAttributionInt64(req.RuntimePodID)
+}
+
+func applyCostRuntimeAttribution(record *models.CostRecord, req ChatCompletionRequest) {
+ if record == nil {
+ return
+ }
+ record.InstanceMode = runtimeAttributionString(req.InstanceMode)
+ record.RuntimeType = runtimeAttributionString(req.RuntimeType)
+ record.GatewayID = runtimeAttributionString(req.GatewayID)
+ record.RuntimePodID = runtimeAttributionInt64(req.RuntimePodID)
+}
+
+func riskHitAttribution(req ChatCompletionRequest) *services.RiskHitAttribution {
+ attribution := &services.RiskHitAttribution{
+ InstanceMode: runtimeAttributionString(req.InstanceMode),
+ RuntimeType: runtimeAttributionString(req.RuntimeType),
+ GatewayID: runtimeAttributionString(req.GatewayID),
+ RuntimePodID: runtimeAttributionInt64(req.RuntimePodID),
+ }
+ if attribution.InstanceMode == nil && attribution.RuntimeType == nil && attribution.GatewayID == nil && attribution.RuntimePodID == nil {
+ return nil
+ }
+ return attribution
+}
+
func userIDOrZero(value *int) int {
if value == nil {
return 0
diff --git a/backend/internal/aigateway/service_test.go b/backend/internal/aigateway/service_test.go
index 32f00b7..18772a7 100644
--- a/backend/internal/aigateway/service_test.go
+++ b/backend/internal/aigateway/service_test.go
@@ -47,7 +47,7 @@ func (s *stubChatSessionService) EnsureSession(sessionID string, userID, instanc
func TestBuildProviderRequestPreservesToolConfiguration(t *testing.T) {
req := ChatCompletionRequest{
Model: "gateway-model",
- RawBody: []byte(`{"model":"gateway-model","messages":[{"role":"user","content":[{"type":"text","text":"weather in shanghai"}]}],"tools":[{"type":"function","function":{"name":"get_weather","parameters":{"type":"object"}}}],"tool_choice":{"type":"function","function":{"name":"get_weather"}},"stream":true,"stream_options":{"include_usage":false,"custom":"value"},"custom_field":{"nested":[1,2,3]},"session_id":"sess_123","request_id":"req_123","trace_id":"trc_123","instance_id":99}`),
+ RawBody: []byte(`{"model":"gateway-model","messages":[{"role":"user","content":[{"type":"text","text":"weather in shanghai"}]}],"tools":[{"type":"function","function":{"name":"get_weather","parameters":{"type":"object"}}}],"tool_choice":{"type":"function","function":{"name":"get_weather"}},"stream":true,"stream_options":{"include_usage":false,"custom":"value"},"custom_field":{"nested":[1,2,3]},"session_id":"sess_123","request_id":"req_123","trace_id":"trc_123","instance_id":99,"instance_mode":"lite","runtime_type":"gateway","gateway_id":"gw_123","runtime_pod_id":5}`),
}
model := &models.LLMModel{
@@ -95,6 +95,18 @@ func TestBuildProviderRequestPreservesToolConfiguration(t *testing.T) {
if _, ok := payload["instance_id"]; ok {
t.Fatalf("expected internal instance_id to be stripped")
}
+ if _, ok := payload["instance_mode"]; ok {
+ t.Fatalf("expected internal instance_mode to be stripped")
+ }
+ if _, ok := payload["runtime_type"]; ok {
+ t.Fatalf("expected internal runtime_type to be stripped")
+ }
+ if _, ok := payload["gateway_id"]; ok {
+ t.Fatalf("expected internal gateway_id to be stripped")
+ }
+ if _, ok := payload["runtime_pod_id"]; ok {
+ t.Fatalf("expected internal runtime_pod_id to be stripped")
+ }
if string(payload["stream_options"]) != `{"include_usage":false,"custom":"value"}` {
t.Fatalf("expected stream_options to pass through unchanged, got %s", string(payload["stream_options"]))
}
@@ -135,6 +147,48 @@ func TestBuildProviderRequestUsesAnthropicProtocolForLocalModel(t *testing.T) {
}
}
+func TestRuntimeAttributionHelpersPopulateRecords(t *testing.T) {
+ mode := "lite"
+ runtimeType := "gateway"
+ gatewayID := "gw_instance_1"
+ runtimePodID := int64(17)
+ req := ChatCompletionRequest{
+ InstanceMode: &mode,
+ RuntimeType: &runtimeType,
+ GatewayID: &gatewayID,
+ RuntimePodID: &runtimePodID,
+ }
+
+ invocation := &models.ModelInvocation{}
+ applyInvocationRuntimeAttribution(invocation, req)
+ if invocation.InstanceMode == nil || *invocation.InstanceMode != mode {
+ t.Fatalf("invocation instance mode not populated")
+ }
+ if invocation.GatewayID == nil || *invocation.GatewayID != gatewayID {
+ t.Fatalf("invocation gateway ID not populated")
+ }
+
+ audit := &models.AuditEvent{}
+ applyAuditRuntimeAttribution(audit, req)
+ if audit.RuntimeType == nil || *audit.RuntimeType != runtimeType {
+ t.Fatalf("audit runtime type not populated")
+ }
+ if audit.RuntimePodID == nil || *audit.RuntimePodID != runtimePodID {
+ t.Fatalf("audit runtime pod ID not populated")
+ }
+
+ cost := &models.CostRecord{}
+ applyCostRuntimeAttribution(cost, req)
+ if cost.GatewayID == nil || *cost.GatewayID != gatewayID {
+ t.Fatalf("cost gateway ID not populated")
+ }
+
+ risk := riskHitAttribution(req)
+ if risk == nil || risk.GatewayID == nil || *risk.GatewayID != gatewayID {
+ t.Fatalf("risk attribution gateway ID not populated")
+ }
+}
+
func TestRewriteStreamLineKeepsToolCalls(t *testing.T) {
line := "data: {\"id\":\"chatcmpl-1\",\"model\":\"provider-model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"id\":\"call_1\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"{\\\"city\\\":\\\"Shanghai\\\"}\"}}]},\"finish_reason\":null}]}\n"
diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go
index b9560ba..84bc033 100644
--- a/backend/internal/config/config.go
+++ b/backend/internal/config/config.go
@@ -3,7 +3,9 @@ package config
import (
"fmt"
"os"
+ "strconv"
"strings"
+ "time"
"gopkg.in/yaml.v3"
)
@@ -14,6 +16,7 @@ type Config struct {
Database DatabaseConfig `yaml:"database"`
JWT JWTConfig `yaml:"jwt"`
Kubernetes KubernetesConfig `yaml:"kubernetes"`
+ Runtime RuntimePoolConfig `yaml:"runtime"`
ObjectStorage ObjectStorageConfig `yaml:"objectStorage"`
SkillScanner SkillScannerConfig `yaml:"skillScanner"`
}
@@ -42,12 +45,12 @@ type JWTConfig struct {
// KubernetesConfig holds Kubernetes-related configuration
type KubernetesConfig struct {
- Mode string `yaml:"mode"` // 连接模式: auto, incluster, outofcluster
- OutOfCluster OutOfClusterConfig `yaml:"outOfCluster"`
- InCluster InClusterConfig `yaml:"inCluster"`
- Common CommonKubernetesConfig `yaml:"common"`
- Runtime RuntimeConfig `yaml:"runtime"`
- Logging LoggingConfig `yaml:"logging"`
+ Mode string `yaml:"mode"` // 连接模式: auto, incluster, outofcluster
+ OutOfCluster OutOfClusterConfig `yaml:"outOfCluster"`
+ InCluster InClusterConfig `yaml:"inCluster"`
+ Common CommonKubernetesConfig `yaml:"common"`
+ Runtime KubernetesRuntimeConfig `yaml:"runtime"`
+ Logging LoggingConfig `yaml:"logging"`
}
// OutOfClusterConfig holds out-of-cluster Kubernetes configuration
@@ -82,8 +85,8 @@ type CommonKubernetesConfig struct {
AutoCreateNamespace bool `yaml:"autoCreateNamespace"`
}
-// RuntimeConfig holds runtime configuration
-type RuntimeConfig struct {
+// KubernetesRuntimeConfig holds Kubernetes runtime configuration
+type KubernetesRuntimeConfig struct {
Pod RuntimePodConfig `yaml:"pod"`
PVC RuntimePVCConfig `yaml:"pvc"`
}
@@ -116,6 +119,26 @@ type RuntimePVCConfig struct {
HostPathPrefix string `yaml:"hostPathPrefix"`
}
+// RuntimePoolConfig holds shared V2 runtime pool configuration.
+type RuntimePoolConfig struct {
+ Namespace string `yaml:"namespace"`
+ WorkspaceRoot string `yaml:"workspaceRoot"`
+ WorkspaceNFSServer string `yaml:"workspaceNfsServer"`
+ WorkspaceNFSPath string `yaml:"workspaceNfsPath"`
+ AgentControlToken string `yaml:"agentControlToken"`
+ AgentReportToken string `yaml:"agentReportToken"`
+ BackendReplicaID string `yaml:"backendReplicaId"`
+ RedisURL string `yaml:"redisUrl"`
+ SchedulerEnabled bool `yaml:"schedulerEnabled"`
+ HeartbeatTimeout time.Duration `yaml:"heartbeatTimeout"`
+ SchedulerTick time.Duration `yaml:"schedulerTick"`
+ OpenClawImage string `yaml:"openClawImage"`
+ HermesImage string `yaml:"hermesImage"`
+ MaxGatewaysPerPod int `yaml:"maxGatewaysPerPod"`
+ GatewayPortStart int `yaml:"gatewayPortStart"`
+ GatewayPortEnd int `yaml:"gatewayPortEnd"`
+}
+
// LoggingConfig holds logging configuration
type LoggingConfig struct {
Level string `yaml:"level"`
@@ -135,14 +158,15 @@ type ObjectStorageConfig struct {
}
type SkillScannerConfig struct {
- BaseURL string `yaml:"baseUrl"`
- APIKey string `yaml:"apiKey"`
- TimeoutSeconds int `yaml:"timeoutSeconds"`
- Enabled bool `yaml:"enabled"`
+ BaseURL string `yaml:"baseUrl"`
+ APIKey string `yaml:"apiKey"`
+ TimeoutSeconds int `yaml:"timeoutSeconds"`
+ Enabled bool `yaml:"enabled"`
}
// Load loads configuration from file and environment variables
func Load() (*Config, error) {
+ runtimeNamespace := getEnv("RUNTIME_NAMESPACE", getEnv("K8S_NAMESPACE", "clawmanager-system"))
config := &Config{
Server: ServerConfig{
Address: ":9001",
@@ -177,7 +201,7 @@ func Load() (*Config, error) {
RetryCount: 3,
AutoCreateNamespace: true,
},
- Runtime: RuntimeConfig{
+ Runtime: KubernetesRuntimeConfig{
Pod: RuntimePodConfig{
ImageRegistry: "docker.io/clawreef",
ContainerPort: 3001,
@@ -199,6 +223,24 @@ func Load() (*Config, error) {
LogAPICalls: false,
},
},
+ Runtime: RuntimePoolConfig{
+ Namespace: runtimeNamespace,
+ WorkspaceRoot: getEnv("RUNTIME_WORKSPACE_ROOT", "/workspaces"),
+ WorkspaceNFSServer: getEnv("RUNTIME_WORKSPACE_NFS_SERVER", defaultWorkspaceNFSServer(runtimeNamespace)),
+ WorkspaceNFSPath: getEnv("RUNTIME_WORKSPACE_NFS_PATH", "/"),
+ AgentControlToken: getEnv("RUNTIME_AGENT_CONTROL_TOKEN", ""),
+ AgentReportToken: getEnv("RUNTIME_AGENT_REPORT_TOKEN", ""),
+ BackendReplicaID: getEnv("HOSTNAME", "clawmanager-backend-local"),
+ RedisURL: getEnv("PLATFORM_REDIS_URL", getEnv("TEAM_REDIS_URL", "")),
+ SchedulerEnabled: getEnvBool("RUNTIME_SCHEDULER_ENABLED", true),
+ HeartbeatTimeout: getEnvDuration("RUNTIME_HEARTBEAT_TIMEOUT", 10*time.Second),
+ SchedulerTick: getEnvDuration("RUNTIME_SCHEDULER_TICK", 2*time.Second),
+ OpenClawImage: getEnv("OPENCLAW_RUNTIME_IMAGE", "ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest"),
+ HermesImage: getEnv("HERMES_RUNTIME_IMAGE", "ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest"),
+ MaxGatewaysPerPod: getEnvInt("RUNTIME_MAX_GATEWAYS_PER_POD", 100),
+ GatewayPortStart: getEnvInt("RUNTIME_GATEWAY_PORT_START", 20000),
+ GatewayPortEnd: getEnvInt("RUNTIME_GATEWAY_PORT_END", 20099),
+ },
ObjectStorage: ObjectStorageConfig{
Endpoint: getEnv("OBJECT_STORAGE_ENDPOINT", ""),
Region: getEnv("OBJECT_STORAGE_REGION", ""),
@@ -211,10 +253,10 @@ func Load() (*Config, error) {
LocalFallback: getEnv("OBJECT_STORAGE_LOCAL_FALLBACK", ".data/object-storage"),
},
SkillScanner: SkillScannerConfig{
- BaseURL: getEnv("SKILL_SCANNER_BASE_URL", ""),
- APIKey: getEnv("SKILL_SCANNER_API_KEY", ""),
+ BaseURL: getEnv("SKILL_SCANNER_BASE_URL", ""),
+ APIKey: getEnv("SKILL_SCANNER_API_KEY", ""),
TimeoutSeconds: 30,
- Enabled: strings.EqualFold(getEnv("SKILL_SCANNER_ENABLED", "false"), "true"),
+ Enabled: strings.EqualFold(getEnv("SKILL_SCANNER_ENABLED", "false"), "true"),
},
}
@@ -308,6 +350,29 @@ func applyEnvOverrides(config *Config) {
config.Kubernetes.Runtime.PVC.HostPathPrefix = hostPathPrefix
}
+ config.Runtime.Namespace = getEnv("RUNTIME_NAMESPACE", getEnv("K8S_NAMESPACE", config.Runtime.Namespace))
+ config.Runtime.WorkspaceRoot = getEnv("RUNTIME_WORKSPACE_ROOT", config.Runtime.WorkspaceRoot)
+ config.Runtime.WorkspaceNFSServer = getEnv("RUNTIME_WORKSPACE_NFS_SERVER", config.Runtime.WorkspaceNFSServer)
+ config.Runtime.WorkspaceNFSPath = getEnv("RUNTIME_WORKSPACE_NFS_PATH", config.Runtime.WorkspaceNFSPath)
+ if strings.TrimSpace(config.Runtime.WorkspaceNFSServer) == "" {
+ config.Runtime.WorkspaceNFSServer = defaultWorkspaceNFSServer(config.Runtime.Namespace)
+ }
+ if strings.TrimSpace(config.Runtime.WorkspaceNFSPath) == "" {
+ config.Runtime.WorkspaceNFSPath = "/"
+ }
+ config.Runtime.AgentControlToken = getEnv("RUNTIME_AGENT_CONTROL_TOKEN", config.Runtime.AgentControlToken)
+ config.Runtime.AgentReportToken = getEnv("RUNTIME_AGENT_REPORT_TOKEN", config.Runtime.AgentReportToken)
+ config.Runtime.BackendReplicaID = getEnv("HOSTNAME", config.Runtime.BackendReplicaID)
+ config.Runtime.RedisURL = getEnv("PLATFORM_REDIS_URL", getEnv("TEAM_REDIS_URL", config.Runtime.RedisURL))
+ config.Runtime.SchedulerEnabled = getEnvBool("RUNTIME_SCHEDULER_ENABLED", config.Runtime.SchedulerEnabled)
+ config.Runtime.HeartbeatTimeout = getEnvDuration("RUNTIME_HEARTBEAT_TIMEOUT", config.Runtime.HeartbeatTimeout)
+ config.Runtime.SchedulerTick = getEnvDuration("RUNTIME_SCHEDULER_TICK", config.Runtime.SchedulerTick)
+ config.Runtime.OpenClawImage = getEnv("OPENCLAW_RUNTIME_IMAGE", config.Runtime.OpenClawImage)
+ config.Runtime.HermesImage = getEnv("HERMES_RUNTIME_IMAGE", config.Runtime.HermesImage)
+ config.Runtime.MaxGatewaysPerPod = getEnvInt("RUNTIME_MAX_GATEWAYS_PER_POD", config.Runtime.MaxGatewaysPerPod)
+ config.Runtime.GatewayPortStart = getEnvInt("RUNTIME_GATEWAY_PORT_START", config.Runtime.GatewayPortStart)
+ config.Runtime.GatewayPortEnd = getEnvInt("RUNTIME_GATEWAY_PORT_END", config.Runtime.GatewayPortEnd)
+
if endpoint := os.Getenv("OBJECT_STORAGE_ENDPOINT"); endpoint != "" {
config.ObjectStorage.Endpoint = endpoint
}
@@ -356,6 +421,50 @@ func getEnv(key, defaultValue string) string {
return defaultValue
}
+func defaultWorkspaceNFSServer(namespace string) string {
+ namespace = strings.TrimSpace(namespace)
+ if namespace == "" {
+ namespace = "clawmanager-system"
+ }
+ return fmt.Sprintf("workspace-store.%s.svc.cluster.local", namespace)
+}
+
+func getEnvInt(key string, defaultValue int) int {
+ value := os.Getenv(key)
+ if value == "" {
+ return defaultValue
+ }
+ parsed, err := strconv.Atoi(value)
+ if err != nil {
+ return defaultValue
+ }
+ return parsed
+}
+
+func getEnvBool(key string, defaultValue bool) bool {
+ value := os.Getenv(key)
+ if value == "" {
+ return defaultValue
+ }
+ parsed, err := strconv.ParseBool(value)
+ if err != nil {
+ return defaultValue
+ }
+ return parsed
+}
+
+func getEnvDuration(key string, defaultValue time.Duration) time.Duration {
+ value := os.Getenv(key)
+ if value == "" {
+ return defaultValue
+ }
+ parsed, err := time.ParseDuration(value)
+ if err != nil {
+ return defaultValue
+ }
+ return parsed
+}
+
// GetKubeconfigPath returns the kubeconfig path for out-of-cluster mode
func (c *Config) GetKubeconfigPath() string {
return c.Kubernetes.OutOfCluster.Kubeconfig
diff --git a/backend/internal/config/config_test.go b/backend/internal/config/config_test.go
new file mode 100644
index 0000000..7df0dcc
--- /dev/null
+++ b/backend/internal/config/config_test.go
@@ -0,0 +1,29 @@
+package config
+
+import "testing"
+
+func TestLoadRuntimeDefaults(t *testing.T) {
+ for _, key := range []string{
+ "HERMES_RUNTIME_IMAGE",
+ "OPENCLAW_RUNTIME_IMAGE",
+ "RUNTIME_NAMESPACE",
+ "K8S_NAMESPACE",
+ "HOSTNAME",
+ "PLATFORM_REDIS_URL",
+ "TEAM_REDIS_URL",
+ } {
+ t.Setenv(key, "")
+ }
+
+ cfg, err := Load()
+ if err != nil {
+ t.Fatalf("Load returned error: %v", err)
+ }
+
+ if got, want := cfg.Runtime.HermesImage, "ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest"; got != want {
+ t.Fatalf("expected Hermes default image %q, got %q", want, got)
+ }
+ if got, want := cfg.Runtime.OpenClawImage, "ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest"; got != want {
+ t.Fatalf("expected OpenClaw default image %q, got %q", want, got)
+ }
+}
diff --git a/backend/internal/db/migrations/021_add_openclaw_shell_runtime_image.sql b/backend/internal/db/migrations/021_add_openclaw_shell_runtime_image.sql
index 4e8f129..eddf004 100644
--- a/backend/internal/db/migrations/021_add_openclaw_shell_runtime_image.sql
+++ b/backend/internal/db/migrations/021_add_openclaw_shell_runtime_image.sql
@@ -61,8 +61,8 @@ INSERT INTO system_image_settings (instance_type, runtime_type, display_name, im
SELECT
'openclaw',
'shell',
- 'OpenClaw Shell',
- 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-shell:latest',
+ 'OpenClaw Lite',
+ 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest',
TRUE
WHERE NOT EXISTS (
SELECT 1
diff --git a/backend/internal/db/migrations/023_add_runtime_pool_v2.sql b/backend/internal/db/migrations/023_add_runtime_pool_v2.sql
new file mode 100644
index 0000000..76838d0
--- /dev/null
+++ b/backend/internal/db/migrations/023_add_runtime_pool_v2.sql
@@ -0,0 +1,121 @@
+SET @stmt = IF(
+ (SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'instances' AND COLUMN_NAME = 'workspace_path') = 0,
+ 'ALTER TABLE instances ADD COLUMN workspace_path VARCHAR(1024) NULL AFTER mount_path',
+ 'SELECT 1'
+);
+PREPARE stmt FROM @stmt;
+EXECUTE stmt;
+DEALLOCATE PREPARE stmt;
+
+SET @stmt = IF(
+ (SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'instances' AND COLUMN_NAME = 'workspace_usage_bytes') = 0,
+ 'ALTER TABLE instances ADD COLUMN workspace_usage_bytes BIGINT NOT NULL DEFAULT 0 AFTER workspace_path',
+ 'SELECT 1'
+);
+PREPARE stmt FROM @stmt;
+EXECUTE stmt;
+DEALLOCATE PREPARE stmt;
+
+SET @stmt = IF(
+ (SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'instances' AND COLUMN_NAME = 'runtime_generation') = 0,
+ 'ALTER TABLE instances ADD COLUMN runtime_generation INT NOT NULL DEFAULT 1 AFTER workspace_usage_bytes',
+ 'SELECT 1'
+);
+PREPARE stmt FROM @stmt;
+EXECUTE stmt;
+DEALLOCATE PREPARE stmt;
+
+SET @stmt = IF(
+ (SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'instances' AND COLUMN_NAME = 'runtime_error_message') = 0,
+ 'ALTER TABLE instances ADD COLUMN runtime_error_message TEXT NULL AFTER runtime_generation',
+ 'SELECT 1'
+);
+PREPARE stmt FROM @stmt;
+EXECUTE stmt;
+DEALLOCATE PREPARE stmt;
+
+ALTER TABLE instances
+ MODIFY COLUMN runtime_type ENUM('desktop', 'shell', 'gateway') NOT NULL DEFAULT 'desktop';
+
+CREATE TABLE IF NOT EXISTS runtime_pods (
+ id BIGINT PRIMARY KEY AUTO_INCREMENT,
+ runtime_type VARCHAR(32) NOT NULL,
+ namespace VARCHAR(128) NOT NULL,
+ pod_name VARCHAR(255) NOT NULL,
+ pod_uid VARCHAR(128) NULL,
+ pod_ip VARCHAR(64) NULL,
+ node_name VARCHAR(255) NULL,
+ deployment_name VARCHAR(255) NOT NULL,
+ image_ref VARCHAR(512) NOT NULL,
+ agent_endpoint VARCHAR(255) NULL,
+ state VARCHAR(32) NOT NULL DEFAULT 'pending',
+ capacity INT NOT NULL DEFAULT 100,
+ used_slots INT NOT NULL DEFAULT 0,
+ draining TINYINT(1) NOT NULL DEFAULT 0,
+ cpu_millis_used BIGINT NOT NULL DEFAULT 0,
+ memory_bytes_used BIGINT NOT NULL DEFAULT 0,
+ disk_bytes_used BIGINT NOT NULL DEFAULT 0,
+ network_rx_bytes BIGINT NOT NULL DEFAULT 0,
+ network_tx_bytes BIGINT NOT NULL DEFAULT 0,
+ metrics_json JSON NULL,
+ last_seen_at DATETIME NULL,
+ created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+ UNIQUE KEY uk_runtime_pod_namespace_name (namespace, pod_name),
+ KEY idx_runtime_pods_schedulable (runtime_type, state, draining, used_slots, capacity),
+ KEY idx_runtime_pods_last_seen (last_seen_at)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+CREATE TABLE IF NOT EXISTS instance_runtime_bindings (
+ id BIGINT PRIMARY KEY AUTO_INCREMENT,
+ instance_id INT NOT NULL,
+ runtime_pod_id BIGINT NOT NULL,
+ runtime_type VARCHAR(32) NOT NULL,
+ gateway_id VARCHAR(128) NOT NULL,
+ gateway_port INT NOT NULL,
+ gateway_pid INT NULL,
+ workspace_path VARCHAR(1024) NOT NULL,
+ state VARCHAR(32) NOT NULL DEFAULT 'creating',
+ generation INT NOT NULL DEFAULT 1,
+ last_health_at DATETIME NULL,
+ error_message TEXT NULL,
+ created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+ UNIQUE KEY uk_instance_runtime_binding_instance (instance_id),
+ UNIQUE KEY uk_instance_runtime_binding_gateway (runtime_pod_id, gateway_port),
+ KEY idx_instance_runtime_binding_pod_state (runtime_pod_id, state),
+ CONSTRAINT fk_instance_runtime_binding_instance
+ FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE,
+ CONSTRAINT fk_instance_runtime_binding_pod
+ FOREIGN KEY (runtime_pod_id) REFERENCES runtime_pods(id) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+CREATE TABLE IF NOT EXISTS runtime_rollouts (
+ id BIGINT PRIMARY KEY AUTO_INCREMENT,
+ runtime_type VARCHAR(32) NOT NULL,
+ target_image_ref VARCHAR(512) NOT NULL,
+ status VARCHAR(32) NOT NULL DEFAULT 'pending',
+ batch_size INT NOT NULL DEFAULT 1,
+ max_unavailable INT NOT NULL DEFAULT 1,
+ started_by INT NULL,
+ started_at DATETIME NULL,
+ finished_at DATETIME NULL,
+ error_message TEXT NULL,
+ created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+ KEY idx_runtime_rollouts_type_status (runtime_type, status)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+CREATE TABLE IF NOT EXISTS workspace_file_audits (
+ id BIGINT PRIMARY KEY AUTO_INCREMENT,
+ instance_id INT NOT NULL,
+ user_id INT NOT NULL,
+ action VARCHAR(32) NOT NULL,
+ relative_path VARCHAR(1024) NOT NULL,
+ bytes BIGINT NOT NULL DEFAULT 0,
+ created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ KEY idx_workspace_file_audits_instance_time (instance_id, created_at),
+ KEY idx_workspace_file_audits_user_time (user_id, created_at),
+ CONSTRAINT fk_workspace_file_audits_instance
+ FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
diff --git a/backend/internal/db/migrations/024_add_gateway_runtime_type.sql b/backend/internal/db/migrations/024_add_gateway_runtime_type.sql
new file mode 100644
index 0000000..82f5f5d
--- /dev/null
+++ b/backend/internal/db/migrations/024_add_gateway_runtime_type.sql
@@ -0,0 +1,2 @@
+ALTER TABLE instances
+ MODIFY COLUMN runtime_type ENUM('desktop', 'shell', 'gateway') NOT NULL DEFAULT 'desktop';
diff --git a/backend/internal/db/migrations/025_add_instance_mode.sql b/backend/internal/db/migrations/025_add_instance_mode.sql
new file mode 100644
index 0000000..8a24cf6
--- /dev/null
+++ b/backend/internal/db/migrations/025_add_instance_mode.sql
@@ -0,0 +1,24 @@
+SET @instance_mode_column_exists = (
+ SELECT COUNT(*)
+ FROM information_schema.COLUMNS
+ WHERE TABLE_SCHEMA = DATABASE()
+ AND TABLE_NAME = 'instances'
+ AND COLUMN_NAME = 'instance_mode'
+);
+SET @instance_mode_column_sql = IF(
+ @instance_mode_column_exists = 0,
+ 'ALTER TABLE instances ADD COLUMN instance_mode ENUM(''lite'', ''pro'') NOT NULL DEFAULT ''lite'' AFTER runtime_type',
+ 'SELECT 1'
+);
+PREPARE instance_mode_column_stmt FROM @instance_mode_column_sql;
+EXECUTE instance_mode_column_stmt;
+DEALLOCATE PREPARE instance_mode_column_stmt;
+
+UPDATE instances
+SET instance_mode = CASE
+ WHEN runtime_type = 'gateway' THEN 'lite'
+ ELSE 'pro'
+END
+WHERE instance_mode IS NULL
+ OR instance_mode = ''
+ OR (runtime_type <> 'gateway' AND instance_mode = 'lite');
diff --git a/backend/internal/db/migrations/027_add_instance_external_access.sql b/backend/internal/db/migrations/027_add_instance_external_access.sql
new file mode 100644
index 0000000..18208c0
--- /dev/null
+++ b/backend/internal/db/migrations/027_add_instance_external_access.sql
@@ -0,0 +1,23 @@
+CREATE TABLE IF NOT EXISTS instance_external_access (
+ id BIGINT PRIMARY KEY AUTO_INCREMENT,
+ instance_id INT NOT NULL,
+ enabled TINYINT(1) NOT NULL DEFAULT 0,
+ auth_mode VARCHAR(32) NOT NULL DEFAULT 'public_link',
+ public_slug VARCHAR(64) NULL,
+ public_token_hash VARCHAR(128) NULL,
+ short_code_hash VARCHAR(128) NULL,
+ api_key_hash VARCHAR(128) NULL,
+ password_value VARCHAR(128) NULL,
+ api_key_prefix VARCHAR(32) NULL,
+ expires_at DATETIME NULL,
+ created_by INT NULL,
+ last_used_at DATETIME NULL,
+ created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+ UNIQUE KEY uk_instance_external_access_instance (instance_id),
+ UNIQUE KEY uk_instance_external_access_slug (public_slug),
+ UNIQUE KEY uk_instance_external_access_short_code (short_code_hash),
+ KEY idx_instance_external_access_enabled (enabled, auth_mode),
+ CONSTRAINT fk_instance_external_access_instance
+ FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
diff --git a/backend/internal/db/migrations/028_add_ai_gateway_runtime_attribution.sql b/backend/internal/db/migrations/028_add_ai_gateway_runtime_attribution.sql
new file mode 100644
index 0000000..1f5b6cc
--- /dev/null
+++ b/backend/internal/db/migrations/028_add_ai_gateway_runtime_attribution.sql
@@ -0,0 +1,266 @@
+CREATE TABLE IF NOT EXISTS model_invocations (
+ id INT AUTO_INCREMENT PRIMARY KEY,
+ trace_id VARCHAR(100) NOT NULL,
+ session_id VARCHAR(100) NULL,
+ request_id VARCHAR(100) NOT NULL,
+ user_id INT NULL,
+ instance_id INT NULL,
+ instance_mode VARCHAR(16) NULL,
+ runtime_type VARCHAR(32) NULL,
+ gateway_id VARCHAR(128) NULL,
+ runtime_pod_id BIGINT NULL,
+ model_id INT NULL,
+ provider_type VARCHAR(100) NOT NULL,
+ requested_model VARCHAR(255) NOT NULL,
+ actual_provider_model VARCHAR(255) NOT NULL,
+ traffic_class VARCHAR(50) NOT NULL,
+ request_payload LONGTEXT NULL,
+ response_payload LONGTEXT NULL,
+ prompt_tokens INT NOT NULL DEFAULT 0,
+ completion_tokens INT NOT NULL DEFAULT 0,
+ total_tokens INT NOT NULL DEFAULT 0,
+ cached_tokens INT NULL,
+ reasoning_tokens INT NULL,
+ latency_ms INT NULL,
+ is_streaming BOOLEAN NOT NULL DEFAULT FALSE,
+ status VARCHAR(50) NOT NULL,
+ error_message TEXT NULL,
+ created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+ completed_at TIMESTAMP NULL,
+ INDEX idx_model_invocations_trace_id (trace_id),
+ INDEX idx_model_invocations_session_id (session_id),
+ INDEX idx_model_invocations_request_id (request_id),
+ INDEX idx_model_invocations_user_id (user_id),
+ INDEX idx_model_invocations_instance_id (instance_id),
+ INDEX idx_model_invocations_gateway_id (gateway_id),
+ INDEX idx_model_invocations_model_id (model_id),
+ INDEX idx_model_invocations_status (status),
+ INDEX idx_model_invocations_created_at (created_at)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+CREATE TABLE IF NOT EXISTS audit_events (
+ id INT AUTO_INCREMENT PRIMARY KEY,
+ trace_id VARCHAR(100) NOT NULL,
+ session_id VARCHAR(100) NULL,
+ request_id VARCHAR(100) NULL,
+ user_id INT NULL,
+ instance_id INT NULL,
+ instance_mode VARCHAR(16) NULL,
+ runtime_type VARCHAR(32) NULL,
+ gateway_id VARCHAR(128) NULL,
+ runtime_pod_id BIGINT NULL,
+ invocation_id INT NULL,
+ event_type VARCHAR(100) NOT NULL,
+ traffic_class VARCHAR(50) NOT NULL,
+ severity VARCHAR(20) NOT NULL,
+ message VARCHAR(500) NOT NULL,
+ details LONGTEXT NULL,
+ created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+ INDEX idx_audit_events_trace_id (trace_id),
+ INDEX idx_audit_events_request_id (request_id),
+ INDEX idx_audit_events_user_id (user_id),
+ INDEX idx_audit_events_gateway_id (gateway_id),
+ INDEX idx_audit_events_invocation_id (invocation_id),
+ INDEX idx_audit_events_event_type (event_type),
+ INDEX idx_audit_events_created_at (created_at)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+CREATE TABLE IF NOT EXISTS cost_records (
+ id INT AUTO_INCREMENT PRIMARY KEY,
+ trace_id VARCHAR(100) NOT NULL,
+ session_id VARCHAR(100) NULL,
+ request_id VARCHAR(100) NULL,
+ user_id INT NULL,
+ instance_id INT NULL,
+ instance_mode VARCHAR(16) NULL,
+ runtime_type VARCHAR(32) NULL,
+ gateway_id VARCHAR(128) NULL,
+ runtime_pod_id BIGINT NULL,
+ invocation_id INT NULL,
+ model_id INT NULL,
+ provider_type VARCHAR(100) NOT NULL,
+ model_name VARCHAR(255) NOT NULL,
+ currency VARCHAR(16) NOT NULL DEFAULT 'USD',
+ prompt_tokens INT NOT NULL DEFAULT 0,
+ completion_tokens INT NOT NULL DEFAULT 0,
+ total_tokens INT NOT NULL DEFAULT 0,
+ input_unit_price DECIMAL(18,8) NOT NULL DEFAULT 0,
+ output_unit_price DECIMAL(18,8) NOT NULL DEFAULT 0,
+ estimated_cost DECIMAL(18,8) NOT NULL DEFAULT 0,
+ actual_cost DECIMAL(18,8) NULL,
+ internal_cost DECIMAL(18,8) NOT NULL DEFAULT 0,
+ recorded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+ INDEX idx_cost_records_trace_id (trace_id),
+ INDEX idx_cost_records_user_id (user_id),
+ INDEX idx_cost_records_gateway_id (gateway_id),
+ INDEX idx_cost_records_model_id (model_id),
+ INDEX idx_cost_records_provider_type (provider_type),
+ INDEX idx_cost_records_recorded_at (recorded_at)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+CREATE TABLE IF NOT EXISTS risk_hits (
+ id INT AUTO_INCREMENT PRIMARY KEY,
+ trace_id VARCHAR(100) NOT NULL,
+ session_id VARCHAR(100) NULL,
+ request_id VARCHAR(100) NULL,
+ user_id INT NULL,
+ instance_id INT NULL,
+ instance_mode VARCHAR(16) NULL,
+ runtime_type VARCHAR(32) NULL,
+ gateway_id VARCHAR(128) NULL,
+ runtime_pod_id BIGINT NULL,
+ invocation_id INT NULL,
+ rule_id VARCHAR(100) NOT NULL,
+ rule_name VARCHAR(255) NOT NULL,
+ severity VARCHAR(20) NOT NULL,
+ action VARCHAR(50) NOT NULL,
+ match_summary VARCHAR(500) NOT NULL,
+ created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+ INDEX idx_risk_hits_trace_id (trace_id),
+ INDEX idx_risk_hits_request_id (request_id),
+ INDEX idx_risk_hits_user_id (user_id),
+ INDEX idx_risk_hits_gateway_id (gateway_id),
+ INDEX idx_risk_hits_invocation_id (invocation_id),
+ INDEX idx_risk_hits_severity (severity),
+ INDEX idx_risk_hits_created_at (created_at)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+SET @column_exists = (
+ SELECT COUNT(*) FROM information_schema.COLUMNS
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'model_invocations' AND COLUMN_NAME = 'instance_mode'
+);
+SET @ddl = IF(@column_exists = 0, 'ALTER TABLE model_invocations ADD COLUMN instance_mode VARCHAR(16) NULL AFTER instance_id', 'SELECT 1');
+PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+SET @column_exists = (
+ SELECT COUNT(*) FROM information_schema.COLUMNS
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'model_invocations' AND COLUMN_NAME = 'runtime_type'
+);
+SET @ddl = IF(@column_exists = 0, 'ALTER TABLE model_invocations ADD COLUMN runtime_type VARCHAR(32) NULL AFTER instance_mode', 'SELECT 1');
+PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+SET @column_exists = (
+ SELECT COUNT(*) FROM information_schema.COLUMNS
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'model_invocations' AND COLUMN_NAME = 'gateway_id'
+);
+SET @ddl = IF(@column_exists = 0, 'ALTER TABLE model_invocations ADD COLUMN gateway_id VARCHAR(128) NULL AFTER runtime_type', 'SELECT 1');
+PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+SET @column_exists = (
+ SELECT COUNT(*) FROM information_schema.COLUMNS
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'model_invocations' AND COLUMN_NAME = 'runtime_pod_id'
+);
+SET @ddl = IF(@column_exists = 0, 'ALTER TABLE model_invocations ADD COLUMN runtime_pod_id BIGINT NULL AFTER gateway_id', 'SELECT 1');
+PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+SET @column_exists = (
+ SELECT COUNT(*) FROM information_schema.COLUMNS
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'audit_events' AND COLUMN_NAME = 'instance_mode'
+);
+SET @ddl = IF(@column_exists = 0, 'ALTER TABLE audit_events ADD COLUMN instance_mode VARCHAR(16) NULL AFTER instance_id', 'SELECT 1');
+PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+SET @column_exists = (
+ SELECT COUNT(*) FROM information_schema.COLUMNS
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'audit_events' AND COLUMN_NAME = 'runtime_type'
+);
+SET @ddl = IF(@column_exists = 0, 'ALTER TABLE audit_events ADD COLUMN runtime_type VARCHAR(32) NULL AFTER instance_mode', 'SELECT 1');
+PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+SET @column_exists = (
+ SELECT COUNT(*) FROM information_schema.COLUMNS
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'audit_events' AND COLUMN_NAME = 'gateway_id'
+);
+SET @ddl = IF(@column_exists = 0, 'ALTER TABLE audit_events ADD COLUMN gateway_id VARCHAR(128) NULL AFTER runtime_type', 'SELECT 1');
+PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+SET @column_exists = (
+ SELECT COUNT(*) FROM information_schema.COLUMNS
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'audit_events' AND COLUMN_NAME = 'runtime_pod_id'
+);
+SET @ddl = IF(@column_exists = 0, 'ALTER TABLE audit_events ADD COLUMN runtime_pod_id BIGINT NULL AFTER gateway_id', 'SELECT 1');
+PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+SET @column_exists = (
+ SELECT COUNT(*) FROM information_schema.COLUMNS
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'cost_records' AND COLUMN_NAME = 'instance_mode'
+);
+SET @ddl = IF(@column_exists = 0, 'ALTER TABLE cost_records ADD COLUMN instance_mode VARCHAR(16) NULL AFTER instance_id', 'SELECT 1');
+PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+SET @column_exists = (
+ SELECT COUNT(*) FROM information_schema.COLUMNS
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'cost_records' AND COLUMN_NAME = 'runtime_type'
+);
+SET @ddl = IF(@column_exists = 0, 'ALTER TABLE cost_records ADD COLUMN runtime_type VARCHAR(32) NULL AFTER instance_mode', 'SELECT 1');
+PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+SET @column_exists = (
+ SELECT COUNT(*) FROM information_schema.COLUMNS
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'cost_records' AND COLUMN_NAME = 'gateway_id'
+);
+SET @ddl = IF(@column_exists = 0, 'ALTER TABLE cost_records ADD COLUMN gateway_id VARCHAR(128) NULL AFTER runtime_type', 'SELECT 1');
+PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+SET @column_exists = (
+ SELECT COUNT(*) FROM information_schema.COLUMNS
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'cost_records' AND COLUMN_NAME = 'runtime_pod_id'
+);
+SET @ddl = IF(@column_exists = 0, 'ALTER TABLE cost_records ADD COLUMN runtime_pod_id BIGINT NULL AFTER gateway_id', 'SELECT 1');
+PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+SET @column_exists = (
+ SELECT COUNT(*) FROM information_schema.COLUMNS
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'risk_hits' AND COLUMN_NAME = 'instance_mode'
+);
+SET @ddl = IF(@column_exists = 0, 'ALTER TABLE risk_hits ADD COLUMN instance_mode VARCHAR(16) NULL AFTER instance_id', 'SELECT 1');
+PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+SET @column_exists = (
+ SELECT COUNT(*) FROM information_schema.COLUMNS
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'risk_hits' AND COLUMN_NAME = 'runtime_type'
+);
+SET @ddl = IF(@column_exists = 0, 'ALTER TABLE risk_hits ADD COLUMN runtime_type VARCHAR(32) NULL AFTER instance_mode', 'SELECT 1');
+PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+SET @column_exists = (
+ SELECT COUNT(*) FROM information_schema.COLUMNS
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'risk_hits' AND COLUMN_NAME = 'gateway_id'
+);
+SET @ddl = IF(@column_exists = 0, 'ALTER TABLE risk_hits ADD COLUMN gateway_id VARCHAR(128) NULL AFTER runtime_type', 'SELECT 1');
+PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+SET @column_exists = (
+ SELECT COUNT(*) FROM information_schema.COLUMNS
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'risk_hits' AND COLUMN_NAME = 'runtime_pod_id'
+);
+SET @ddl = IF(@column_exists = 0, 'ALTER TABLE risk_hits ADD COLUMN runtime_pod_id BIGINT NULL AFTER gateway_id', 'SELECT 1');
+PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+SET @index_exists = (
+ SELECT COUNT(*) FROM information_schema.STATISTICS
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'model_invocations' AND INDEX_NAME = 'idx_model_invocations_gateway_id'
+);
+SET @ddl = IF(@index_exists = 0, 'ALTER TABLE model_invocations ADD INDEX idx_model_invocations_gateway_id (gateway_id)', 'SELECT 1');
+PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+SET @index_exists = (
+ SELECT COUNT(*) FROM information_schema.STATISTICS
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'audit_events' AND INDEX_NAME = 'idx_audit_events_gateway_id'
+);
+SET @ddl = IF(@index_exists = 0, 'ALTER TABLE audit_events ADD INDEX idx_audit_events_gateway_id (gateway_id)', 'SELECT 1');
+PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+SET @index_exists = (
+ SELECT COUNT(*) FROM information_schema.STATISTICS
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'cost_records' AND INDEX_NAME = 'idx_cost_records_gateway_id'
+);
+SET @ddl = IF(@index_exists = 0, 'ALTER TABLE cost_records ADD INDEX idx_cost_records_gateway_id (gateway_id)', 'SELECT 1');
+PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+SET @index_exists = (
+ SELECT COUNT(*) FROM information_schema.STATISTICS
+ WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'risk_hits' AND INDEX_NAME = 'idx_risk_hits_gateway_id'
+);
+SET @ddl = IF(@index_exists = 0, 'ALTER TABLE risk_hits ADD INDEX idx_risk_hits_gateway_id (gateway_id)', 'SELECT 1');
+PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt;
diff --git a/backend/internal/db/migrations/029_add_short_external_access_links.sql b/backend/internal/db/migrations/029_add_short_external_access_links.sql
new file mode 100644
index 0000000..291932e
--- /dev/null
+++ b/backend/internal/db/migrations/029_add_short_external_access_links.sql
@@ -0,0 +1,47 @@
+SET @column_exists = (
+ SELECT COUNT(*)
+ FROM information_schema.COLUMNS
+ WHERE TABLE_SCHEMA = DATABASE()
+ AND TABLE_NAME = 'instance_external_access'
+ AND COLUMN_NAME = 'short_code_hash'
+);
+SET @ddl = IF(
+ @column_exists = 0,
+ 'ALTER TABLE instance_external_access ADD COLUMN short_code_hash VARCHAR(128) NULL AFTER public_token_hash',
+ 'SELECT 1'
+);
+PREPARE stmt FROM @ddl;
+EXECUTE stmt;
+DEALLOCATE PREPARE stmt;
+
+SET @public_slug_exists = (
+ SELECT COUNT(*)
+ FROM information_schema.COLUMNS
+ WHERE TABLE_SCHEMA = DATABASE()
+ AND TABLE_NAME = 'instance_external_access'
+ AND COLUMN_NAME = 'public_slug'
+);
+SET @ddl = IF(
+ @public_slug_exists > 0,
+ 'ALTER TABLE instance_external_access MODIFY COLUMN public_slug VARCHAR(64) NULL',
+ 'SELECT 1'
+);
+PREPARE stmt FROM @ddl;
+EXECUTE stmt;
+DEALLOCATE PREPARE stmt;
+
+SET @index_exists = (
+ SELECT COUNT(*)
+ FROM information_schema.STATISTICS
+ WHERE TABLE_SCHEMA = DATABASE()
+ AND TABLE_NAME = 'instance_external_access'
+ AND INDEX_NAME = 'uk_instance_external_access_short_code'
+);
+SET @ddl = IF(
+ @index_exists = 0,
+ 'ALTER TABLE instance_external_access ADD UNIQUE KEY uk_instance_external_access_short_code (short_code_hash)',
+ 'SELECT 1'
+);
+PREPARE stmt FROM @ddl;
+EXECUTE stmt;
+DEALLOCATE PREPARE stmt;
diff --git a/backend/internal/db/migrations/030_add_external_access_password_value.sql b/backend/internal/db/migrations/030_add_external_access_password_value.sql
new file mode 100644
index 0000000..ee56933
--- /dev/null
+++ b/backend/internal/db/migrations/030_add_external_access_password_value.sql
@@ -0,0 +1,15 @@
+SET @column_exists = (
+ SELECT COUNT(*)
+ FROM information_schema.COLUMNS
+ WHERE TABLE_SCHEMA = DATABASE()
+ AND TABLE_NAME = 'instance_external_access'
+ AND COLUMN_NAME = 'password_value'
+);
+SET @ddl = IF(
+ @column_exists = 0,
+ 'ALTER TABLE instance_external_access ADD COLUMN password_value VARCHAR(128) NULL AFTER api_key_hash',
+ 'SELECT 1'
+);
+PREPARE stmt FROM @ddl;
+EXECUTE stmt;
+DEALLOCATE PREPARE stmt;
diff --git a/backend/internal/db/migrations/031_add_hermes_lite_runtime_image.sql b/backend/internal/db/migrations/031_add_hermes_lite_runtime_image.sql
new file mode 100644
index 0000000..fa7855f
--- /dev/null
+++ b/backend/internal/db/migrations/031_add_hermes_lite_runtime_image.sql
@@ -0,0 +1,31 @@
+UPDATE system_image_settings
+SET
+ image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest',
+ display_name = 'Hermes Lite',
+ is_enabled = TRUE
+WHERE instance_type = 'hermes'
+ AND runtime_type = 'shell'
+ AND image IN (
+ 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-shell:latest',
+ 'registry.example.com/your-custom-shell-image:latest'
+ );
+
+INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled)
+SELECT
+ 'hermes',
+ 'shell',
+ 'Hermes Lite',
+ 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest',
+ TRUE
+WHERE NOT EXISTS (
+ SELECT 1
+ FROM system_image_settings
+ WHERE instance_type = 'hermes'
+ AND runtime_type = 'shell'
+)
+AND NOT EXISTS (
+ SELECT 1
+ FROM system_image_settings
+ WHERE instance_type = 'hermes'
+ AND is_enabled = FALSE
+);
diff --git a/backend/internal/db/migrations/032_add_team_member_instance_mode.sql b/backend/internal/db/migrations/032_add_team_member_instance_mode.sql
new file mode 100644
index 0000000..57d9dec
--- /dev/null
+++ b/backend/internal/db/migrations/032_add_team_member_instance_mode.sql
@@ -0,0 +1,27 @@
+SET @team_member_instance_mode_column_exists = (
+ SELECT COUNT(*)
+ FROM information_schema.COLUMNS
+ WHERE TABLE_SCHEMA = DATABASE()
+ AND TABLE_NAME = 'team_members'
+ AND COLUMN_NAME = 'instance_mode'
+);
+SET @team_member_instance_mode_column_sql = IF(
+ @team_member_instance_mode_column_exists = 0,
+ 'ALTER TABLE team_members ADD COLUMN instance_mode ENUM(''lite'', ''pro'') NOT NULL DEFAULT ''lite'' AFTER runtime_type',
+ 'SELECT 1'
+);
+PREPARE team_member_instance_mode_column_stmt FROM @team_member_instance_mode_column_sql;
+EXECUTE team_member_instance_mode_column_stmt;
+DEALLOCATE PREPARE team_member_instance_mode_column_stmt;
+
+UPDATE team_members tm
+LEFT JOIN instances i ON i.id = tm.instance_id
+SET tm.instance_mode = CASE
+ WHEN i.instance_mode IN ('lite', 'pro') THEN i.instance_mode
+ WHEN i.runtime_type = 'gateway' THEN 'lite'
+ WHEN i.id IS NOT NULL THEN 'pro'
+ ELSE tm.instance_mode
+END
+WHERE tm.instance_mode IS NULL
+ OR tm.instance_mode = ''
+ OR (i.id IS NOT NULL AND tm.instance_mode <> COALESCE(i.instance_mode, CASE WHEN i.runtime_type = 'gateway' THEN 'lite' ELSE 'pro' END));
diff --git a/backend/internal/db/migrations/033_system_image_gateway_runtime_type.sql b/backend/internal/db/migrations/033_system_image_gateway_runtime_type.sql
new file mode 100644
index 0000000..674c9f5
--- /dev/null
+++ b/backend/internal/db/migrations/033_system_image_gateway_runtime_type.sql
@@ -0,0 +1,26 @@
+ALTER TABLE system_image_settings
+ MODIFY COLUMN runtime_type ENUM('desktop', 'shell', 'gateway') NOT NULL DEFAULT 'desktop';
+
+UPDATE system_image_settings
+SET runtime_type = 'gateway',
+ display_name = 'OpenClaw Lite'
+WHERE instance_type = 'openclaw'
+ AND runtime_type = 'shell';
+
+UPDATE system_image_settings
+SET runtime_type = 'gateway',
+ display_name = 'Hermes Lite'
+WHERE instance_type = 'hermes'
+ AND runtime_type = 'shell';
+
+UPDATE system_image_settings
+SET display_name = 'OpenClaw Pro'
+WHERE instance_type = 'openclaw'
+ AND runtime_type = 'desktop'
+ AND display_name IN ('OpenClaw Desktop', 'OpenClaw Runtime');
+
+UPDATE system_image_settings
+SET display_name = 'Hermes Pro'
+WHERE instance_type = 'hermes'
+ AND runtime_type = 'desktop'
+ AND display_name IN ('Hermes Runtime', 'Hermes Desktop');
diff --git a/backend/internal/db/migrations/034_update_lite_default_images.sql b/backend/internal/db/migrations/034_update_lite_default_images.sql
new file mode 100644
index 0000000..e819b76
--- /dev/null
+++ b/backend/internal/db/migrations/034_update_lite_default_images.sql
@@ -0,0 +1,54 @@
+UPDATE system_image_settings
+SET
+ image = 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest',
+ display_name = 'OpenClaw Lite',
+ is_enabled = TRUE
+WHERE instance_type = 'openclaw'
+ AND runtime_type IN ('shell', 'gateway')
+ AND image IN (
+ 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-shell:latest',
+ '172.16.1.12:5010/openclaw-shell:local5',
+ '172.16.1.12:5010/openclaw:v2dev'
+ );
+
+UPDATE system_image_settings
+SET
+ image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest',
+ display_name = 'Hermes Lite',
+ is_enabled = TRUE
+WHERE instance_type = 'hermes'
+ AND runtime_type IN ('shell', 'gateway')
+ AND image IN (
+ 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-shell:latest',
+ '172.16.1.12:5010/hermes:codex-shared-agent-tui-bundle-20260609153019',
+ '172.16.1.12:5010/hermes:team-lite-tui-dist-20260612174205',
+ 'registry.example.com/your-custom-shell-image:latest'
+ );
+
+INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled)
+SELECT
+ 'openclaw',
+ 'gateway',
+ 'OpenClaw Lite',
+ 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest',
+ TRUE
+WHERE NOT EXISTS (
+ SELECT 1
+ FROM system_image_settings
+ WHERE instance_type = 'openclaw'
+ AND runtime_type IN ('shell', 'gateway')
+);
+
+INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled)
+SELECT
+ 'hermes',
+ 'gateway',
+ 'Hermes Lite',
+ 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest',
+ TRUE
+WHERE NOT EXISTS (
+ SELECT 1
+ FROM system_image_settings
+ WHERE instance_type = 'hermes'
+ AND runtime_type IN ('shell', 'gateway')
+);
diff --git a/backend/internal/db/migrations_test.go b/backend/internal/db/migrations_test.go
index 9eec03e..16fe86f 100644
--- a/backend/internal/db/migrations_test.go
+++ b/backend/internal/db/migrations_test.go
@@ -2,6 +2,7 @@ package db
import (
"reflect"
+ "strings"
"testing"
)
@@ -29,3 +30,69 @@ UPDATE demo SET note = "a;quoted" WHERE id = 1;
t.Fatalf("unexpected statements:\nwant: %#v\ngot: %#v", want, got)
}
}
+
+func TestMigration023IsEmbedded(t *testing.T) {
+ files, err := embeddedMigrations.ReadDir("migrations")
+ if err != nil {
+ t.Fatalf("read embedded migrations: %v", err)
+ }
+
+ found := false
+ for _, file := range files {
+ if file.Name() == "023_add_runtime_pool_v2.sql" {
+ found = true
+ break
+ }
+ }
+ if !found {
+ t.Fatalf("migration 023_add_runtime_pool_v2.sql is not embedded")
+ }
+}
+
+func TestMigration034UpdatesLiteDefaultImages(t *testing.T) {
+ raw, err := embeddedMigrations.ReadFile("migrations/034_update_lite_default_images.sql")
+ if err != nil {
+ t.Fatalf("read migration 034: %v", err)
+ }
+ sql := string(raw)
+ for _, image := range []string{
+ "ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest",
+ "ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest",
+ } {
+ if !strings.Contains(sql, image) {
+ t.Fatalf("migration 034 must update lite image %s", image)
+ }
+ }
+}
+
+func TestMigration023IsRetrySafe(t *testing.T) {
+ raw, err := embeddedMigrations.ReadFile("migrations/023_add_runtime_pool_v2.sql")
+ if err != nil {
+ t.Fatalf("read migration 023: %v", err)
+ }
+
+ sql := string(raw)
+ if !strings.Contains(sql, "information_schema.COLUMNS") {
+ t.Fatalf("migration 023 must guard instance column additions with information_schema.COLUMNS")
+ }
+ for _, column := range []string{
+ "workspace_path",
+ "workspace_usage_bytes",
+ "runtime_generation",
+ "runtime_error_message",
+ } {
+ if !strings.Contains(sql, "COLUMN_NAME = '"+column+"'") {
+ t.Fatalf("migration 023 must guard %s column addition", column)
+ }
+ }
+ for _, table := range []string{
+ "runtime_pods",
+ "instance_runtime_bindings",
+ "runtime_rollouts",
+ "workspace_file_audits",
+ } {
+ if !strings.Contains(sql, "CREATE TABLE IF NOT EXISTS "+table) {
+ t.Fatalf("migration 023 must create %s idempotently", table)
+ }
+ }
+}
diff --git a/backend/internal/db/runtime_manifest_test.go b/backend/internal/db/runtime_manifest_test.go
new file mode 100644
index 0000000..36e6233
--- /dev/null
+++ b/backend/internal/db/runtime_manifest_test.go
@@ -0,0 +1,56 @@
+package db
+
+import (
+ "os"
+ "path/filepath"
+ "regexp"
+ "strings"
+ "testing"
+)
+
+func TestRuntimeManifestsStartHermesRuntime(t *testing.T) {
+ repoRoot := filepath.Clean(filepath.Join("..", "..", ".."))
+ for _, manifest := range []string{
+ filepath.Join(repoRoot, "deployments", "k8s", "clawmanager.yaml"),
+ filepath.Join(repoRoot, "deployments", "k3s", "clawmanager.yaml"),
+ } {
+ t.Run(manifest, func(t *testing.T) {
+ raw, err := os.ReadFile(manifest)
+ if err != nil {
+ t.Fatalf("read manifest: %v", err)
+ }
+ pattern := regexp.MustCompile(`(?s)name:\s+hermes-runtime.*?spec:\s+replicas:\s+([0-9]+)`)
+ matches := pattern.FindSubmatch(raw)
+ if len(matches) != 2 {
+ t.Fatalf("could not find hermes-runtime replicas in %s", manifest)
+ }
+ if string(matches[1]) != "1" {
+ t.Fatalf("expected hermes-runtime replicas 1 in %s, got %s", manifest, matches[1])
+ }
+ })
+ }
+}
+
+func TestRuntimeManifestsSeedLiteDefaultImages(t *testing.T) {
+ repoRoot := filepath.Clean(filepath.Join("..", "..", ".."))
+ for _, manifest := range []string{
+ filepath.Join(repoRoot, "deployments", "k8s", "clawmanager.yaml"),
+ filepath.Join(repoRoot, "deployments", "k3s", "clawmanager.yaml"),
+ filepath.Join(repoRoot, "backend", "deployments", "k8s", "clawreef-incluster.yaml"),
+ } {
+ t.Run(manifest, func(t *testing.T) {
+ raw, err := os.ReadFile(manifest)
+ if err != nil {
+ t.Fatalf("read manifest: %v", err)
+ }
+ for _, image := range []string{
+ "ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest",
+ "ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest",
+ } {
+ if !strings.Contains(string(raw), image) {
+ t.Fatalf("manifest %s must seed lite image %s", manifest, image)
+ }
+ }
+ })
+ }
+}
diff --git a/backend/internal/handlers/ai_gateway_handler.go b/backend/internal/handlers/ai_gateway_handler.go
index 2dee250..28e77d0 100644
--- a/backend/internal/handlers/ai_gateway_handler.go
+++ b/backend/internal/handlers/ai_gateway_handler.go
@@ -77,6 +77,13 @@ func (h *AIGatewayHandler) ChatCompletions(c *gin.Context) {
}
}
}
+ if !setStringMetadata(c, &req.InstanceMode, "instanceMode") ||
+ !setStringMetadata(c, &req.RuntimeType, "runtimeType") ||
+ !setStringMetadata(c, &req.GatewayID, "gatewayID") ||
+ !setInt64Metadata(c, &req.RuntimePodID, "runtimePodID") {
+ utils.Error(c, http.StatusForbidden, "Gateway token metadata does not match request")
+ return
+ }
if req.Stream {
traceID, err := h.service.StreamChatCompletions(c.Request.Context(), userID.(int), req, c.Writer)
@@ -107,3 +114,47 @@ func (h *AIGatewayHandler) ChatCompletions(c *gin.Context) {
c.Status(response.StatusCode)
_, _ = c.Writer.Write(response.Body)
}
+
+func setStringMetadata(c *gin.Context, field **string, key string) bool {
+ raw, exists := c.Get(key)
+ if !exists {
+ return true
+ }
+ value, ok := raw.(string)
+ if !ok {
+ return true
+ }
+ value = strings.TrimSpace(value)
+ if value == "" {
+ return true
+ }
+ if *field == nil {
+ *field = &value
+ return true
+ }
+ return strings.TrimSpace(**field) == value
+}
+
+func setInt64Metadata(c *gin.Context, field **int64, key string) bool {
+ raw, exists := c.Get(key)
+ if !exists {
+ return true
+ }
+ var value int64
+ switch typed := raw.(type) {
+ case int64:
+ value = typed
+ case int:
+ value = int64(typed)
+ default:
+ return true
+ }
+ if value <= 0 {
+ return true
+ }
+ if *field == nil {
+ *field = &value
+ return true
+ }
+ return **field == value
+}
diff --git a/backend/internal/handlers/instance_handler.go b/backend/internal/handlers/instance_handler.go
index fc4733f..98dc1bf 100644
--- a/backend/internal/handlers/instance_handler.go
+++ b/backend/internal/handlers/instance_handler.go
@@ -3,8 +3,10 @@ package handlers
import (
"errors"
"fmt"
+ "html"
"io"
"net/http"
+ "net/url"
"os"
"strconv"
"strings"
@@ -57,10 +59,11 @@ type InstanceHandler struct {
openClawTransferService services.OpenClawTransferService
openClawConfigService services.OpenClawConfigService
skillService services.SkillService
+ externalAccessService services.InstanceExternalAccessService
}
// NewInstanceHandler creates a new instance handler
-func NewInstanceHandler(instanceService services.InstanceService, instanceAgentService services.InstanceAgentService, runtimeStatusService services.InstanceRuntimeStatusService, instanceCommandService services.InstanceCommandService, instanceConfigRevisionService services.InstanceConfigRevisionService, openClawConfigService services.OpenClawConfigService, skillService services.SkillService) *InstanceHandler {
+func NewInstanceHandler(instanceService services.InstanceService, instanceAgentService services.InstanceAgentService, runtimeStatusService services.InstanceRuntimeStatusService, instanceCommandService services.InstanceCommandService, instanceConfigRevisionService services.InstanceConfigRevisionService, openClawConfigService services.OpenClawConfigService, skillService services.SkillService, externalAccessService services.InstanceExternalAccessService, proxyOptions ...services.InstanceProxyServiceOption) *InstanceHandler {
accessService := services.NewInstanceAccessService()
return &InstanceHandler{
instanceService: instanceService,
@@ -69,11 +72,12 @@ func NewInstanceHandler(instanceService services.InstanceService, instanceAgentS
instanceCommandService: instanceCommandService,
instanceConfigRevisionService: instanceConfigRevisionService,
accessService: accessService,
- proxyService: services.NewInstanceProxyService(accessService),
+ proxyService: services.NewInstanceProxyService(accessService, proxyOptions...),
shellService: services.NewInstanceShellService(),
openClawTransferService: services.NewOpenClawTransferService(),
openClawConfigService: openClawConfigService,
skillService: skillService,
+ externalAccessService: externalAccessService,
}
}
@@ -98,12 +102,20 @@ type PublishConfigRevisionRequest struct {
SnapshotID int `json:"snapshot_id" binding:"required,min=1"`
}
+type ExternalAccessRequest struct {
+ ExpiresMode string `json:"expires_mode,omitempty"`
+ ExpiresPreset string `json:"expires_preset,omitempty"`
+ ExpiresAt *time.Time `json:"expires_at,omitempty"`
+}
+
// CreateInstanceRequest represents a create instance request
type CreateInstanceRequest struct {
Name string `json:"name" binding:"required,min=3,max=50"`
Description *string `json:"description,omitempty"`
- Type string `json:"type" binding:"required,oneof=openclaw ubuntu debian centos custom webtop hermes"`
- RuntimeType string `json:"runtime_type" binding:"omitempty,oneof=desktop shell"`
+ Type string `json:"type" binding:"required,oneof=openclaw hermes"`
+ Mode string `json:"mode" binding:"omitempty,oneof=lite pro"`
+ InstanceMode string `json:"instance_mode" binding:"omitempty,oneof=lite pro"`
+ RuntimeType string `json:"runtime_type" binding:"omitempty,oneof=gateway desktop shell"`
CPUCores float64 `json:"cpu_cores" binding:"required,min=0.1,max=32"`
MemoryGB int `json:"memory_gb" binding:"required,min=1,max=128"`
DiskGB int `json:"disk_gb" binding:"required,min=10,max=1000"`
@@ -210,6 +222,8 @@ func (h *InstanceHandler) CreateInstance(c *gin.Context) {
Name: req.Name,
Description: req.Description,
Type: req.Type,
+ Mode: req.Mode,
+ InstanceMode: req.InstanceMode,
RuntimeType: req.RuntimeType,
CPUCores: req.CPUCores,
MemoryGB: req.MemoryGB,
@@ -550,6 +564,13 @@ func (h *InstanceHandler) GetInstanceStatus(c *gin.Context) {
return
}
+ if userRole != "admin" {
+ utils.Success(c, http.StatusOK, "Instance status retrieved successfully", gin.H{
+ "instance_status": buildUserSafeInstanceStatus(instance, status),
+ })
+ return
+ }
+
runtime, _ := h.runtimeStatusService.GetByInstanceID(id)
agent, _ := h.instanceAgentService.GetPayloadByInstanceID(id)
@@ -560,6 +581,43 @@ func (h *InstanceHandler) GetInstanceStatus(c *gin.Context) {
})
}
+func buildUserSafeInstanceStatus(instance *models.Instance, status *services.InstanceStatus) map[string]interface{} {
+ if status == nil {
+ return map[string]interface{}{}
+ }
+ payload := map[string]interface{}{
+ "instance_id": status.InstanceID,
+ "status": status.Status,
+ "availability": status.Availability,
+ "created_at": status.CreatedAt,
+ }
+ if payload["availability"] == "" {
+ payload["availability"] = availabilityForInstanceStatus(status.Status)
+ }
+ if status.StartedAt != nil {
+ payload["started_at"] = status.StartedAt
+ }
+ if instance == nil {
+ return payload
+ }
+ if agentType, ok := services.NormalizeV2RuntimeType(instance.Type); ok && (strings.EqualFold(strings.TrimSpace(instance.RuntimeType), "gateway") || strings.EqualFold(strings.TrimSpace(instance.InstanceMode), "lite")) {
+ payload["agent_type"] = agentType
+ payload["workspace_usage_bytes"] = instance.WorkspaceUsageBytes
+ }
+ return payload
+}
+
+func availabilityForInstanceStatus(status string) string {
+ switch strings.ToLower(strings.TrimSpace(status)) {
+ case "running":
+ return "available"
+ case "creating":
+ return "starting"
+ default:
+ return "unavailable"
+ }
+}
+
func (h *InstanceHandler) GetRuntimeDetails(c *gin.Context) {
id, _, ok := h.resolveOwnedInstance(c)
if !ok {
@@ -780,7 +838,7 @@ func (h *InstanceHandler) GenerateAccessToken(c *gin.Context) {
}
// Generate proxy entry URL. The actual Service remains internal-only.
- accessURL := h.proxyService.GetProxyURL(instance.ID, "")
+ accessURL := h.proxyService.GetProxyURLForInstance(instance, "")
if accessURL == "" {
utils.Error(c, http.StatusServiceUnavailable, "Unable to generate access URL")
@@ -818,7 +876,7 @@ func (h *InstanceHandler) GenerateAccessToken(c *gin.Context) {
response := map[string]interface{}{
"token": token.Token,
"access_url": accessURL,
- "proxy_url": h.proxyService.GetProxyURL(instance.ID, token.Token),
+ "proxy_url": h.proxyService.GetProxyURLForInstance(instance, token.Token),
"expires_at": token.ExpiresAt,
}
@@ -930,41 +988,62 @@ func (h *InstanceHandler) ProxyInstance(c *gin.Context) {
return
}
- // Get token from query parameter
- token := c.Query("token")
- if token == "" {
- cookieToken, err := c.Cookie(fmt.Sprintf("instance_access_%d", id))
- if err != nil || cookieToken == "" {
- utils.Error(c, http.StatusBadRequest, "Access token required")
- return
- }
- token = cookieToken
- } else {
- // Promote the one-time query token into a cookie so iframe subresources and
- // websocket requests can reuse it without appending the token everywhere.
- c.SetCookie(
- fmt.Sprintf("instance_access_%d", id),
- token,
- int(time.Hour.Seconds()),
- fmt.Sprintf("/api/v1/instances/%d/proxy", id),
- "",
- false,
- true,
- )
+ token, ok := h.proxyAccessToken(c, id)
+ if !ok {
+ return
}
+ h.proxyInstanceWithToken(c, id, token)
+}
+
+func (h *InstanceHandler) proxyAccessToken(c *gin.Context, id int) (string, bool) {
+ cookieName := fmt.Sprintf("instance_access_%d", id)
+ if cookieToken, err := c.Cookie(cookieName); err == nil && strings.TrimSpace(cookieToken) != "" {
+ if accessToken, validateErr := h.accessService.ValidateToken(cookieToken); validateErr == nil && accessToken.InstanceID == id {
+ return cookieToken, true
+ }
+ }
+
+ queryToken := strings.TrimSpace(c.Query("token"))
+ if queryToken == "" {
+ utils.Error(c, http.StatusBadRequest, "Access token required")
+ return "", false
+ }
+ accessToken, err := h.accessService.ValidateToken(queryToken)
+ if err != nil || accessToken.InstanceID != id {
+ utils.Error(c, http.StatusUnauthorized, "Access token expired or invalid")
+ return "", false
+ }
+
+ // Promote only a validated ClawManager access token. Runtime applications may
+ // also use a token query parameter for their own websocket/session protocol.
+ c.SetCookie(
+ cookieName,
+ queryToken,
+ int(time.Hour.Seconds()),
+ fmt.Sprintf("/api/v1/instances/%d/proxy", id),
+ "",
+ false,
+ true,
+ )
+ return queryToken, true
+}
+
+func (h *InstanceHandler) proxyInstanceWithToken(c *gin.Context, id int, token string) {
// Check if it's a WebSocket upgrade request
if strings.EqualFold(c.GetHeader("Upgrade"), "websocket") {
- err = h.proxyService.ProxyWebSocket(c.Request.Context(), id, token, c.Writer, c.Request)
- if err != nil {
- http.Error(c.Writer, err.Error(), http.StatusBadGateway)
+ if err := h.proxyService.ProxyWebSocket(c.Request.Context(), id, token, c.Writer, c.Request); err != nil {
+ if errors.Is(err, services.ErrInstanceGatewayUnavailable) {
+ http.Error(c.Writer, "Instance gateway is not available", http.StatusServiceUnavailable)
+ } else {
+ http.Error(c.Writer, err.Error(), http.StatusBadGateway)
+ }
}
return
}
// Proxy regular HTTP request
- err = h.proxyService.ProxyRequest(c.Request.Context(), id, token, c.Writer, c.Request)
- if err != nil {
+ if err := h.proxyService.ProxyRequest(c.Request.Context(), id, token, c.Writer, c.Request); err != nil {
// Log the error
fmt.Printf("Proxy error for instance %d: %v\n", id, err)
@@ -974,6 +1053,8 @@ func (h *InstanceHandler) ProxyInstance(c *gin.Context) {
http.Error(c.Writer, "Access token expired or invalid", http.StatusUnauthorized)
} else if err.Error() == "token does not match instance" {
http.Error(c.Writer, "Token does not match instance", http.StatusForbidden)
+ } else if errors.Is(err, services.ErrInstanceGatewayUnavailable) {
+ http.Error(c.Writer, "Instance gateway is not available", http.StatusServiceUnavailable)
} else {
http.Error(c.Writer, fmt.Sprintf("Failed to proxy request: %v", err), http.StatusBadGateway)
}
@@ -1204,6 +1285,428 @@ func (h *InstanceHandler) requireOwnedInstance(c *gin.Context) (*models.Instance
return instance, true
}
+func (h *InstanceHandler) GetExternalAccess(c *gin.Context) {
+ instance, ok := h.requireOwnedInstance(c)
+ if !ok {
+ return
+ }
+ if h.externalAccessService == nil {
+ utils.Error(c, http.StatusServiceUnavailable, "External access is not configured")
+ return
+ }
+ access, err := h.externalAccessService.Get(c.Request.Context(), instance.ID)
+ if err != nil {
+ utils.HandleError(c, err)
+ return
+ }
+ payload := gin.H{"external_access": access}
+ if shareURL := services.ExternalAccessShareURL(access); shareURL != "" {
+ payload["share_url"] = shareURL
+ }
+ if password := services.ExternalAccessPassword(access); password != "" {
+ payload["password"] = password
+ }
+ utils.Success(c, http.StatusOK, "External access retrieved successfully", payload)
+}
+
+func (h *InstanceHandler) EnableShareLink(c *gin.Context) {
+ instance, ok := h.requireOwnedInstance(c)
+ if !ok {
+ return
+ }
+ if h.externalAccessService == nil {
+ utils.Error(c, http.StatusServiceUnavailable, "External access is not configured")
+ return
+ }
+ var req ExternalAccessRequest
+ if err := c.ShouldBindJSON(&req); err != nil && err != io.EOF {
+ utils.ValidationError(c, err)
+ return
+ }
+ userID, _ := c.Get("userID")
+ result, err := h.externalAccessService.EnableShareLink(c.Request.Context(), instance.ID, userID.(int), externalAccessExpirationRequest(req))
+ if err != nil {
+ utils.HandleError(c, err)
+ return
+ }
+ utils.Success(c, http.StatusOK, "Share link enabled successfully", result)
+}
+
+func (h *InstanceHandler) CreateExternalAccessPassword(c *gin.Context) {
+ instance, ok := h.requireOwnedInstance(c)
+ if !ok {
+ return
+ }
+ if h.externalAccessService == nil {
+ utils.Error(c, http.StatusServiceUnavailable, "External access is not configured")
+ return
+ }
+ var req ExternalAccessRequest
+ if err := c.ShouldBindJSON(&req); err != nil && err != io.EOF {
+ utils.ValidationError(c, err)
+ return
+ }
+ userID, _ := c.Get("userID")
+ result, err := h.externalAccessService.CreatePassword(c.Request.Context(), instance.ID, userID.(int), externalAccessExpirationRequest(req))
+ if err != nil {
+ utils.HandleError(c, err)
+ return
+ }
+ utils.Success(c, http.StatusOK, "Share link password created successfully", result)
+}
+
+func (h *InstanceHandler) DisableExternalAccess(c *gin.Context) {
+ instance, ok := h.requireOwnedInstance(c)
+ if !ok {
+ return
+ }
+ if h.externalAccessService == nil {
+ utils.Error(c, http.StatusServiceUnavailable, "External access is not configured")
+ return
+ }
+ if err := h.externalAccessService.Disable(c.Request.Context(), instance.ID); err != nil {
+ utils.HandleError(c, err)
+ return
+ }
+ utils.Success(c, http.StatusOK, "External access disabled successfully", nil)
+}
+
+func (h *InstanceHandler) OpenShortExternalAccess(c *gin.Context) {
+ if h.externalAccessService == nil {
+ utils.Error(c, http.StatusServiceUnavailable, "External access is not configured")
+ return
+ }
+ code := strings.TrimSpace(c.Param("code"))
+ access, err := h.externalAccessService.ResolveShortLink(c.Request.Context(), code)
+ if err != nil {
+ utils.Error(c, http.StatusUnauthorized, err.Error())
+ return
+ }
+
+ var instance *models.Instance
+ var token string
+ canonicalAccessURL := ""
+ switch access.AuthMode {
+ case services.ExternalAccessModeShareLink:
+ if _, err := h.externalAccessService.ValidateShortLink(c.Request.Context(), code, ""); err != nil {
+ utils.Error(c, http.StatusUnauthorized, err.Error())
+ return
+ }
+ case services.ExternalAccessModePassword:
+ token = h.validShortLinkAccessToken(c, code, access.InstanceID)
+ if token == "" {
+ password := externalPassword(c)
+ isPasswordFormPost := c.Request.Method == http.MethodPost && password == ""
+ if isPasswordFormPost {
+ password = c.PostForm("password")
+ }
+ if strings.TrimSpace(password) == "" {
+ if c.Request.Method == http.MethodGet || c.Request.Method == http.MethodHead {
+ renderShortLinkPasswordForm(c, code, "")
+ return
+ }
+ utils.Error(c, http.StatusUnauthorized, "share link password is required")
+ return
+ }
+ if _, err := h.externalAccessService.ValidateShortLink(c.Request.Context(), code, password); err != nil {
+ if c.Request.Method == http.MethodPost || shortExternalAccessWantsHTML(c) {
+ renderShortLinkPasswordForm(c, code, "Invalid password")
+ return
+ }
+ utils.Error(c, http.StatusUnauthorized, err.Error())
+ return
+ }
+ var ok bool
+ instance, ok = h.requireExternalAccessInstance(c, access)
+ if !ok {
+ return
+ }
+ instanceToken, ok := h.issueShortExternalAccessToken(c, instance, code)
+ if !ok {
+ return
+ }
+ token = instanceToken.Token
+ canonicalAccessURL = instanceToken.AccessURL
+ if c.Request.Method == http.MethodPost {
+ c.Redirect(http.StatusSeeOther, shortExternalAccessEntryPath(code))
+ return
+ }
+ }
+ default:
+ utils.Error(c, http.StatusBadRequest, "Unsupported share link mode")
+ return
+ }
+
+ if instance == nil {
+ var ok bool
+ instance, ok = h.requireExternalAccessInstance(c, access)
+ if !ok {
+ return
+ }
+ }
+ if token == "" {
+ instanceToken, ok := h.issueShortExternalAccessToken(c, instance, code)
+ if !ok {
+ return
+ }
+ token = instanceToken.Token
+ canonicalAccessURL = instanceToken.AccessURL
+ } else {
+ setShortExternalAccessCookies(c, instance.ID, code, token, int(time.Hour.Seconds()))
+ canonicalAccessURL = h.proxyService.GetProxyURLForInstance(instance, "")
+ }
+
+ originalPath := c.Request.URL.Path
+ if redirectTarget := shortExternalAccessEntryRedirectTarget(c.Request.Method, originalPath, code, canonicalAccessURL); redirectTarget != "" {
+ c.Redirect(http.StatusSeeOther, redirectTarget)
+ return
+ }
+
+ originalRawPath := c.Request.URL.RawPath
+ originalAuthorization := c.Request.Header.Get("Authorization")
+ originalPassword := c.Request.Header.Get("X-Password")
+ c.Request.URL.Path = shortExternalAccessProxyPath(originalPath, code, instance.ID)
+ c.Request.URL.RawPath = ""
+ c.Request.Header.Del("Authorization")
+ c.Request.Header.Del("X-Password")
+ defer func() {
+ c.Request.URL.Path = originalPath
+ c.Request.URL.RawPath = originalRawPath
+ if originalAuthorization != "" {
+ c.Request.Header.Set("Authorization", originalAuthorization)
+ }
+ if originalPassword != "" {
+ c.Request.Header.Set("X-Password", originalPassword)
+ }
+ }()
+
+ h.proxyInstanceWithToken(c, instance.ID, token)
+}
+
+func bearerToken(header string) string {
+ value := strings.TrimSpace(header)
+ if value == "" {
+ return ""
+ }
+ if strings.HasPrefix(strings.ToLower(value), "bearer ") {
+ return strings.TrimSpace(value[len("bearer "):])
+ }
+ return value
+}
+
+func externalPassword(c *gin.Context) string {
+ if password := strings.TrimSpace(c.GetHeader("X-Password")); password != "" {
+ return password
+ }
+ return bearerToken(c.GetHeader("Authorization"))
+}
+
+func externalAccessExpirationRequest(req ExternalAccessRequest) services.ExternalAccessExpirationRequest {
+ return services.ExternalAccessExpirationRequest{
+ Mode: strings.TrimSpace(req.ExpiresMode),
+ Preset: strings.TrimSpace(req.ExpiresPreset),
+ ExpiresAt: req.ExpiresAt,
+ }
+}
+
+func (h *InstanceHandler) requireExternalAccessInstance(c *gin.Context, access *models.InstanceExternalAccess) (*models.Instance, bool) {
+ if access == nil {
+ utils.Error(c, http.StatusUnauthorized, "External access is not enabled")
+ return nil, false
+ }
+ instance, err := h.instanceService.GetByID(access.InstanceID)
+ if err != nil {
+ utils.HandleError(c, err)
+ return nil, false
+ }
+ if instance == nil {
+ utils.Error(c, http.StatusNotFound, "Instance not found")
+ return nil, false
+ }
+ if instance.Status != "running" {
+ utils.Error(c, http.StatusServiceUnavailable, "Instance is not running")
+ return nil, false
+ }
+ if strings.EqualFold(strings.TrimSpace(instance.RuntimeType), "shell") {
+ utils.Error(c, http.StatusBadRequest, "External desktop access is not available for shell runtime instances")
+ return nil, false
+ }
+ return instance, true
+}
+
+func (h *InstanceHandler) issueShortExternalAccessToken(c *gin.Context, instance *models.Instance, code string) (*services.AccessToken, bool) {
+ accessURL := h.proxyService.GetProxyURLForInstance(instance, "")
+ if accessURL == "" {
+ utils.Error(c, http.StatusServiceUnavailable, "Unable to generate access URL")
+ return nil, false
+ }
+ instanceToken, err := h.accessService.GenerateToken(
+ instance.UserID,
+ instance.ID,
+ instance.Type,
+ accessURL,
+ h.proxyService.GetTargetPortForInstance(instance),
+ 1*time.Hour,
+ )
+ if err != nil {
+ utils.HandleError(c, err)
+ return nil, false
+ }
+ setShortExternalAccessCookies(c, instance.ID, code, instanceToken.Token, int(time.Hour.Seconds()))
+ return instanceToken, true
+}
+
+func (h *InstanceHandler) validShortLinkAccessToken(c *gin.Context, code string, instanceID int) string {
+ if h == nil || h.accessService == nil {
+ return ""
+ }
+ token, err := c.Cookie(shortExternalAccessCookieName(code))
+ if err != nil || strings.TrimSpace(token) == "" {
+ return ""
+ }
+ accessToken, err := h.accessService.ValidateToken(token)
+ if err != nil || accessToken.InstanceID != instanceID {
+ return ""
+ }
+ return token
+}
+
+func setShortExternalAccessCookies(c *gin.Context, instanceID int, code, token string, maxAge int) {
+ c.SetCookie(
+ fmt.Sprintf("instance_access_%d", instanceID),
+ token,
+ maxAge,
+ fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID),
+ "",
+ false,
+ true,
+ )
+ c.SetCookie(
+ shortExternalAccessCookieName(code),
+ token,
+ maxAge,
+ shortExternalAccessCookiePath(code),
+ "",
+ false,
+ true,
+ )
+}
+
+func shortExternalAccessCookieName(code string) string {
+ code = strings.Trim(strings.TrimSpace(code), "/")
+ var builder strings.Builder
+ for _, r := range code {
+ switch {
+ case r >= 'a' && r <= 'z':
+ builder.WriteRune(r)
+ case r >= 'A' && r <= 'Z':
+ builder.WriteRune(r)
+ case r >= '0' && r <= '9':
+ builder.WriteRune(r)
+ case r == '_' || r == '-':
+ builder.WriteRune(r)
+ default:
+ builder.WriteByte('_')
+ }
+ }
+ if builder.Len() == 0 {
+ return "share_link_access"
+ }
+ return "share_link_access_" + builder.String()
+}
+
+func shortExternalAccessCookiePath(code string) string {
+ code = strings.Trim(strings.TrimSpace(code), "/")
+ if code == "" {
+ return "/s"
+ }
+ return "/s/" + code
+}
+
+func shortExternalAccessEntryPath(code string) string {
+ return shortExternalAccessCookiePath(code) + "/"
+}
+
+func shortExternalAccessEntryRedirectTarget(method, requestPath, code, canonicalPath string) string {
+ if method != http.MethodGet && method != http.MethodHead {
+ return ""
+ }
+ path := strings.TrimSpace(requestPath)
+ entryPath := shortExternalAccessEntryPath(code)
+ if path != entryPath && path != strings.TrimSuffix(entryPath, "/") {
+ return ""
+ }
+ target := strings.TrimSpace(canonicalPath)
+ if target == "" {
+ return ""
+ }
+ parsed, err := url.Parse(target)
+ if err != nil || parsed.Path == "" {
+ return ""
+ }
+ if !strings.HasPrefix(parsed.Path, "/api/v1/instances/") {
+ return ""
+ }
+ return parsed.Path
+}
+
+func shortExternalAccessWantsHTML(c *gin.Context) bool {
+ accept := strings.ToLower(c.GetHeader("Accept"))
+ return strings.Contains(accept, "text/html")
+}
+
+func renderShortLinkPasswordForm(c *gin.Context, code, errorMessage string) {
+ action := html.EscapeString(shortExternalAccessEntryPath(code))
+ errorHTML := ""
+ if strings.TrimSpace(errorMessage) != "" {
+ errorHTML = fmt.Sprintf(`
%s
`, html.EscapeString(errorMessage))
+ }
+ body := fmt.Sprintf(`
+
+
+
+
+ Share link password
+
+
+
+
+ Share link password
+ %s
+
+
+
+`, errorHTML, action)
+ c.Data(http.StatusOK, "text/html; charset=utf-8", []byte(body))
+}
+
+func shortExternalAccessProxyPath(requestPath, code string, instanceID int) string {
+ internalPrefix := fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID)
+ shortPrefix := fmt.Sprintf("/s/%s", strings.Trim(strings.TrimSpace(code), "/"))
+ path := strings.TrimSpace(requestPath)
+ if path == "" || path == shortPrefix || path == shortPrefix+"/" {
+ return internalPrefix + "/"
+ }
+ if strings.HasPrefix(path, shortPrefix+"/") {
+ return internalPrefix + strings.TrimPrefix(path, shortPrefix)
+ }
+ return internalPrefix + "/"
+}
+
func sanitizeDownloadName(name string, fallback ...string) string {
name = strings.TrimSpace(name)
if name == "" {
diff --git a/backend/internal/handlers/instance_handler_test.go b/backend/internal/handlers/instance_handler_test.go
index 684871b..b1a5342 100644
--- a/backend/internal/handlers/instance_handler_test.go
+++ b/backend/internal/handlers/instance_handler_test.go
@@ -1,6 +1,17 @@
package handlers
-import "testing"
+import (
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ "clawreef/internal/models"
+ "clawreef/internal/services"
+
+ "github.com/gin-gonic/gin"
+)
func TestWorkspaceArchiveMaxMiB(t *testing.T) {
t.Setenv(workspaceArchiveMaxMiBEnv, "")
@@ -23,3 +34,198 @@ func TestWorkspaceArchiveMaxMiB(t *testing.T) {
t.Fatalf("expected unparsable archive limit to fall back to %d MiB, got %d", defaultWorkspaceArchiveMaxMiB, got)
}
}
+
+func TestBuildUserSafeInstanceStatusHidesRuntimeSchedulingDetails(t *testing.T) {
+ podName := "runtime-openclaw-abc"
+ podNamespace := "clawmanager-system"
+ podIP := "10.42.0.12"
+ startedAt := time.Now().UTC()
+ workspacePath := "/workspaces/openclaw/user-45/instance-123"
+ instance := &models.Instance{
+ ID: 123,
+ Type: "openclaw",
+ RuntimeType: "gateway",
+ Status: "running",
+ WorkspacePath: &workspacePath,
+ WorkspaceUsageBytes: 123456,
+ }
+ status := &services.InstanceStatus{
+ InstanceID: 123,
+ Status: "running",
+ PodName: &podName,
+ PodNamespace: &podNamespace,
+ PodIP: &podIP,
+ PodStatus: "Running",
+ StartedAt: &startedAt,
+ }
+
+ payload := buildUserSafeInstanceStatus(instance, status)
+
+ for _, forbiddenKey := range []string{"pod_name", "pod_namespace", "pod_ip", "pod_status", "gateway_port", "capacity", "node_name"} {
+ if _, exists := payload[forbiddenKey]; exists {
+ t.Fatalf("user-safe status exposed %q: %#v", forbiddenKey, payload)
+ }
+ }
+ if payload["availability"] != "available" {
+ t.Fatalf("availability = %v, want available", payload["availability"])
+ }
+ if payload["agent_type"] != "openclaw" {
+ t.Fatalf("agent_type = %v, want openclaw", payload["agent_type"])
+ }
+ if payload["workspace_usage_bytes"] != int64(123456) {
+ t.Fatalf("workspace_usage_bytes = %v, want 123456", payload["workspace_usage_bytes"])
+ }
+}
+
+func TestShortExternalAccessProxyPath(t *testing.T) {
+ tests := []struct {
+ name string
+ requestPath string
+ want string
+ }{
+ {name: "entry without slash", requestPath: "/s/abc123", want: "/api/v1/instances/71/proxy/"},
+ {name: "entry with slash", requestPath: "/s/abc123/", want: "/api/v1/instances/71/proxy/"},
+ {name: "asset", requestPath: "/s/abc123/assets/index.js", want: "/api/v1/instances/71/proxy/assets/index.js"},
+ {name: "nested", requestPath: "/s/abc123/apps/openclaw/settings", want: "/api/v1/instances/71/proxy/apps/openclaw/settings"},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := shortExternalAccessProxyPath(tt.requestPath, "abc123", 71); got != tt.want {
+ t.Fatalf("shortExternalAccessProxyPath(%q) = %q, want %q", tt.requestPath, got, tt.want)
+ }
+ })
+ }
+}
+
+func TestShortExternalAccessEntryRedirectTarget(t *testing.T) {
+ tests := []struct {
+ name string
+ method string
+ requestPath string
+ code string
+ canonicalPath string
+ want string
+ }{
+ {
+ name: "html entry redirects to canonical proxy path",
+ method: http.MethodGet,
+ requestPath: "/s/sl_abc123/",
+ code: "sl_abc123",
+ canonicalPath: "/api/v1/instances/71/proxy/chat/",
+ want: "/api/v1/instances/71/proxy/chat/",
+ },
+ {
+ name: "entry without trailing slash redirects",
+ method: http.MethodGet,
+ requestPath: "/s/sl_abc123",
+ code: "sl_abc123",
+ canonicalPath: "/api/v1/instances/71/proxy/chat/",
+ want: "/api/v1/instances/71/proxy/chat/",
+ },
+ {
+ name: "asset path keeps short proxy handling",
+ method: http.MethodGet,
+ requestPath: "/s/sl_abc123/assets/index.js",
+ code: "sl_abc123",
+ canonicalPath: "/api/v1/instances/71/proxy/chat/",
+ want: "",
+ },
+ {
+ name: "post keeps password form handling",
+ method: http.MethodPost,
+ requestPath: "/s/sl_abc123/",
+ code: "sl_abc123",
+ canonicalPath: "/api/v1/instances/71/proxy/chat/",
+ want: "",
+ },
+ {
+ name: "target strips accidental token query",
+ method: http.MethodGet,
+ requestPath: "/s/sl_abc123/",
+ code: "sl_abc123",
+ canonicalPath: "/api/v1/instances/71/proxy/chat/?token=secret",
+ want: "/api/v1/instances/71/proxy/chat/",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := shortExternalAccessEntryRedirectTarget(tt.method, tt.requestPath, tt.code, tt.canonicalPath); got != tt.want {
+ t.Fatalf("shortExternalAccessEntryRedirectTarget() = %q, want %q", got, tt.want)
+ }
+ })
+ }
+}
+
+func TestRenderShortLinkPasswordForm(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ recorder := httptest.NewRecorder()
+ c, _ := gin.CreateTestContext(recorder)
+ c.Request = httptest.NewRequest(http.MethodGet, "/s/sl_abc123/", nil)
+
+ renderShortLinkPasswordForm(c, "sl_abc123", "")
+
+ if recorder.Code != http.StatusOK {
+ t.Fatalf("status = %d, want %d", recorder.Code, http.StatusOK)
+ }
+ body := recorder.Body.String()
+ for _, want := range []string{
+ "Share link password",
+ `name="password"`,
+ `type="password"`,
+ `action="/s/sl_abc123/"`,
+ } {
+ if !strings.Contains(body, want) {
+ t.Fatalf("password form missing %q in body:\n%s", want, body)
+ }
+ }
+ if contentType := recorder.Header().Get("Content-Type"); !strings.Contains(contentType, "text/html") {
+ t.Fatalf("content type = %q, want text/html", contentType)
+ }
+}
+
+func TestProxyAccessTokenPrefersCookieOverRuntimeQueryToken(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ accessService := services.NewInstanceAccessService()
+ defer accessService.Stop()
+ token, err := accessService.GenerateToken(1, 76, "hermes", "/api/v1/instances/76/proxy/chat/", 3000, time.Hour)
+ if err != nil {
+ t.Fatalf("GenerateToken returned error: %v", err)
+ }
+
+ recorder := httptest.NewRecorder()
+ c, _ := gin.CreateTestContext(recorder)
+ c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/instances/76/proxy/api/ws?token=hermes-session-token", nil)
+ c.Request.AddCookie(&http.Cookie{Name: "instance_access_76", Value: token.Token})
+
+ handler := &InstanceHandler{accessService: accessService}
+ got, ok := handler.proxyAccessToken(c, 76)
+ if !ok {
+ t.Fatal("proxyAccessToken rejected valid cookie")
+ }
+ if got != token.Token {
+ t.Fatalf("proxyAccessToken = %q, want cookie token", got)
+ }
+ if recorder.Code != http.StatusOK {
+ t.Fatalf("unexpected response status %d", recorder.Code)
+ }
+}
+
+func TestProxyAccessTokenRejectsRuntimeQueryTokenWithoutCookie(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ accessService := services.NewInstanceAccessService()
+ defer accessService.Stop()
+
+ recorder := httptest.NewRecorder()
+ c, _ := gin.CreateTestContext(recorder)
+ c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/instances/76/proxy/api/ws?token=hermes-session-token", nil)
+
+ handler := &InstanceHandler{accessService: accessService}
+ if got, ok := handler.proxyAccessToken(c, 76); ok || got != "" {
+ t.Fatalf("proxyAccessToken = %q/%v, want rejected", got, ok)
+ }
+ if recorder.Code != http.StatusUnauthorized {
+ t.Fatalf("status = %d, want %d", recorder.Code, http.StatusUnauthorized)
+ }
+}
diff --git a/backend/internal/handlers/runtime_agent_handler.go b/backend/internal/handlers/runtime_agent_handler.go
new file mode 100644
index 0000000..b29c66c
--- /dev/null
+++ b/backend/internal/handlers/runtime_agent_handler.go
@@ -0,0 +1,373 @@
+package handlers
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "strings"
+ "time"
+
+ "clawreef/internal/config"
+ "clawreef/internal/models"
+ "clawreef/internal/repository"
+ "clawreef/internal/services"
+ "clawreef/internal/utils"
+
+ "github.com/gin-gonic/gin"
+)
+
+const runtimeAgentTokenHeader = "X-ClawManager-Agent-Token"
+
+type runtimeEventPublisher interface {
+ Publish(ctx context.Context, eventType string, payload any) error
+}
+
+type RuntimeAgentHandler struct {
+ cfg config.RuntimePoolConfig
+ podRepo repository.RuntimePodRepository
+ bindingRepo repository.InstanceRuntimeBindingRepository
+ events runtimeEventPublisher
+}
+
+type runtimeAgentPodIdentity struct {
+ PodID int64 `json:"pod_id"`
+ Namespace string `json:"namespace"`
+ PodName string `json:"pod_name"`
+}
+
+type runtimeAgentRegisterRequest struct {
+ RuntimeType string `json:"runtime_type" binding:"required"`
+ Namespace string `json:"namespace" binding:"required"`
+ PodName string `json:"pod_name" binding:"required"`
+ PodUID *string `json:"pod_uid,omitempty"`
+ PodIP *string `json:"pod_ip,omitempty"`
+ NodeName *string `json:"node_name,omitempty"`
+ DeploymentName string `json:"deployment_name" binding:"required"`
+ ImageRef string `json:"image_ref" binding:"required"`
+ AgentEndpoint *string `json:"agent_endpoint,omitempty"`
+ State string `json:"state"`
+ Capacity int `json:"capacity"`
+ UsedSlots int `json:"used_slots"`
+ Draining bool `json:"draining"`
+ Metrics json.RawMessage `json:"metrics,omitempty"`
+ ReportedAt *time.Time `json:"reported_at,omitempty"`
+}
+
+type runtimeAgentHeartbeatRequest struct {
+ runtimeAgentPodIdentity
+ State string `json:"state" binding:"required"`
+ UsedSlots int `json:"used_slots"`
+ Draining bool `json:"draining"`
+ ReportedAt *time.Time `json:"reported_at,omitempty"`
+}
+
+type runtimeAgentMetricsRequest struct {
+ runtimeAgentPodIdentity
+ CPUMillisUsed int64 `json:"cpu_millis_used"`
+ MemoryBytesUsed int64 `json:"memory_bytes_used"`
+ DiskBytesUsed int64 `json:"disk_bytes_used"`
+ NetworkRXBytes int64 `json:"network_rx_bytes"`
+ NetworkTXBytes int64 `json:"network_tx_bytes"`
+ Metrics json.RawMessage `json:"metrics,omitempty"`
+ ReportedAt *time.Time `json:"reported_at,omitempty"`
+}
+
+type runtimeAgentGatewaysRequest struct {
+ runtimeAgentPodIdentity
+ Gateways []runtimeAgentGatewayReport `json:"gateways" binding:"required"`
+}
+
+type runtimeAgentGatewayReport struct {
+ InstanceID int `json:"instance_id" binding:"required"`
+ GatewayID string `json:"gateway_id"`
+ GatewayPort int `json:"gateway_port"`
+ GatewayPID *int `json:"gateway_pid,omitempty"`
+ State string `json:"state" binding:"required"`
+ Generation int `json:"generation" binding:"required"`
+ ErrorMessage *string `json:"error_message,omitempty"`
+ HealthAt *time.Time `json:"health_at,omitempty"`
+}
+
+func NewRuntimeAgentHandler(cfg config.RuntimePoolConfig, podRepo repository.RuntimePodRepository, bindingRepo repository.InstanceRuntimeBindingRepository, events runtimeEventPublisher) *RuntimeAgentHandler {
+ return &RuntimeAgentHandler{
+ cfg: cfg,
+ podRepo: podRepo,
+ bindingRepo: bindingRepo,
+ events: events,
+ }
+}
+
+func (h *RuntimeAgentHandler) Register(c *gin.Context) {
+ if !h.requireAgentToken(c) {
+ return
+ }
+ var req runtimeAgentRegisterRequest
+ if err := c.ShouldBindJSON(&req); err != nil {
+ utils.ValidationError(c, err)
+ return
+ }
+ runtimeType, ok := services.NormalizeV2RuntimeType(req.RuntimeType)
+ if !ok {
+ utils.Error(c, http.StatusBadRequest, "unsupported runtime type")
+ return
+ }
+ state := strings.TrimSpace(req.State)
+ if state == "" {
+ state = "ready"
+ }
+ capacity := runtimePodCapacityFromReport(req.Capacity, h.cfg.MaxGatewaysPerPod)
+ lastSeen := time.Now().UTC()
+ if req.ReportedAt != nil && !req.ReportedAt.IsZero() {
+ lastSeen = req.ReportedAt.UTC()
+ }
+ var metricsJSON *string
+ if len(req.Metrics) > 0 {
+ if !json.Valid(req.Metrics) {
+ utils.Error(c, http.StatusBadRequest, "metrics must be valid JSON")
+ return
+ }
+ raw := string(req.Metrics)
+ metricsJSON = &raw
+ }
+ pod := &models.RuntimePod{
+ RuntimeType: runtimeType,
+ Namespace: strings.TrimSpace(req.Namespace),
+ PodName: strings.TrimSpace(req.PodName),
+ PodUID: trimStringPtr(req.PodUID),
+ PodIP: trimStringPtr(req.PodIP),
+ NodeName: trimStringPtr(req.NodeName),
+ DeploymentName: strings.TrimSpace(req.DeploymentName),
+ ImageRef: strings.TrimSpace(req.ImageRef),
+ AgentEndpoint: trimStringPtr(req.AgentEndpoint),
+ State: state,
+ Capacity: capacity,
+ UsedSlots: req.UsedSlots,
+ Draining: req.Draining,
+ MetricsJSON: metricsJSON,
+ LastSeenAt: &lastSeen,
+ CPUMillisUsed: 0,
+ MemoryBytesUsed: 0,
+ DiskBytesUsed: 0,
+ NetworkRXBytes: 0,
+ NetworkTXBytes: 0,
+ }
+ if err := h.podRepo.UpsertFromAgent(c.Request.Context(), pod); err != nil {
+ utils.HandleError(c, err)
+ return
+ }
+ h.publish(c.Request.Context(), "runtime_pod_state", map[string]any{
+ "pod_id": pod.ID,
+ "runtime_type": runtimeType,
+ "namespace": pod.Namespace,
+ "pod_name": pod.PodName,
+ "state": state,
+ "used_slots": req.UsedSlots,
+ "capacity": capacity,
+ "draining": req.Draining,
+ "last_seen_at": lastSeen,
+ })
+ utils.Success(c, http.StatusOK, "Runtime pod registered successfully", gin.H{"pod": pod})
+}
+
+func (h *RuntimeAgentHandler) Heartbeat(c *gin.Context) {
+ if !h.requireAgentToken(c) {
+ return
+ }
+ var req runtimeAgentHeartbeatRequest
+ if err := c.ShouldBindJSON(&req); err != nil {
+ utils.ValidationError(c, err)
+ return
+ }
+ podID, ok := h.resolvePodID(c, req.runtimeAgentPodIdentity)
+ if !ok {
+ return
+ }
+ lastSeen := time.Now().UTC()
+ if req.ReportedAt != nil && !req.ReportedAt.IsZero() {
+ lastSeen = req.ReportedAt.UTC()
+ }
+ capacity := runtimePodCapacityFromReport(0, h.cfg.MaxGatewaysPerPod)
+ if err := h.podRepo.UpdateHeartbeat(c.Request.Context(), podID, strings.TrimSpace(req.State), req.UsedSlots, capacity, req.Draining, lastSeen); err != nil {
+ utils.HandleError(c, err)
+ return
+ }
+ h.publish(c.Request.Context(), "runtime_pod_state", map[string]any{
+ "pod_id": podID,
+ "state": strings.TrimSpace(req.State),
+ "used_slots": req.UsedSlots,
+ "capacity": capacity,
+ "draining": req.Draining,
+ "last_seen_at": lastSeen,
+ })
+ utils.Success(c, http.StatusOK, "Runtime pod heartbeat accepted", nil)
+}
+
+func (h *RuntimeAgentHandler) ReportMetrics(c *gin.Context) {
+ if !h.requireAgentToken(c) {
+ return
+ }
+ var req runtimeAgentMetricsRequest
+ if err := c.ShouldBindJSON(&req); err != nil {
+ utils.ValidationError(c, err)
+ return
+ }
+ podID, ok := h.resolvePodID(c, req.runtimeAgentPodIdentity)
+ if !ok {
+ return
+ }
+ var metricsJSON *string
+ if len(req.Metrics) > 0 {
+ if !json.Valid(req.Metrics) {
+ utils.Error(c, http.StatusBadRequest, "metrics must be valid JSON")
+ return
+ }
+ raw := string(req.Metrics)
+ metricsJSON = &raw
+ }
+ lastSeen := time.Now().UTC()
+ if req.ReportedAt != nil && !req.ReportedAt.IsZero() {
+ lastSeen = req.ReportedAt.UTC()
+ }
+ update := repository.RuntimePodMetricsUpdate{
+ CPUMillisUsed: req.CPUMillisUsed,
+ MemoryBytesUsed: req.MemoryBytesUsed,
+ DiskBytesUsed: req.DiskBytesUsed,
+ NetworkRXBytes: req.NetworkRXBytes,
+ NetworkTXBytes: req.NetworkTXBytes,
+ MetricsJSON: metricsJSON,
+ LastSeenAt: &lastSeen,
+ }
+ if err := h.podRepo.UpdateMetrics(c.Request.Context(), podID, update); err != nil {
+ utils.HandleError(c, err)
+ return
+ }
+ payload := map[string]any{
+ "pod_id": podID,
+ "cpu_millis_used": req.CPUMillisUsed,
+ "memory_bytes_used": req.MemoryBytesUsed,
+ "disk_bytes_used": req.DiskBytesUsed,
+ "network_rx_bytes": req.NetworkRXBytes,
+ "network_tx_bytes": req.NetworkTXBytes,
+ "last_seen_at": lastSeen,
+ }
+ if metricsJSON != nil {
+ payload["metrics"] = json.RawMessage(*metricsJSON)
+ }
+ h.publish(c.Request.Context(), "runtime_pod_metrics", payload)
+ utils.Success(c, http.StatusOK, "Runtime pod metrics accepted", nil)
+}
+
+func (h *RuntimeAgentHandler) ReportGateways(c *gin.Context) {
+ if !h.requireAgentToken(c) {
+ return
+ }
+ var req runtimeAgentGatewaysRequest
+ if err := c.ShouldBindJSON(&req); err != nil {
+ utils.ValidationError(c, err)
+ return
+ }
+ podID, ok := h.resolvePodID(c, req.runtimeAgentPodIdentity)
+ if !ok {
+ return
+ }
+ for _, gateway := range req.Gateways {
+ binding, err := h.bindingRepo.GetByInstanceID(c.Request.Context(), gateway.InstanceID)
+ if err != nil {
+ utils.HandleError(c, err)
+ return
+ }
+ if binding == nil || binding.RuntimePodID != podID || binding.Generation != gateway.Generation {
+ continue
+ }
+ state := strings.TrimSpace(gateway.State)
+ switch state {
+ case "running", "healthy":
+ if err := h.bindingRepo.UpdateRunning(c.Request.Context(), gateway.InstanceID, gateway.Generation, strings.TrimSpace(gateway.GatewayID), gateway.GatewayPort, gateway.GatewayPID); err != nil {
+ utils.HandleError(c, err)
+ return
+ }
+ default:
+ if err := h.bindingRepo.UpdateState(c.Request.Context(), gateway.InstanceID, gateway.Generation, state, gateway.ErrorMessage); err != nil {
+ utils.HandleError(c, err)
+ return
+ }
+ }
+ }
+ h.publish(c.Request.Context(), "runtime_pod_gateways_reported", map[string]any{
+ "pod_id": podID,
+ "gateway_count": len(req.Gateways),
+ })
+ utils.Success(c, http.StatusOK, "Runtime gateway report accepted", nil)
+}
+
+func (h *RuntimeAgentHandler) ReportSkills(c *gin.Context) {
+ if !h.requireAgentToken(c) {
+ return
+ }
+ var payload map[string]any
+ if err := c.ShouldBindJSON(&payload); err != nil {
+ utils.ValidationError(c, err)
+ return
+ }
+ h.publish(c.Request.Context(), "runtime_agent_skills_reported", payload)
+ utils.Success(c, http.StatusOK, "Runtime agent skills report accepted", nil)
+}
+
+func (h *RuntimeAgentHandler) requireAgentToken(c *gin.Context) bool {
+ if strings.TrimSpace(h.cfg.AgentReportToken) == "" || c.GetHeader(runtimeAgentTokenHeader) != h.cfg.AgentReportToken {
+ utils.Error(c, http.StatusUnauthorized, "invalid runtime agent token")
+ return false
+ }
+ return true
+}
+
+func (h *RuntimeAgentHandler) resolvePodID(c *gin.Context, identity runtimeAgentPodIdentity) (int64, bool) {
+ if identity.PodID > 0 {
+ return identity.PodID, true
+ }
+ namespace := strings.TrimSpace(identity.Namespace)
+ podName := strings.TrimSpace(identity.PodName)
+ if namespace == "" || podName == "" {
+ utils.Error(c, http.StatusBadRequest, "pod_id or namespace and pod_name are required")
+ return 0, false
+ }
+ pod, err := h.podRepo.GetByNamespaceName(c.Request.Context(), namespace, podName)
+ if err != nil {
+ utils.HandleError(c, err)
+ return 0, false
+ }
+ if pod == nil {
+ utils.Error(c, http.StatusNotFound, "runtime pod not found")
+ return 0, false
+ }
+ return pod.ID, true
+}
+
+func (h *RuntimeAgentHandler) publish(ctx context.Context, eventType string, payload any) {
+ if h.events == nil {
+ return
+ }
+ _ = h.events.Publish(ctx, eventType, payload)
+}
+
+func runtimePodCapacityFromReport(reported, configured int) int {
+ if configured > 0 {
+ return configured
+ }
+ capacity := reported
+ if capacity <= 0 {
+ capacity = services.RuntimePodCapacity
+ }
+ return capacity
+}
+
+func trimStringPtr(value *string) *string {
+ if value == nil {
+ return nil
+ }
+ trimmed := strings.TrimSpace(*value)
+ if trimmed == "" {
+ return nil
+ }
+ return &trimmed
+}
diff --git a/backend/internal/handlers/runtime_agent_handler_test.go b/backend/internal/handlers/runtime_agent_handler_test.go
new file mode 100644
index 0000000..ee825f9
--- /dev/null
+++ b/backend/internal/handlers/runtime_agent_handler_test.go
@@ -0,0 +1,342 @@
+package handlers
+
+import (
+ "bytes"
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "clawreef/internal/config"
+ "clawreef/internal/models"
+ "clawreef/internal/repository"
+
+ "github.com/gin-gonic/gin"
+)
+
+func TestRuntimeAgentHandlerRejectsInvalidToken(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ podRepo := &runtimeAgentHandlerPodRepo{}
+ handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, podRepo, &runtimeAgentHandlerBindingRepo{}, &runtimeAgentHandlerEvents{})
+
+ router := gin.New()
+ router.POST("/api/v1/runtime-agent/metrics/report", handler.ReportMetrics)
+
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodPost, "/api/v1/runtime-agent/metrics/report", bytes.NewBufferString(`{"pod_id":9}`))
+ req.Header.Set("Content-Type", "application/json")
+ router.ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusUnauthorized {
+ t.Fatalf("status = %d, body = %s, want 401", rec.Code, rec.Body.String())
+ }
+ if podRepo.updateMetricsCalls != 0 {
+ t.Fatalf("UpdateMetrics called %d times, want 0", podRepo.updateMetricsCalls)
+ }
+}
+
+func TestRuntimeAgentHandlerRegisterUsesConfiguredCapacity(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ podRepo := &runtimeAgentHandlerPodRepo{}
+ events := &runtimeAgentHandlerEvents{}
+ handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{
+ AgentReportToken: "secret",
+ MaxGatewaysPerPod: 33,
+ }, podRepo, &runtimeAgentHandlerBindingRepo{}, events)
+
+ router := gin.New()
+ router.POST("/api/v1/runtime-agent/register", handler.Register)
+
+ body := `{
+ "runtime_type": "openclaw",
+ "namespace": "clawmanager-system",
+ "pod_name": "openclaw-runtime-test",
+ "deployment_name": "openclaw-runtime",
+ "image_ref": "registry/openclaw:v1",
+ "agent_endpoint": "http://10.0.0.1:19090",
+ "state": "ready",
+ "capacity": 10,
+ "used_slots": 7
+ }`
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodPost, "/api/v1/runtime-agent/register", bytes.NewBufferString(body))
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("X-ClawManager-Agent-Token", "secret")
+ router.ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String())
+ }
+ pod := podRepo.podsByID[1]
+ if pod == nil {
+ t.Fatal("runtime pod was not persisted")
+ }
+ if pod.Capacity != 33 {
+ t.Fatalf("stored capacity = %d, want configured capacity 33", pod.Capacity)
+ }
+ if events.lastType != "runtime_pod_state" {
+ t.Fatalf("event type = %q, want runtime_pod_state", events.lastType)
+ }
+ payload, ok := events.lastPayload.(map[string]any)
+ if !ok {
+ t.Fatalf("event payload = %#v, want map", events.lastPayload)
+ }
+ if payload["capacity"] != 33 {
+ t.Fatalf("event capacity = %#v, want configured capacity 33", payload["capacity"])
+ }
+}
+
+func TestRuntimeAgentHandlerHeartbeatUsesConfiguredCapacity(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ podRepo := &runtimeAgentHandlerPodRepo{}
+ events := &runtimeAgentHandlerEvents{}
+ handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{
+ AgentReportToken: "secret",
+ MaxGatewaysPerPod: 44,
+ }, podRepo, &runtimeAgentHandlerBindingRepo{}, events)
+
+ router := gin.New()
+ router.POST("/api/v1/runtime-agent/heartbeat", handler.Heartbeat)
+
+ body := `{
+ "pod_id": 9,
+ "state": "ready",
+ "used_slots": 8,
+ "draining": false
+ }`
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodPost, "/api/v1/runtime-agent/heartbeat", bytes.NewBufferString(body))
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("X-ClawManager-Agent-Token", "secret")
+ router.ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String())
+ }
+ if podRepo.updatedPodID != 9 {
+ t.Fatalf("updated pod id = %d, want 9", podRepo.updatedPodID)
+ }
+ if podRepo.lastHeartbeatCapacity != 44 {
+ t.Fatalf("heartbeat capacity = %d, want configured capacity 44", podRepo.lastHeartbeatCapacity)
+ }
+ payload, ok := events.lastPayload.(map[string]any)
+ if !ok {
+ t.Fatalf("event payload = %#v, want map", events.lastPayload)
+ }
+ if payload["capacity"] != 44 {
+ t.Fatalf("event capacity = %#v, want configured capacity 44", payload["capacity"])
+ }
+}
+
+func TestRuntimeAgentHandlerMetricsReportUpdatesPodAndPublishesEvent(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ podRepo := &runtimeAgentHandlerPodRepo{}
+ events := &runtimeAgentHandlerEvents{}
+ handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, podRepo, &runtimeAgentHandlerBindingRepo{}, events)
+
+ router := gin.New()
+ router.POST("/api/v1/runtime-agent/metrics/report", handler.ReportMetrics)
+
+ body := `{
+ "pod_id": 9,
+ "cpu_millis_used": 2400,
+ "memory_bytes_used": 4096,
+ "disk_bytes_used": 8192,
+ "network_rx_bytes": 12,
+ "network_tx_bytes": 34,
+ "metrics": {"load": 0.42}
+ }`
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodPost, "/api/v1/runtime-agent/metrics/report", bytes.NewBufferString(body))
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("X-ClawManager-Agent-Token", "secret")
+ router.ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String())
+ }
+ if podRepo.updatedPodID != 9 {
+ t.Fatalf("updated pod id = %d, want 9", podRepo.updatedPodID)
+ }
+ if podRepo.lastMetrics.CPUMillisUsed != 2400 || podRepo.lastMetrics.MemoryBytesUsed != 4096 || podRepo.lastMetrics.DiskBytesUsed != 8192 {
+ t.Fatalf("metrics = %#v, want cpu/memory/disk values from request", podRepo.lastMetrics)
+ }
+ if podRepo.lastMetrics.MetricsJSON == nil || *podRepo.lastMetrics.MetricsJSON == "" {
+ t.Fatalf("metrics json was not persisted")
+ }
+ if events.lastType != "runtime_pod_metrics" {
+ t.Fatalf("event type = %q, want runtime_pod_metrics", events.lastType)
+ }
+ payload, ok := events.lastPayload.(map[string]any)
+ if !ok {
+ t.Fatalf("event payload = %#v, want map", events.lastPayload)
+ }
+ if payload["pod_id"] != int64(9) {
+ t.Fatalf("event pod_id = %#v, want 9", payload["pod_id"])
+ }
+}
+
+func TestRuntimeAgentHandlerGatewayReportOnlyUpdatesCurrentPodBinding(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ bindingRepo := &runtimeAgentHandlerBindingRepo{
+ bindings: map[int]*models.InstanceRuntimeBinding{
+ 10: {InstanceID: 10, RuntimePodID: 9, Generation: 2},
+ 11: {InstanceID: 11, RuntimePodID: 99, Generation: 2},
+ 12: {InstanceID: 12, RuntimePodID: 9, Generation: 3},
+ },
+ }
+ handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, &runtimeAgentHandlerPodRepo{}, bindingRepo, &runtimeAgentHandlerEvents{})
+
+ router := gin.New()
+ router.POST("/api/v1/runtime-agent/gateways/report", handler.ReportGateways)
+
+ body := `{
+ "pod_id": 9,
+ "gateways": [
+ {"instance_id":10,"gateway_id":"gw-10","gateway_port":20010,"state":"running","generation":2},
+ {"instance_id":11,"gateway_id":"gw-11","gateway_port":20011,"state":"running","generation":2},
+ {"instance_id":12,"gateway_id":"gw-12","gateway_port":20012,"state":"running","generation":2}
+ ]
+ }`
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodPost, "/api/v1/runtime-agent/gateways/report", bytes.NewBufferString(body))
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("X-ClawManager-Agent-Token", "secret")
+ router.ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String())
+ }
+ if bindingRepo.updateRunningCalls != 1 {
+ t.Fatalf("UpdateRunning calls = %d, want 1", bindingRepo.updateRunningCalls)
+ }
+}
+
+type runtimeAgentHandlerPodRepo struct {
+ updatedPodID int64
+ lastMetrics repository.RuntimePodMetricsUpdate
+ lastHeartbeatCapacity int
+ updateMetricsCalls int
+ podsByID map[int64]*models.RuntimePod
+}
+
+func (r *runtimeAgentHandlerPodRepo) UpsertFromAgent(ctx context.Context, pod *models.RuntimePod) error {
+ if r.podsByID == nil {
+ r.podsByID = map[int64]*models.RuntimePod{}
+ }
+ if pod.ID == 0 {
+ pod.ID = int64(len(r.podsByID) + 1)
+ }
+ cp := *pod
+ r.podsByID[pod.ID] = &cp
+ return nil
+}
+
+func (r *runtimeAgentHandlerPodRepo) GetByID(ctx context.Context, id int64) (*models.RuntimePod, error) {
+ if r.podsByID == nil {
+ return nil, nil
+ }
+ return r.podsByID[id], nil
+}
+
+func (r *runtimeAgentHandlerPodRepo) GetByNamespaceName(ctx context.Context, namespace, podName string) (*models.RuntimePod, error) {
+ for _, pod := range r.podsByID {
+ if pod.Namespace == namespace && pod.PodName == podName {
+ return pod, nil
+ }
+ }
+ return nil, nil
+}
+
+func (r *runtimeAgentHandlerPodRepo) List(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) {
+ return nil, nil
+}
+
+func (r *runtimeAgentHandlerPodRepo) ListSchedulable(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) {
+ return nil, nil
+}
+
+func (r *runtimeAgentHandlerPodRepo) TryClaimSlot(ctx context.Context, podID int64) (bool, error) {
+ return false, nil
+}
+
+func (r *runtimeAgentHandlerPodRepo) ReleaseSlot(ctx context.Context, podID int64) error {
+ return nil
+}
+
+func (r *runtimeAgentHandlerPodRepo) MarkState(ctx context.Context, podID int64, state string, draining bool) error {
+ return nil
+}
+
+func (r *runtimeAgentHandlerPodRepo) UpdateHeartbeat(ctx context.Context, podID int64, state string, usedSlots int, capacity int, draining bool, lastSeenAt time.Time) error {
+ r.updatedPodID = podID
+ r.lastHeartbeatCapacity = capacity
+ return nil
+}
+
+func (r *runtimeAgentHandlerPodRepo) MarkUnseenUnhealthy(ctx context.Context, cutoff time.Time) error {
+ return nil
+}
+
+func (r *runtimeAgentHandlerPodRepo) UpdateMetrics(ctx context.Context, podID int64, metrics repository.RuntimePodMetricsUpdate) error {
+ r.updatedPodID = podID
+ r.lastMetrics = metrics
+ r.updateMetricsCalls++
+ return nil
+}
+
+type runtimeAgentHandlerBindingRepo struct {
+ updateRunningCalls int
+ updateStateCalls int
+ bindings map[int]*models.InstanceRuntimeBinding
+}
+
+func (r *runtimeAgentHandlerBindingRepo) Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error {
+ return nil
+}
+
+func (r *runtimeAgentHandlerBindingRepo) GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) {
+ return r.bindings[instanceID], nil
+}
+
+func (r *runtimeAgentHandlerBindingRepo) GetRunningByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) {
+ return nil, nil
+}
+
+func (r *runtimeAgentHandlerBindingRepo) ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error) {
+ return nil, nil
+}
+
+func (r *runtimeAgentHandlerBindingRepo) ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error) {
+ return nil, nil
+}
+
+func (r *runtimeAgentHandlerBindingRepo) UpdateRunning(ctx context.Context, instanceID int, generation int, gatewayID string, port int, pid *int) error {
+ r.updateRunningCalls++
+ return nil
+}
+
+func (r *runtimeAgentHandlerBindingRepo) UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error {
+ r.updateStateCalls++
+ return nil
+}
+
+func (r *runtimeAgentHandlerBindingRepo) DeleteByInstanceID(ctx context.Context, instanceID int) error {
+ return nil
+}
+
+func (r *runtimeAgentHandlerBindingRepo) DeleteByInstanceIDAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64) error {
+ return nil
+}
+
+type runtimeAgentHandlerEvents struct {
+ lastType string
+ lastPayload any
+}
+
+func (e *runtimeAgentHandlerEvents) Publish(ctx context.Context, eventType string, payload any) error {
+ e.lastType = eventType
+ e.lastPayload = payload
+ return nil
+}
diff --git a/backend/internal/handlers/runtime_pool_handler.go b/backend/internal/handlers/runtime_pool_handler.go
new file mode 100644
index 0000000..af703ad
--- /dev/null
+++ b/backend/internal/handlers/runtime_pool_handler.go
@@ -0,0 +1,270 @@
+package handlers
+
+import (
+ "context"
+ "log"
+ "net/http"
+ "strconv"
+ "strings"
+ "time"
+
+ "clawreef/internal/models"
+ "clawreef/internal/repository"
+ "clawreef/internal/services"
+ "clawreef/internal/utils"
+
+ "github.com/gin-gonic/gin"
+)
+
+type RuntimePoolHandler struct {
+ podRepo repository.RuntimePodRepository
+ bindingRepo repository.InstanceRuntimeBindingRepository
+ rolloutRepo repository.RuntimeRolloutRepository
+ scheduler *services.RuntimeScheduler
+ events runtimeEventPublisher
+}
+
+const (
+ runtimePodListFallbackHeartbeatTimeout = 10 * time.Second
+ runtimePodListStaleWindowMultiplier = 3
+)
+
+type startRuntimeRolloutRequest struct {
+ RuntimeType string `json:"runtime_type" binding:"required"`
+ TargetImageRef string `json:"target_image_ref" binding:"required"`
+ BatchSize int `json:"batch_size"`
+ MaxUnavailable int `json:"max_unavailable"`
+}
+
+type runtimePoolPodListItem struct {
+ models.RuntimePod
+ AgentReported bool `json:"agent_reported"`
+}
+
+func NewRuntimePoolHandler(
+ podRepo repository.RuntimePodRepository,
+ bindingRepo repository.InstanceRuntimeBindingRepository,
+ rolloutRepo repository.RuntimeRolloutRepository,
+ scheduler *services.RuntimeScheduler,
+ events runtimeEventPublisher,
+) *RuntimePoolHandler {
+ return &RuntimePoolHandler{
+ podRepo: podRepo,
+ bindingRepo: bindingRepo,
+ rolloutRepo: rolloutRepo,
+ scheduler: scheduler,
+ events: events,
+ }
+}
+
+func (h *RuntimePoolHandler) ListPods(c *gin.Context) {
+ runtimeType := strings.TrimSpace(c.Query("runtime_type"))
+ if runtimeType != "" {
+ normalized, ok := services.NormalizeV2RuntimeType(runtimeType)
+ if !ok {
+ utils.Error(c, http.StatusBadRequest, "unsupported runtime type")
+ return
+ }
+ runtimeType = normalized
+ }
+ pods, err := h.podRepo.List(c.Request.Context(), runtimeType)
+ if err != nil {
+ utils.HandleError(c, err)
+ return
+ }
+ currentPods := filterCurrentRuntimePods(pods, time.Now().UTC(), h.runtimePodListHeartbeatTimeout())
+ items := runtimePoolPodListItems(currentPods, true)
+ if h.scheduler != nil {
+ deploymentPods, err := h.scheduler.RuntimeDeploymentPods(c.Request.Context(), runtimeType)
+ if err != nil {
+ log.Printf("runtime pool list deployment pods failed: %v", err)
+ } else {
+ items = mergeRuntimePoolDeploymentPods(items, deploymentPods)
+ }
+ }
+ utils.Success(c, http.StatusOK, "Runtime pods retrieved successfully", gin.H{"pods": items})
+}
+
+func (h *RuntimePoolHandler) GetPodGateways(c *gin.Context) {
+ podID, ok := parseRuntimePodID(c)
+ if !ok {
+ return
+ }
+ bindings, err := h.bindingRepo.ListByRuntimePodID(c.Request.Context(), podID)
+ if err != nil {
+ utils.HandleError(c, err)
+ return
+ }
+ utils.Success(c, http.StatusOK, "Runtime pod gateways retrieved successfully", gin.H{"gateways": bindings})
+}
+
+func (h *RuntimePoolHandler) DrainPod(c *gin.Context) {
+ podID, ok := parseRuntimePodID(c)
+ if !ok {
+ return
+ }
+ if h.scheduler != nil {
+ if err := h.scheduler.DrainPod(c.Request.Context(), podID); err != nil {
+ utils.HandleError(c, err)
+ return
+ }
+ } else if err := h.podRepo.MarkState(c.Request.Context(), podID, "draining", true); err != nil {
+ utils.HandleError(c, err)
+ return
+ }
+ h.publish(c.Request.Context(), "runtime_pod_state", map[string]any{
+ "pod_id": podID,
+ "state": "draining",
+ "draining": true,
+ })
+ utils.Success(c, http.StatusOK, "Runtime pod drain started", nil)
+}
+
+func (h *RuntimePoolHandler) StartRollout(c *gin.Context) {
+ var req startRuntimeRolloutRequest
+ if err := c.ShouldBindJSON(&req); err != nil {
+ utils.ValidationError(c, err)
+ return
+ }
+ runtimeType, ok := services.NormalizeV2RuntimeType(req.RuntimeType)
+ if !ok {
+ utils.Error(c, http.StatusBadRequest, "unsupported runtime type")
+ return
+ }
+ targetImage := strings.TrimSpace(req.TargetImageRef)
+ if targetImage == "" {
+ utils.Error(c, http.StatusBadRequest, "target image ref is required")
+ return
+ }
+ batchSize := req.BatchSize
+ if batchSize <= 0 {
+ batchSize = 1
+ }
+ maxUnavailable := req.MaxUnavailable
+ if maxUnavailable <= 0 {
+ maxUnavailable = 1
+ }
+ startedBy := currentUserIDPtr(c)
+ rollout := &models.RuntimeRollout{
+ RuntimeType: runtimeType,
+ TargetImageRef: targetImage,
+ Status: "pending",
+ BatchSize: batchSize,
+ MaxUnavailable: maxUnavailable,
+ StartedBy: startedBy,
+ }
+ if err := h.rolloutRepo.Create(c.Request.Context(), rollout); err != nil {
+ utils.HandleError(c, err)
+ return
+ }
+ if h.scheduler != nil {
+ if err := h.scheduler.StartRollout(c.Request.Context(), rollout.ID); err != nil {
+ utils.HandleError(c, err)
+ return
+ }
+ }
+ h.publish(c.Request.Context(), "runtime_rollout", map[string]any{
+ "rollout_id": rollout.ID,
+ "runtime_type": rollout.RuntimeType,
+ "target_image_ref": rollout.TargetImageRef,
+ "status": rollout.Status,
+ "batch_size": rollout.BatchSize,
+ "max_unavailable": rollout.MaxUnavailable,
+ "started_by": rollout.StartedBy,
+ })
+ utils.Success(c, http.StatusCreated, "Runtime rollout created successfully", gin.H{"rollout": rollout})
+}
+
+func (h *RuntimePoolHandler) publish(ctx context.Context, eventType string, payload any) {
+ if h.events == nil {
+ return
+ }
+ _ = h.events.Publish(ctx, eventType, payload)
+}
+
+func (h *RuntimePoolHandler) runtimePodListHeartbeatTimeout() time.Duration {
+ if h != nil && h.scheduler != nil {
+ if timeout := h.scheduler.HeartbeatTimeout(); timeout > 0 {
+ return timeout
+ }
+ }
+ return runtimePodListFallbackHeartbeatTimeout
+}
+
+func filterCurrentRuntimePods(pods []models.RuntimePod, now time.Time, heartbeatTimeout time.Duration) []models.RuntimePod {
+ if heartbeatTimeout <= 0 {
+ return pods
+ }
+ cutoff := now.UTC().Add(-runtimePodListStaleWindowMultiplier * heartbeatTimeout)
+ current := pods[:0]
+ for _, pod := range pods {
+ if pod.LastSeenAt != nil && pod.LastSeenAt.UTC().Before(cutoff) {
+ continue
+ }
+ current = append(current, pod)
+ }
+ return current
+}
+
+func runtimePoolPodListItems(pods []models.RuntimePod, agentReported bool) []runtimePoolPodListItem {
+ items := make([]runtimePoolPodListItem, 0, len(pods))
+ for _, pod := range pods {
+ items = append(items, runtimePoolPodListItem{
+ RuntimePod: pod,
+ AgentReported: agentReported,
+ })
+ }
+ return items
+}
+
+func mergeRuntimePoolDeploymentPods(items []runtimePoolPodListItem, deploymentPods []models.RuntimePod) []runtimePoolPodListItem {
+ seen := map[string]struct{}{}
+ for _, item := range items {
+ seen[runtimePoolPodKey(item.Namespace, item.PodName)] = struct{}{}
+ }
+ for _, pod := range deploymentPods {
+ key := runtimePoolPodKey(pod.Namespace, pod.PodName)
+ if key == "" {
+ continue
+ }
+ if _, ok := seen[key]; ok {
+ continue
+ }
+ seen[key] = struct{}{}
+ items = append(items, runtimePoolPodListItem{
+ RuntimePod: pod,
+ AgentReported: false,
+ })
+ }
+ return items
+}
+
+func runtimePoolPodKey(namespace, podName string) string {
+ namespace = strings.TrimSpace(namespace)
+ podName = strings.TrimSpace(podName)
+ if namespace == "" || podName == "" {
+ return ""
+ }
+ return namespace + "/" + podName
+}
+
+func parseRuntimePodID(c *gin.Context) (int64, bool) {
+ podID, err := strconv.ParseInt(c.Param("id"), 10, 64)
+ if err != nil || podID <= 0 {
+ utils.Error(c, http.StatusBadRequest, "invalid runtime pod ID")
+ return 0, false
+ }
+ return podID, true
+}
+
+func currentUserIDPtr(c *gin.Context) *int {
+ raw, ok := c.Get("userID")
+ if !ok {
+ return nil
+ }
+ userID, ok := raw.(int)
+ if !ok {
+ return nil
+ }
+ return &userID
+}
diff --git a/backend/internal/handlers/runtime_pool_handler_test.go b/backend/internal/handlers/runtime_pool_handler_test.go
new file mode 100644
index 0000000..51405c9
--- /dev/null
+++ b/backend/internal/handlers/runtime_pool_handler_test.go
@@ -0,0 +1,532 @@
+package handlers
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "clawreef/internal/middleware"
+ "clawreef/internal/models"
+ "clawreef/internal/repository"
+ "clawreef/internal/services"
+ "clawreef/internal/services/k8s"
+
+ "github.com/gin-gonic/gin"
+)
+
+func TestRuntimePoolHandlerRejectsNonAdmin(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ handler := NewRuntimePoolHandler(&runtimePoolHandlerPodRepo{}, &runtimePoolHandlerBindingRepo{}, &runtimePoolHandlerRolloutRepo{}, nil, &runtimePoolHandlerEvents{})
+
+ router := runtimePoolHandlerRouter(20, "user", handler)
+
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/runtime-pods", nil)
+ router.ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusForbidden {
+ t.Fatalf("status = %d, body = %s, want 403", rec.Code, rec.Body.String())
+ }
+}
+
+func TestRuntimePoolHandlerListPodsReturnsMetricsForAdmin(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ now := time.Now().UTC()
+ podIP := "10.42.0.31"
+ nodeName := "node-a"
+ handler := NewRuntimePoolHandler(&runtimePoolHandlerPodRepo{pods: []models.RuntimePod{{
+ ID: 9,
+ RuntimeType: "openclaw",
+ PodName: "openclaw-runtime-abcde",
+ PodIP: &podIP,
+ NodeName: &nodeName,
+ State: "ready",
+ UsedSlots: 37,
+ Capacity: 100,
+ Draining: false,
+ CPUMillisUsed: 13600,
+ MemoryBytesUsed: 42949672960,
+ DiskBytesUsed: 214748364800,
+ NetworkRXBytes: 9223372,
+ NetworkTXBytes: 19223372,
+ LastSeenAt: &now,
+ }}}, &runtimePoolHandlerBindingRepo{}, &runtimePoolHandlerRolloutRepo{}, nil, &runtimePoolHandlerEvents{})
+
+ router := runtimePoolHandlerRouter(1, "admin", handler)
+
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/runtime-pods", nil)
+ router.ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String())
+ }
+
+ var resp struct {
+ Data struct {
+ Pods []struct {
+ ID int64 `json:"id"`
+ RuntimeType string `json:"runtime_type"`
+ PodName string `json:"pod_name"`
+ CPUMillisUsed int64 `json:"cpu_millis_used"`
+ MemoryBytesUsed int64 `json:"memory_bytes_used"`
+ DiskBytesUsed int64 `json:"disk_bytes_used"`
+ NetworkRXBytes int64 `json:"network_rx_bytes"`
+ NetworkTXBytes int64 `json:"network_tx_bytes"`
+ LastSeenAt string `json:"last_seen_at"`
+ } `json:"pods"`
+ } `json:"data"`
+ }
+ if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
+ t.Fatalf("decode response: %v", err)
+ }
+ if len(resp.Data.Pods) != 1 {
+ t.Fatalf("pods length = %d, want 1", len(resp.Data.Pods))
+ }
+ pod := resp.Data.Pods[0]
+ if pod.ID != 9 || pod.RuntimeType != "openclaw" || pod.PodName != "openclaw-runtime-abcde" {
+ t.Fatalf("pod identity = %#v, want runtime pod data", pod)
+ }
+ if pod.CPUMillisUsed != 13600 || pod.MemoryBytesUsed != 42949672960 || pod.DiskBytesUsed != 214748364800 || pod.NetworkRXBytes != 9223372 || pod.NetworkTXBytes != 19223372 {
+ t.Fatalf("pod metrics = %#v, want persisted metrics", pod)
+ }
+}
+
+func TestRuntimePoolHandlerListPodsOmitsStaleRuntimePods(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ recentSeen := time.Now().UTC()
+ staleSeen := recentSeen.Add(-5 * time.Minute)
+ handler := NewRuntimePoolHandler(&runtimePoolHandlerPodRepo{pods: []models.RuntimePod{
+ {ID: 1, RuntimeType: "openclaw", PodName: "openclaw-runtime-live", State: "ready", LastSeenAt: &recentSeen},
+ {ID: 2, RuntimeType: "openclaw", PodName: "openclaw-runtime-deleted", State: "unhealthy", LastSeenAt: &staleSeen},
+ {ID: 3, RuntimeType: "hermes", PodName: "hermes-runtime-recent-unhealthy", State: "unhealthy", LastSeenAt: &recentSeen},
+ }}, &runtimePoolHandlerBindingRepo{}, &runtimePoolHandlerRolloutRepo{}, nil, &runtimePoolHandlerEvents{})
+
+ router := runtimePoolHandlerRouter(1, "admin", handler)
+
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/runtime-pods", nil)
+ router.ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String())
+ }
+
+ var resp struct {
+ Data struct {
+ Pods []struct {
+ ID int64 `json:"id"`
+ PodName string `json:"pod_name"`
+ } `json:"pods"`
+ } `json:"data"`
+ }
+ if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
+ t.Fatalf("decode response: %v", err)
+ }
+ if len(resp.Data.Pods) != 2 {
+ t.Fatalf("pods length = %d, want 2: %#v", len(resp.Data.Pods), resp.Data.Pods)
+ }
+ for _, pod := range resp.Data.Pods {
+ if pod.ID == 2 || pod.PodName == "openclaw-runtime-deleted" {
+ t.Fatalf("stale deleted pod was returned: %#v", pod)
+ }
+ }
+}
+
+func TestRuntimePoolHandlerListPodsIncludesUnreportedDeploymentPods(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ deployments := &runtimePoolHandlerDeploymentService{
+ pods: []k8s.RuntimeDeploymentPod{{
+ RuntimeType: "openclaw",
+ Namespace: "clawmanager-system",
+ DeploymentName: "openclaw-runtime",
+ PodName: "openclaw-runtime-current",
+ ImageRef: "registry/openclaw-lite:final2",
+ State: "ready",
+ }},
+ }
+ scheduler := services.NewRuntimeScheduler(
+ nil,
+ &runtimePoolHandlerPodRepo{},
+ nil,
+ &runtimePoolHandlerRolloutRepo{},
+ &runtimePoolHandlerAgentClient{},
+ services.NewRuntimeEventService(nil),
+ nil,
+ deployments,
+ time.Second,
+ services.WithRuntimeSchedulerNamespace("clawmanager-system"),
+ services.WithRuntimeSchedulerMaxGatewaysPerPod(33),
+ )
+ handler := NewRuntimePoolHandler(&runtimePoolHandlerPodRepo{}, &runtimePoolHandlerBindingRepo{}, &runtimePoolHandlerRolloutRepo{}, scheduler, &runtimePoolHandlerEvents{})
+
+ router := runtimePoolHandlerRouter(1, "admin", handler)
+
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/runtime-pods?runtime_type=openclaw", nil)
+ router.ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String())
+ }
+
+ var resp struct {
+ Data struct {
+ Pods []struct {
+ ID int64 `json:"id"`
+ RuntimeType string `json:"runtime_type"`
+ PodName string `json:"pod_name"`
+ ImageRef string `json:"image_ref"`
+ State string `json:"state"`
+ Capacity int `json:"capacity"`
+ AgentReported bool `json:"agent_reported"`
+ DeploymentName string `json:"deployment_name"`
+ } `json:"pods"`
+ } `json:"data"`
+ }
+ if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
+ t.Fatalf("decode response: %v", err)
+ }
+ if len(resp.Data.Pods) != 1 {
+ t.Fatalf("pods length = %d, want 1: %#v", len(resp.Data.Pods), resp.Data.Pods)
+ }
+ pod := resp.Data.Pods[0]
+ if pod.ID >= 0 || pod.AgentReported {
+ t.Fatalf("fallback pod identity = id:%d agent_reported:%t, want synthetic unreported pod", pod.ID, pod.AgentReported)
+ }
+ if pod.RuntimeType != "openclaw" || pod.PodName != "openclaw-runtime-current" || pod.DeploymentName != "openclaw-runtime" {
+ t.Fatalf("fallback pod metadata = %#v, want OpenClaw deployment pod", pod)
+ }
+ if pod.ImageRef != "registry/openclaw-lite:final2" {
+ t.Fatalf("fallback pod image = %q, want registry/openclaw-lite:final2", pod.ImageRef)
+ }
+ if pod.State != "unhealthy" {
+ t.Fatalf("fallback pod state = %q, want unhealthy while agent is not reporting", pod.State)
+ }
+ if pod.Capacity != 33 {
+ t.Fatalf("fallback pod capacity = %d, want configured capacity 33", pod.Capacity)
+ }
+}
+
+func TestRuntimePoolHandlerStartRolloutStoresRequesterAndPublishesEvent(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ rolloutRepo := &runtimePoolHandlerRolloutRepo{}
+ events := &runtimePoolHandlerEvents{}
+ handler := NewRuntimePoolHandler(&runtimePoolHandlerPodRepo{}, &runtimePoolHandlerBindingRepo{}, rolloutRepo, nil, events)
+
+ router := runtimePoolHandlerRouter(7, "admin", handler)
+
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodPost, "/api/v1/admin/runtime-rollouts", bytes.NewBufferString(`{
+ "runtime_type": "hermes",
+ "target_image_ref": "ghcr.io/example/hermes:v2",
+ "batch_size": 2,
+ "max_unavailable": 1
+ }`))
+ req.Header.Set("Content-Type", "application/json")
+ router.ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusCreated {
+ t.Fatalf("status = %d, body = %s, want 201", rec.Code, rec.Body.String())
+ }
+ if rolloutRepo.created == nil {
+ t.Fatalf("rollout was not created")
+ }
+ if rolloutRepo.created.StartedBy == nil || *rolloutRepo.created.StartedBy != 7 {
+ t.Fatalf("started_by = %#v, want 7", rolloutRepo.created.StartedBy)
+ }
+ if events.lastType != "runtime_rollout" {
+ t.Fatalf("event type = %q, want runtime_rollout", events.lastType)
+ }
+}
+
+func TestRuntimePoolHandlerStartRolloutRunsSchedulerImmediately(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ rolloutRepo := &runtimePoolHandlerRolloutRepo{}
+ podRepo := &runtimePoolHandlerPodRepo{pods: []models.RuntimePod{{
+ ID: 21,
+ RuntimeType: "hermes",
+ Namespace: "clawmanager-system",
+ PodName: "hermes-runtime-old",
+ DeploymentName: "hermes-runtime",
+ ImageRef: "registry/hermes:v1",
+ State: "ready",
+ Capacity: 100,
+ UsedSlots: 7,
+ }}}
+ deployments := &runtimePoolHandlerDeploymentService{}
+ scheduler := services.NewRuntimeScheduler(
+ nil,
+ podRepo,
+ nil,
+ rolloutRepo,
+ &runtimePoolHandlerAgentClient{},
+ services.NewRuntimeEventService(nil),
+ nil,
+ deployments,
+ time.Second,
+ )
+ handler := NewRuntimePoolHandler(podRepo, &runtimePoolHandlerBindingRepo{}, rolloutRepo, scheduler, &runtimePoolHandlerEvents{})
+
+ router := runtimePoolHandlerRouter(7, "admin", handler)
+
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodPost, "/api/v1/admin/runtime-rollouts", bytes.NewBufferString(`{
+ "runtime_type": "hermes",
+ "target_image_ref": "registry/hermes:v2",
+ "batch_size": 1,
+ "max_unavailable": 1
+ }`))
+ req.Header.Set("Content-Type", "application/json")
+ router.ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusCreated {
+ t.Fatalf("status = %d, body = %s, want 201", rec.Code, rec.Body.String())
+ }
+ if got := len(deployments.rollouts); got != 1 {
+ t.Fatalf("deployment rollouts = %d, want 1", got)
+ }
+ rollout := deployments.rollouts[0]
+ if rollout.namespace != "clawmanager-system" || rollout.name != "hermes-runtime" || rollout.image != "registry/hermes:v2" {
+ t.Fatalf("deployment rollout = %+v, want hermes-runtime registry/hermes:v2", rollout)
+ }
+ if podRepo.markedPodID != 21 || podRepo.markedState != "draining" || !podRepo.markedDraining {
+ t.Fatalf("pod mark = id:%d state:%q draining:%t, want drain pod 21", podRepo.markedPodID, podRepo.markedState, podRepo.markedDraining)
+ }
+}
+
+func runtimePoolHandlerRouter(userID int, role string, handler *RuntimePoolHandler) *gin.Engine {
+ userRepo := &runtimePoolHandlerUserRepo{users: map[int]*models.User{
+ userID: {ID: userID, Username: "tester", Role: role},
+ }}
+ router := gin.New()
+ admin := router.Group("/api/v1/admin")
+ admin.Use(func(c *gin.Context) {
+ c.Set("userID", userID)
+ c.Next()
+ })
+ admin.Use(middleware.SetUserInfo(userRepo))
+ admin.Use(middleware.NewAdminAuth(userRepo))
+ {
+ admin.GET("/runtime-pods", handler.ListPods)
+ admin.POST("/runtime-rollouts", handler.StartRollout)
+ }
+ return router
+}
+
+type runtimePoolHandlerUserRepo struct {
+ users map[int]*models.User
+}
+
+func (r *runtimePoolHandlerUserRepo) Create(user *models.User) error { return nil }
+func (r *runtimePoolHandlerUserRepo) GetByID(id int) (*models.User, error) {
+ return r.users[id], nil
+}
+func (r *runtimePoolHandlerUserRepo) GetByUsername(username string) (*models.User, error) {
+ return nil, nil
+}
+func (r *runtimePoolHandlerUserRepo) GetByEmail(email string) (*models.User, error) {
+ return nil, nil
+}
+func (r *runtimePoolHandlerUserRepo) Update(user *models.User) error { return nil }
+func (r *runtimePoolHandlerUserRepo) Delete(id int) error { return nil }
+func (r *runtimePoolHandlerUserRepo) List(offset, limit int) ([]models.User, error) {
+ return nil, nil
+}
+func (r *runtimePoolHandlerUserRepo) Count() (int, error) { return 0, nil }
+
+type runtimePoolHandlerPodRepo struct {
+ pods []models.RuntimePod
+ markedPodID int64
+ markedState string
+ markedDraining bool
+}
+
+func (r *runtimePoolHandlerPodRepo) UpsertFromAgent(ctx context.Context, pod *models.RuntimePod) error {
+ return nil
+}
+func (r *runtimePoolHandlerPodRepo) GetByID(ctx context.Context, id int64) (*models.RuntimePod, error) {
+ for _, pod := range r.pods {
+ if pod.ID == id {
+ cp := pod
+ return &cp, nil
+ }
+ }
+ return nil, nil
+}
+func (r *runtimePoolHandlerPodRepo) GetByNamespaceName(ctx context.Context, namespace, podName string) (*models.RuntimePod, error) {
+ return nil, nil
+}
+func (r *runtimePoolHandlerPodRepo) List(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) {
+ if runtimeType == "" {
+ return append([]models.RuntimePod(nil), r.pods...), nil
+ }
+ var pods []models.RuntimePod
+ for _, pod := range r.pods {
+ if pod.RuntimeType == runtimeType {
+ pods = append(pods, pod)
+ }
+ }
+ return pods, nil
+}
+func (r *runtimePoolHandlerPodRepo) ListSchedulable(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) {
+ return nil, nil
+}
+func (r *runtimePoolHandlerPodRepo) TryClaimSlot(ctx context.Context, podID int64) (bool, error) {
+ return false, nil
+}
+func (r *runtimePoolHandlerPodRepo) ReleaseSlot(ctx context.Context, podID int64) error {
+ return nil
+}
+func (r *runtimePoolHandlerPodRepo) MarkState(ctx context.Context, podID int64, state string, draining bool) error {
+ r.markedPodID = podID
+ r.markedState = state
+ r.markedDraining = draining
+ return nil
+}
+func (r *runtimePoolHandlerPodRepo) UpdateHeartbeat(ctx context.Context, podID int64, state string, usedSlots int, capacity int, draining bool, lastSeenAt time.Time) error {
+ return nil
+}
+func (r *runtimePoolHandlerPodRepo) MarkUnseenUnhealthy(ctx context.Context, cutoff time.Time) error {
+ return nil
+}
+func (r *runtimePoolHandlerPodRepo) UpdateMetrics(ctx context.Context, podID int64, metrics repository.RuntimePodMetricsUpdate) error {
+ return nil
+}
+
+type runtimePoolHandlerBindingRepo struct {
+ bindings []models.InstanceRuntimeBinding
+}
+
+func (r *runtimePoolHandlerBindingRepo) Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error {
+ return nil
+}
+func (r *runtimePoolHandlerBindingRepo) GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) {
+ return nil, nil
+}
+func (r *runtimePoolHandlerBindingRepo) GetRunningByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) {
+ return nil, nil
+}
+func (r *runtimePoolHandlerBindingRepo) ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error) {
+ var bindings []models.InstanceRuntimeBinding
+ for _, binding := range r.bindings {
+ if binding.RuntimePodID == runtimePodID {
+ bindings = append(bindings, binding)
+ }
+ }
+ return bindings, nil
+}
+func (r *runtimePoolHandlerBindingRepo) ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error) {
+ return nil, nil
+}
+func (r *runtimePoolHandlerBindingRepo) UpdateRunning(ctx context.Context, instanceID int, generation int, gatewayID string, port int, pid *int) error {
+ return nil
+}
+func (r *runtimePoolHandlerBindingRepo) UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error {
+ return nil
+}
+func (r *runtimePoolHandlerBindingRepo) DeleteByInstanceID(ctx context.Context, instanceID int) error {
+ return nil
+}
+func (r *runtimePoolHandlerBindingRepo) DeleteByInstanceIDAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64) error {
+ return nil
+}
+
+type runtimePoolHandlerRolloutRepo struct {
+ created *models.RuntimeRollout
+ byID map[int64]*models.RuntimeRollout
+}
+
+func (r *runtimePoolHandlerRolloutRepo) Create(ctx context.Context, rollout *models.RuntimeRollout) error {
+ rollout.ID = 12
+ cp := *rollout
+ r.created = &cp
+ if r.byID == nil {
+ r.byID = map[int64]*models.RuntimeRollout{}
+ }
+ r.byID[rollout.ID] = &cp
+ return nil
+}
+func (r *runtimePoolHandlerRolloutRepo) GetByID(ctx context.Context, id int64) (*models.RuntimeRollout, error) {
+ if r.byID == nil {
+ return nil, nil
+ }
+ return r.byID[id], nil
+}
+func (r *runtimePoolHandlerRolloutRepo) ListActive(ctx context.Context, runtimeType string) ([]models.RuntimeRollout, error) {
+ return nil, nil
+}
+func (r *runtimePoolHandlerRolloutRepo) UpdateStatus(ctx context.Context, id int64, status string, startedAt *time.Time, finishedAt *time.Time, message *string) error {
+ return nil
+}
+
+type runtimePoolHandlerEvents struct {
+ lastType string
+ lastPayload any
+}
+
+func (e *runtimePoolHandlerEvents) Publish(ctx context.Context, eventType string, payload any) error {
+ e.lastType = eventType
+ e.lastPayload = payload
+ return nil
+}
+
+type runtimePoolHandlerDeploymentService struct {
+ rollouts []runtimePoolHandlerDeploymentRollout
+ pods []k8s.RuntimeDeploymentPod
+}
+
+type runtimePoolHandlerDeploymentRollout struct {
+ namespace string
+ name string
+ image string
+ maxUnavailable int
+ maxSurge int
+}
+
+func (s *runtimePoolHandlerDeploymentService) Ensure(ctx context.Context, spec k8s.RuntimeDeploymentSpec) error {
+ return nil
+}
+func (s *runtimePoolHandlerDeploymentService) Scale(ctx context.Context, namespace, name string, replicas int32) error {
+ return nil
+}
+func (s *runtimePoolHandlerDeploymentService) RolloutImage(ctx context.Context, namespace, name, image string, maxUnavailable, maxSurge int) error {
+ s.rollouts = append(s.rollouts, runtimePoolHandlerDeploymentRollout{
+ namespace: namespace,
+ name: name,
+ image: image,
+ maxUnavailable: maxUnavailable,
+ maxSurge: maxSurge,
+ })
+ return nil
+}
+func (s *runtimePoolHandlerDeploymentService) ListPods(ctx context.Context, namespace, runtimeType string) ([]k8s.RuntimeDeploymentPod, error) {
+ var pods []k8s.RuntimeDeploymentPod
+ for _, pod := range s.pods {
+ if namespace != "" && pod.Namespace != namespace {
+ continue
+ }
+ if runtimeType != "" && pod.RuntimeType != runtimeType {
+ continue
+ }
+ pods = append(pods, pod)
+ }
+ return pods, nil
+}
+
+type runtimePoolHandlerAgentClient struct{}
+
+func (c *runtimePoolHandlerAgentClient) Health(ctx context.Context, endpoint string) error {
+ return nil
+}
+func (c *runtimePoolHandlerAgentClient) CreateGateway(ctx context.Context, endpoint string, req services.RuntimeAgentCreateGatewayRequest) (*services.RuntimeAgentCreateGatewayResponse, error) {
+ return nil, nil
+}
+func (c *runtimePoolHandlerAgentClient) DeleteGateway(ctx context.Context, endpoint, gatewayID string) error {
+ return nil
+}
+func (c *runtimePoolHandlerAgentClient) Drain(ctx context.Context, endpoint string) error { return nil }
diff --git a/backend/internal/handlers/system_settings_handler.go b/backend/internal/handlers/system_settings_handler.go
index 90b3c35..4008aa6 100644
--- a/backend/internal/handlers/system_settings_handler.go
+++ b/backend/internal/handlers/system_settings_handler.go
@@ -19,7 +19,7 @@ type SystemSettingsHandler struct {
type UpsertSystemImageSettingRequest struct {
ID int `json:"id,omitempty"`
InstanceType string `json:"instance_type" binding:"required"`
- RuntimeType string `json:"runtime_type" binding:"omitempty,oneof=desktop shell"`
+ RuntimeType string `json:"runtime_type" binding:"omitempty,oneof=desktop gateway shell"`
DisplayName string `json:"display_name"`
Image string `json:"image" binding:"required"`
}
diff --git a/backend/internal/handlers/websocket_handler.go b/backend/internal/handlers/websocket_handler.go
index 0d1b31b..98c29b4 100644
--- a/backend/internal/handlers/websocket_handler.go
+++ b/backend/internal/handlers/websocket_handler.go
@@ -27,9 +27,19 @@ func (h *WebSocketHandler) HandleWebSocket(c *gin.Context) {
c.JSON(http.StatusUnauthorized, gin.H{"error": "Unauthorized"})
return
}
+ roleRaw, _ := c.Get("userRole")
+ role, _ := roleRaw.(string)
+ topic := services.WebSocketTopicUser
+ if c.Query("topic") == string(services.WebSocketTopicRuntimeAdmin) {
+ if role != "admin" {
+ c.JSON(http.StatusForbidden, gin.H{"error": "Admin access required"})
+ return
+ }
+ topic = services.WebSocketTopicRuntimeAdmin
+ }
// Upgrade HTTP connection to WebSocket
- services.ServeWS(h.hub, c.Writer, c.Request, userID.(int))
+ services.ServeWSWithOptions(h.hub, c.Writer, c.Request, userID.(int), role, topic)
}
// GetConnectionCount returns the number of active WebSocket connections
diff --git a/backend/internal/handlers/workspace_file_handler.go b/backend/internal/handlers/workspace_file_handler.go
new file mode 100644
index 0000000..50ce2d7
--- /dev/null
+++ b/backend/internal/handlers/workspace_file_handler.go
@@ -0,0 +1,317 @@
+package handlers
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "mime"
+ "net/http"
+ "net/url"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "unicode"
+
+ "clawreef/internal/models"
+ "clawreef/internal/services"
+ "clawreef/internal/utils"
+
+ "github.com/gin-gonic/gin"
+)
+
+type WorkspaceFileHandler struct {
+ instanceService services.InstanceService
+ fileService services.WorkspaceFileService
+ runtimeWorkspaceFileService services.WorkspaceFileService
+}
+
+type createWorkspaceFolderRequest struct {
+ Path string `json:"path" binding:"required"`
+}
+
+type renameWorkspaceEntryRequest struct {
+ OldPath string `json:"old_path" binding:"required"`
+ NewPath string `json:"new_path" binding:"required"`
+}
+
+func NewWorkspaceFileHandler(instanceService services.InstanceService, fileService services.WorkspaceFileService, runtimeWorkspaceFileService ...services.WorkspaceFileService) *WorkspaceFileHandler {
+ runtimeFileService := fileService
+ if len(runtimeWorkspaceFileService) > 0 && runtimeWorkspaceFileService[0] != nil {
+ runtimeFileService = runtimeWorkspaceFileService[0]
+ }
+ return &WorkspaceFileHandler{
+ instanceService: instanceService,
+ fileService: fileService,
+ runtimeWorkspaceFileService: runtimeFileService,
+ }
+}
+
+func (h *WorkspaceFileHandler) List(c *gin.Context) {
+ _, service, scope, ok := h.workspaceScope(c)
+ if !ok {
+ return
+ }
+ entries, err := service.List(c.Request.Context(), scope, c.Query("path"))
+ if err != nil {
+ handleWorkspaceFileError(c, err)
+ return
+ }
+ utils.Success(c, http.StatusOK, "Workspace files retrieved successfully", gin.H{"entries": entries})
+}
+
+func (h *WorkspaceFileHandler) Preview(c *gin.Context) {
+ _, service, scope, ok := h.workspaceScope(c)
+ if !ok {
+ return
+ }
+
+ if strings.EqualFold(strings.TrimSpace(c.Query("raw")), "1") {
+ file, contentType, size, err := service.OpenPreview(c.Request.Context(), scope, c.Query("path"))
+ if err != nil {
+ handleWorkspaceFileError(c, err)
+ return
+ }
+ defer file.Close()
+ filename := safeWorkspaceDownloadName(filepath.Base(c.Query("path")))
+ streamWorkspaceFile(c, file, filename, contentType, "inline", size)
+ return
+ }
+
+ preview, err := service.Preview(c.Request.Context(), scope, c.Query("path"))
+ if err != nil {
+ handleWorkspaceFileError(c, err)
+ return
+ }
+ utils.Success(c, http.StatusOK, "Workspace preview retrieved successfully", gin.H{"preview": preview})
+}
+
+func (h *WorkspaceFileHandler) Download(c *gin.Context) {
+ _, service, scope, ok := h.workspaceScope(c)
+ if !ok {
+ return
+ }
+ file, filename, size, err := service.OpenDownload(c.Request.Context(), scope, c.Query("path"))
+ if err != nil {
+ handleWorkspaceFileError(c, err)
+ return
+ }
+ defer file.Close()
+
+ contentType := mime.TypeByExtension(strings.ToLower(filepath.Ext(filename)))
+ if contentType == "" {
+ contentType = "application/octet-stream"
+ }
+ streamWorkspaceFile(c, file, filename, contentType, "attachment", size)
+}
+
+func (h *WorkspaceFileHandler) Upload(c *gin.Context) {
+ _, service, scope, ok := h.workspaceScope(c)
+ if !ok {
+ return
+ }
+
+ maxBytes := services.WorkspaceUploadMaxBytes()
+ c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxBytes+(1<<20))
+ fileHeader, err := c.FormFile("file")
+ if err != nil {
+ var maxBytesErr *http.MaxBytesError
+ if errors.As(err, &maxBytesErr) {
+ utils.Error(c, http.StatusRequestEntityTooLarge, "workspace upload exceeds maximum size")
+ return
+ }
+ utils.Error(c, http.StatusBadRequest, "file is required")
+ return
+ }
+ if fileHeader.Size > maxBytes {
+ utils.Error(c, http.StatusRequestEntityTooLarge, "workspace upload exceeds maximum size")
+ return
+ }
+ file, err := fileHeader.Open()
+ if err != nil {
+ handleWorkspaceFileError(c, err)
+ return
+ }
+ defer file.Close()
+
+ entry, err := service.Upload(c.Request.Context(), scope, c.Query("path"), fileHeader.Filename, file, fileHeader.Size)
+ if err != nil {
+ handleWorkspaceFileError(c, err)
+ return
+ }
+ utils.Success(c, http.StatusCreated, "Workspace file uploaded successfully", entry)
+}
+
+func (h *WorkspaceFileHandler) Mkdir(c *gin.Context) {
+ _, service, scope, ok := h.workspaceScope(c)
+ if !ok {
+ return
+ }
+ var req createWorkspaceFolderRequest
+ if err := c.ShouldBindJSON(&req); err != nil {
+ utils.ValidationError(c, err)
+ return
+ }
+ entry, err := service.Mkdir(c.Request.Context(), scope, req.Path)
+ if err != nil {
+ handleWorkspaceFileError(c, err)
+ return
+ }
+ utils.Success(c, http.StatusCreated, "Workspace folder created successfully", entry)
+}
+
+func (h *WorkspaceFileHandler) Rename(c *gin.Context) {
+ _, service, scope, ok := h.workspaceScope(c)
+ if !ok {
+ return
+ }
+ var req renameWorkspaceEntryRequest
+ if err := c.ShouldBindJSON(&req); err != nil {
+ utils.ValidationError(c, err)
+ return
+ }
+ entry, err := service.Rename(c.Request.Context(), scope, req.OldPath, req.NewPath)
+ if err != nil {
+ handleWorkspaceFileError(c, err)
+ return
+ }
+ utils.Success(c, http.StatusOK, "Workspace entry renamed successfully", entry)
+}
+
+func (h *WorkspaceFileHandler) Delete(c *gin.Context) {
+ _, service, scope, ok := h.workspaceScope(c)
+ if !ok {
+ return
+ }
+ if err := service.Delete(c.Request.Context(), scope, c.Query("path")); err != nil {
+ handleWorkspaceFileError(c, err)
+ return
+ }
+ utils.Success(c, http.StatusOK, "Workspace entry deleted successfully", nil)
+}
+
+func (h *WorkspaceFileHandler) workspaceScope(c *gin.Context) (*models.Instance, services.WorkspaceFileService, services.WorkspaceFileScope, bool) {
+ id, err := strconv.Atoi(c.Param("id"))
+ if err != nil {
+ utils.Error(c, http.StatusBadRequest, "Invalid instance ID")
+ return nil, nil, services.WorkspaceFileScope{}, false
+ }
+
+ instance, err := h.instanceService.GetByID(id)
+ if err != nil {
+ utils.HandleError(c, err)
+ return nil, nil, services.WorkspaceFileScope{}, false
+ }
+ if instance == nil {
+ utils.Error(c, http.StatusNotFound, "Instance not found")
+ return nil, nil, services.WorkspaceFileScope{}, false
+ }
+
+ userIDRaw, ok := c.Get("userID")
+ if !ok {
+ utils.Error(c, http.StatusUnauthorized, "Unauthorized")
+ return nil, nil, services.WorkspaceFileScope{}, false
+ }
+ userID, ok := userIDRaw.(int)
+ if !ok {
+ utils.Error(c, http.StatusUnauthorized, "Unauthorized")
+ return nil, nil, services.WorkspaceFileScope{}, false
+ }
+ roleRaw, _ := c.Get("userRole")
+ userRole, _ := roleRaw.(string)
+ if userRole != "admin" && instance.UserID != userID {
+ utils.Error(c, http.StatusForbidden, "Access denied")
+ return nil, nil, services.WorkspaceFileScope{}, false
+ }
+ if isDesktopWorkspaceInstance(instance) {
+ return instance, h.runtimeWorkspaceFileService, services.WorkspaceFileScope{
+ InstanceID: instance.ID,
+ UserID: instance.UserID,
+ WorkspacePath: "/config",
+ }, true
+ }
+ if instance.WorkspacePath == nil || strings.TrimSpace(*instance.WorkspacePath) == "" {
+ utils.Error(c, http.StatusNotFound, "Workspace not found")
+ return nil, nil, services.WorkspaceFileScope{}, false
+ }
+
+ return instance, h.fileService, services.WorkspaceFileScope{
+ InstanceID: instance.ID,
+ UserID: instance.UserID,
+ WorkspacePath: *instance.WorkspacePath,
+ }, true
+}
+
+func isDesktopWorkspaceInstance(instance *models.Instance) bool {
+ if instance == nil {
+ return false
+ }
+ return strings.EqualFold(strings.TrimSpace(instance.InstanceMode), services.InstanceModePro) ||
+ strings.EqualFold(strings.TrimSpace(instance.RuntimeType), services.RuntimeBackendDesktop)
+}
+
+func streamWorkspaceFile(c *gin.Context, file io.ReadSeeker, filename, contentType, disposition string, size int64) {
+ safeName := safeWorkspaceDownloadName(filename)
+ c.Header("Content-Type", contentType)
+ c.Header("X-Content-Type-Options", "nosniff")
+ c.Header("Content-Disposition", workspaceContentDisposition(disposition, safeName))
+ if size >= 0 {
+ c.Header("Content-Length", strconv.FormatInt(size, 10))
+ }
+ if _, err := io.Copy(c.Writer, file); err != nil && !c.Writer.Written() {
+ utils.HandleError(c, err)
+ }
+}
+
+func workspaceContentDisposition(disposition, filename string) string {
+ disposition = strings.TrimSpace(strings.ToLower(disposition))
+ if disposition != "inline" {
+ disposition = "attachment"
+ }
+ escaped := strings.ReplaceAll(filename, `\`, `\\`)
+ escaped = strings.ReplaceAll(escaped, `"`, `\"`)
+ return fmt.Sprintf("%s; filename=\"%s\"; filename*=UTF-8''%s", disposition, escaped, url.PathEscape(filename))
+}
+
+func safeWorkspaceDownloadName(name string) string {
+ name = strings.TrimSpace(strings.ReplaceAll(name, "\\", "/"))
+ name = filepath.Base(name)
+ if name == "." || name == "/" || name == "" {
+ name = "download"
+ }
+ name = strings.Map(func(r rune) rune {
+ if unicode.IsControl(r) {
+ return -1
+ }
+ switch r {
+ case '/', '\\', ':', '*', '?', '"', '<', '>', '|':
+ return '-'
+ default:
+ return r
+ }
+ }, name)
+ name = strings.TrimSpace(name)
+ if name == "" || name == "." || name == ".." {
+ return "download"
+ }
+ return name
+}
+
+func handleWorkspaceFileError(c *gin.Context, err error) {
+ switch {
+ case errors.Is(err, services.ErrWorkspacePathNotFound):
+ utils.Error(c, http.StatusNotFound, err.Error())
+ case errors.Is(err, services.ErrWorkspacePreviewTooLarge), errors.Is(err, services.ErrWorkspaceUploadTooLarge):
+ utils.Error(c, http.StatusRequestEntityTooLarge, err.Error())
+ case errors.Is(err, services.ErrWorkspacePathEscape):
+ utils.Error(c, http.StatusForbidden, "Access denied")
+ case errors.Is(err, services.ErrWorkspacePathInvalid),
+ errors.Is(err, services.ErrWorkspaceDirectoryExpected),
+ errors.Is(err, services.ErrWorkspaceFileExpected),
+ errors.Is(err, services.ErrWorkspaceRootOperation),
+ errors.Is(err, services.ErrWorkspaceFileNameInvalid),
+ errors.Is(err, services.ErrWorkspaceEntryExists):
+ utils.Error(c, http.StatusBadRequest, err.Error())
+ default:
+ utils.HandleError(c, err)
+ }
+}
diff --git a/backend/internal/handlers/workspace_file_handler_test.go b/backend/internal/handlers/workspace_file_handler_test.go
new file mode 100644
index 0000000..fa95a8d
--- /dev/null
+++ b/backend/internal/handlers/workspace_file_handler_test.go
@@ -0,0 +1,242 @@
+package handlers
+
+import (
+ "context"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "strings"
+ "testing"
+ "time"
+
+ "clawreef/internal/models"
+ "clawreef/internal/services"
+
+ "github.com/gin-gonic/gin"
+)
+
+type fakeWorkspaceHandlerInstanceService struct {
+ instances map[int]*models.Instance
+}
+
+func (s *fakeWorkspaceHandlerInstanceService) Create(userID int, req services.CreateInstanceRequest) (*models.Instance, error) {
+ return nil, nil
+}
+func (s *fakeWorkspaceHandlerInstanceService) ValidateCreateRequests(userID int, requests []services.CreateInstanceRequest) error {
+ return nil
+}
+func (s *fakeWorkspaceHandlerInstanceService) GetByID(id int) (*models.Instance, error) {
+ return s.instances[id], nil
+}
+func (s *fakeWorkspaceHandlerInstanceService) GetByUserID(userID int, offset, limit int) ([]models.Instance, int, error) {
+ return nil, 0, nil
+}
+func (s *fakeWorkspaceHandlerInstanceService) GetAllInstances(offset, limit int) ([]models.Instance, int, error) {
+ return nil, 0, nil
+}
+func (s *fakeWorkspaceHandlerInstanceService) Start(instanceID int) error { return nil }
+func (s *fakeWorkspaceHandlerInstanceService) Stop(instanceID int) error { return nil }
+func (s *fakeWorkspaceHandlerInstanceService) Restart(instanceID int) error { return nil }
+func (s *fakeWorkspaceHandlerInstanceService) Delete(instanceID int) error { return nil }
+func (s *fakeWorkspaceHandlerInstanceService) Update(instanceID int, req services.UpdateInstanceRequest) error {
+ return nil
+}
+func (s *fakeWorkspaceHandlerInstanceService) GetInstanceStatus(instanceID int) (*services.InstanceStatus, error) {
+ return nil, nil
+}
+func (s *fakeWorkspaceHandlerInstanceService) ForceSyncInstance(instanceID int) error { return nil }
+
+type fakeWorkspaceFileService struct {
+ lastScope services.WorkspaceFileScope
+ listCalls int
+ file *os.File
+ filename string
+ size int64
+}
+
+func (s *fakeWorkspaceFileService) List(ctx context.Context, scope services.WorkspaceFileScope, relativePath string) ([]services.WorkspaceEntry, error) {
+ s.lastScope = scope
+ s.listCalls++
+ return []services.WorkspaceEntry{{Name: "readme.md", Path: "readme.md", ModifiedAt: time.Unix(1, 0), Previewable: true, Downloadable: true}}, nil
+}
+func (s *fakeWorkspaceFileService) Preview(ctx context.Context, scope services.WorkspaceFileScope, relativePath string) (*services.WorkspacePreview, error) {
+ s.lastScope = scope
+ return &services.WorkspacePreview{Kind: "text", Text: "ok"}, nil
+}
+func (s *fakeWorkspaceFileService) OpenPreview(ctx context.Context, scope services.WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) {
+ s.lastScope = scope
+ return s.file, "text/plain; charset=utf-8", s.size, nil
+}
+func (s *fakeWorkspaceFileService) OpenDownload(ctx context.Context, scope services.WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) {
+ s.lastScope = scope
+ return s.file, s.filename, s.size, nil
+}
+func (s *fakeWorkspaceFileService) Upload(ctx context.Context, scope services.WorkspaceFileScope, relativeDir string, filename string, reader io.Reader, size int64) (*services.WorkspaceEntry, error) {
+ s.lastScope = scope
+ return &services.WorkspaceEntry{Name: filename, Path: filename, Downloadable: true}, nil
+}
+func (s *fakeWorkspaceFileService) Mkdir(ctx context.Context, scope services.WorkspaceFileScope, relativePath string) (*services.WorkspaceEntry, error) {
+ s.lastScope = scope
+ return &services.WorkspaceEntry{Name: "docs", Path: relativePath, IsDir: true}, nil
+}
+func (s *fakeWorkspaceFileService) Rename(ctx context.Context, scope services.WorkspaceFileScope, oldPath, newPath string) (*services.WorkspaceEntry, error) {
+ s.lastScope = scope
+ return &services.WorkspaceEntry{Name: "renamed", Path: newPath}, nil
+}
+func (s *fakeWorkspaceFileService) Delete(ctx context.Context, scope services.WorkspaceFileScope, relativePath string) error {
+ s.lastScope = scope
+ return nil
+}
+
+func TestWorkspaceFileHandlerRejectsNonOwnerUser(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ workspacePath := t.TempDir()
+ instanceService := &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{
+ 77: {ID: 77, UserID: 20, WorkspacePath: &workspacePath},
+ }}
+ fileService := &fakeWorkspaceFileService{}
+ handler := NewWorkspaceFileHandler(instanceService, fileService)
+
+ router := workspaceFileTestRouter(10, "user")
+ router.GET("/api/v1/instances/:id/workspace/files", handler.List)
+
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/77/workspace/files", nil)
+ router.ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusForbidden {
+ t.Fatalf("status = %d, body = %s, want 403", rec.Code, rec.Body.String())
+ }
+ if fileService.listCalls != 0 {
+ t.Fatalf("workspace file service called %d times, want 0", fileService.listCalls)
+ }
+}
+
+func TestWorkspaceFileHandlerAdminUsesInstanceOwnerScope(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ workspacePath := t.TempDir()
+ instanceService := &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{
+ 77: {ID: 77, UserID: 20, WorkspacePath: &workspacePath},
+ }}
+ fileService := &fakeWorkspaceFileService{}
+ handler := NewWorkspaceFileHandler(instanceService, fileService)
+
+ router := workspaceFileTestRouter(10, "admin")
+ router.GET("/api/v1/instances/:id/workspace/files", handler.List)
+
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/77/workspace/files?path=", nil)
+ router.ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String())
+ }
+ if fileService.lastScope.InstanceID != 77 || fileService.lastScope.UserID != 20 || fileService.lastScope.WorkspacePath != workspacePath {
+ t.Fatalf("scope = %#v, want instance owner scope", fileService.lastScope)
+ }
+}
+
+func TestWorkspaceFileHandlerRequiresWorkspacePath(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ instanceService := &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{
+ 77: {ID: 77, UserID: 10},
+ }}
+ fileService := &fakeWorkspaceFileService{}
+ handler := NewWorkspaceFileHandler(instanceService, fileService)
+
+ router := workspaceFileTestRouter(10, "user")
+ router.GET("/api/v1/instances/:id/workspace/files", handler.List)
+
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/77/workspace/files", nil)
+ router.ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusNotFound {
+ t.Fatalf("status = %d, body = %s, want 404", rec.Code, rec.Body.String())
+ }
+ if fileService.listCalls != 0 {
+ t.Fatalf("workspace file service called %d times, want 0", fileService.listCalls)
+ }
+}
+
+func TestWorkspaceFileHandlerProDesktopUsesRuntimeConfigWorkspace(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ instanceService := &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{
+ 77: {
+ ID: 77,
+ UserID: 10,
+ RuntimeType: services.RuntimeBackendDesktop,
+ InstanceMode: services.InstanceModePro,
+ },
+ }}
+ localFileService := &fakeWorkspaceFileService{}
+ runtimeFileService := &fakeWorkspaceFileService{}
+ handler := NewWorkspaceFileHandler(instanceService, localFileService, runtimeFileService)
+
+ router := workspaceFileTestRouter(10, "user")
+ router.GET("/api/v1/instances/:id/workspace/files", handler.List)
+
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/77/workspace/files?path=", nil)
+ router.ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String())
+ }
+ if localFileService.listCalls != 0 {
+ t.Fatalf("local workspace service called %d times, want 0", localFileService.listCalls)
+ }
+ if runtimeFileService.listCalls != 1 {
+ t.Fatalf("runtime workspace service called %d times, want 1", runtimeFileService.listCalls)
+ }
+ if runtimeFileService.lastScope.WorkspacePath != "/config" {
+ t.Fatalf("runtime workspace path = %q, want /config", runtimeFileService.lastScope.WorkspacePath)
+ }
+}
+
+func TestWorkspaceFileHandlerDownloadUsesSafeContentDisposition(t *testing.T) {
+ gin.SetMode(gin.TestMode)
+ workspacePath := t.TempDir()
+ downloadFile, err := os.CreateTemp(t.TempDir(), "download-*")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := downloadFile.WriteString("body"); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := downloadFile.Seek(0, io.SeekStart); err != nil {
+ t.Fatal(err)
+ }
+
+ instanceService := &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{
+ 77: {ID: 77, UserID: 10, WorkspacePath: &workspacePath},
+ }}
+ fileService := &fakeWorkspaceFileService{file: downloadFile, filename: `bad"name/ignored.txt`, size: int64(len("body"))}
+ handler := NewWorkspaceFileHandler(instanceService, fileService)
+
+ router := workspaceFileTestRouter(10, "user")
+ router.GET("/api/v1/instances/:id/workspace/download", handler.Download)
+
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/77/workspace/download?path=report.txt", nil)
+ router.ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status = %d, body = %s, want 200", rec.Code, rec.Body.String())
+ }
+ disposition := rec.Header().Get("Content-Disposition")
+ if !strings.Contains(disposition, "attachment;") || strings.Contains(disposition, `"name/`) || strings.Contains(disposition, "\r") || strings.Contains(disposition, "\n") {
+ t.Fatalf("Content-Disposition = %q, want safe attachment filename", disposition)
+ }
+}
+
+func workspaceFileTestRouter(userID int, role string) *gin.Engine {
+ router := gin.New()
+ router.Use(func(c *gin.Context) {
+ c.Set("userID", userID)
+ c.Set("userRole", role)
+ c.Next()
+ })
+ return router
+}
diff --git a/backend/internal/middleware/auth_middleware.go b/backend/internal/middleware/auth_middleware.go
index 076e7c7..521dfe2 100644
--- a/backend/internal/middleware/auth_middleware.go
+++ b/backend/internal/middleware/auth_middleware.go
@@ -51,7 +51,11 @@ func Auth() gin.HandlerFunc {
}
// GatewayAuth accepts either a normal user access JWT or an instance lifecycle gateway token.
-func GatewayAuth(instanceRepo repository.InstanceRepository) gin.HandlerFunc {
+func GatewayAuth(instanceRepo repository.InstanceRepository, bindingRepos ...repository.InstanceRuntimeBindingRepository) gin.HandlerFunc {
+ var bindingRepo repository.InstanceRuntimeBindingRepository
+ if len(bindingRepos) > 0 {
+ bindingRepo = bindingRepos[0]
+ }
return func(c *gin.Context) {
tokenString, ok := extractToken(c)
if !ok {
@@ -90,11 +94,34 @@ func GatewayAuth(instanceRepo repository.InstanceRepository) gin.HandlerFunc {
c.Set("userID", instance.UserID)
c.Set("instanceID", instance.ID)
+ c.Set("instanceMode", gatewayInstanceMode(instance.InstanceMode, instance.RuntimeType))
+ c.Set("runtimeType", strings.TrimSpace(instance.RuntimeType))
+ if bindingRepo != nil {
+ if binding, err := bindingRepo.GetRunningByInstanceID(c.Request.Context(), instance.ID); err == nil && binding != nil {
+ c.Set("gatewayID", strings.TrimSpace(binding.GatewayID))
+ c.Set("runtimePodID", binding.RuntimePodID)
+ }
+ }
c.Set("gatewayAuthType", "instance")
c.Next()
}
}
+func gatewayInstanceMode(instanceMode, runtimeType string) string {
+ mode := strings.ToLower(strings.TrimSpace(instanceMode))
+ if mode == "lite" || mode == "pro" {
+ return mode
+ }
+ switch strings.ToLower(strings.TrimSpace(runtimeType)) {
+ case "gateway":
+ return "lite"
+ case "desktop", "shell":
+ return "pro"
+ default:
+ return mode
+ }
+}
+
func extractToken(c *gin.Context) (string, bool) {
authHeader := c.GetHeader("Authorization")
if authHeader != "" {
diff --git a/backend/internal/models/audit_event.go b/backend/internal/models/audit_event.go
index 95874c0..a367d86 100644
--- a/backend/internal/models/audit_event.go
+++ b/backend/internal/models/audit_event.go
@@ -10,6 +10,10 @@ type AuditEvent struct {
RequestID *string `db:"request_id" json:"request_id,omitempty"`
UserID *int `db:"user_id" json:"user_id,omitempty"`
InstanceID *int `db:"instance_id" json:"instance_id,omitempty"`
+ InstanceMode *string `db:"instance_mode" json:"instance_mode,omitempty"`
+ RuntimeType *string `db:"runtime_type" json:"runtime_type,omitempty"`
+ GatewayID *string `db:"gateway_id" json:"gateway_id,omitempty"`
+ RuntimePodID *int64 `db:"runtime_pod_id" json:"runtime_pod_id,omitempty"`
InvocationID *int `db:"invocation_id" json:"invocation_id,omitempty"`
EventType string `db:"event_type" json:"event_type"`
TrafficClass string `db:"traffic_class" json:"traffic_class"`
diff --git a/backend/internal/models/cost_record.go b/backend/internal/models/cost_record.go
index a08fcda..7bdcf69 100644
--- a/backend/internal/models/cost_record.go
+++ b/backend/internal/models/cost_record.go
@@ -10,6 +10,10 @@ type CostRecord struct {
RequestID *string `db:"request_id" json:"request_id,omitempty"`
UserID *int `db:"user_id" json:"user_id,omitempty"`
InstanceID *int `db:"instance_id" json:"instance_id,omitempty"`
+ InstanceMode *string `db:"instance_mode" json:"instance_mode,omitempty"`
+ RuntimeType *string `db:"runtime_type" json:"runtime_type,omitempty"`
+ GatewayID *string `db:"gateway_id" json:"gateway_id,omitempty"`
+ RuntimePodID *int64 `db:"runtime_pod_id" json:"runtime_pod_id,omitempty"`
InvocationID *int `db:"invocation_id" json:"invocation_id,omitempty"`
ModelID *int `db:"model_id" json:"model_id,omitempty"`
ProviderType string `db:"provider_type" json:"provider_type"`
diff --git a/backend/internal/models/instance.go b/backend/internal/models/instance.go
index 2675d43..0de5d32 100644
--- a/backend/internal/models/instance.go
+++ b/backend/internal/models/instance.go
@@ -12,6 +12,7 @@ type Instance struct {
Description *string `db:"description" json:"description,omitempty"`
Type string `db:"type" json:"type"`
RuntimeType string `db:"runtime_type" json:"runtime_type"`
+ InstanceMode string `db:"instance_mode" json:"instance_mode"`
Status string `db:"status" json:"status"`
CPUCores float64 `db:"cpu_cores" json:"cpu_cores"`
MemoryGB int `db:"memory_gb" json:"memory_gb"`
@@ -26,6 +27,10 @@ type Instance struct {
EnvironmentOverridesJSON *string `db:"environment_overrides_json" json:"-"`
StorageClass string `db:"storage_class" json:"storage_class"`
MountPath string `db:"mount_path" json:"mount_path"`
+ WorkspacePath *string `db:"workspace_path" json:"workspace_path,omitempty"`
+ WorkspaceUsageBytes int64 `db:"workspace_usage_bytes" json:"workspace_usage_bytes"`
+ RuntimeGeneration int `db:"runtime_generation" json:"runtime_generation"`
+ RuntimeErrorMessage *string `db:"runtime_error_message" json:"runtime_error_message,omitempty"`
PodName *string `db:"pod_name" json:"pod_name,omitempty"`
PodNamespace *string `db:"pod_namespace" json:"pod_namespace,omitempty"`
PodIP *string `db:"pod_ip" json:"pod_ip,omitempty"`
diff --git a/backend/internal/models/instance_external_access.go b/backend/internal/models/instance_external_access.go
new file mode 100644
index 0000000..1acc5f3
--- /dev/null
+++ b/backend/internal/models/instance_external_access.go
@@ -0,0 +1,25 @@
+package models
+
+import "time"
+
+type InstanceExternalAccess struct {
+ ID int64 `db:"id,primarykey,autoincrement" json:"id"`
+ InstanceID int `db:"instance_id" json:"instance_id"`
+ Enabled bool `db:"enabled" json:"enabled"`
+ AuthMode string `db:"auth_mode" json:"auth_mode"`
+ PublicSlug *string `db:"public_slug" json:"-"`
+ PublicTokenHash *string `db:"public_token_hash" json:"-"`
+ ShortCodeHash *string `db:"short_code_hash" json:"-"`
+ PasswordHash *string `db:"api_key_hash" json:"-"`
+ PasswordValue *string `db:"password_value" json:"-"`
+ PasswordHint *string `db:"api_key_prefix" json:"password_hint,omitempty"`
+ ExpiresAt *time.Time `db:"expires_at" json:"expires_at,omitempty"`
+ CreatedBy *int `db:"created_by" json:"created_by,omitempty"`
+ LastUsedAt *time.Time `db:"last_used_at" json:"last_used_at,omitempty"`
+ CreatedAt time.Time `db:"created_at" json:"created_at"`
+ UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
+}
+
+func (InstanceExternalAccess) TableName() string {
+ return "instance_external_access"
+}
diff --git a/backend/internal/models/instance_runtime_binding.go b/backend/internal/models/instance_runtime_binding.go
new file mode 100644
index 0000000..1dff9e5
--- /dev/null
+++ b/backend/internal/models/instance_runtime_binding.go
@@ -0,0 +1,24 @@
+package models
+
+import "time"
+
+type InstanceRuntimeBinding struct {
+ ID int64 `db:"id,primarykey,autoincrement" json:"id"`
+ InstanceID int `db:"instance_id" json:"instance_id"`
+ RuntimePodID int64 `db:"runtime_pod_id" json:"runtime_pod_id"`
+ RuntimeType string `db:"runtime_type" json:"runtime_type"`
+ GatewayID string `db:"gateway_id" json:"gateway_id"`
+ GatewayPort int `db:"gateway_port" json:"gateway_port"`
+ GatewayPID *int `db:"gateway_pid" json:"gateway_pid,omitempty"`
+ WorkspacePath string `db:"workspace_path" json:"workspace_path"`
+ State string `db:"state" json:"state"`
+ Generation int `db:"generation" json:"generation"`
+ LastHealthAt *time.Time `db:"last_health_at" json:"last_health_at,omitempty"`
+ ErrorMessage *string `db:"error_message" json:"error_message,omitempty"`
+ CreatedAt time.Time `db:"created_at" json:"created_at"`
+ UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
+}
+
+func (InstanceRuntimeBinding) TableName() string {
+ return "instance_runtime_bindings"
+}
diff --git a/backend/internal/models/model_invocation.go b/backend/internal/models/model_invocation.go
index 6a114be..9f9f87b 100644
--- a/backend/internal/models/model_invocation.go
+++ b/backend/internal/models/model_invocation.go
@@ -10,6 +10,10 @@ type ModelInvocation struct {
RequestID string `db:"request_id" json:"request_id"`
UserID *int `db:"user_id" json:"user_id,omitempty"`
InstanceID *int `db:"instance_id" json:"instance_id,omitempty"`
+ InstanceMode *string `db:"instance_mode" json:"instance_mode,omitempty"`
+ RuntimeType *string `db:"runtime_type" json:"runtime_type,omitempty"`
+ GatewayID *string `db:"gateway_id" json:"gateway_id,omitempty"`
+ RuntimePodID *int64 `db:"runtime_pod_id" json:"runtime_pod_id,omitempty"`
ModelID *int `db:"model_id" json:"model_id,omitempty"`
ProviderType string `db:"provider_type" json:"provider_type"`
RequestedModel string `db:"requested_model" json:"requested_model"`
diff --git a/backend/internal/models/risk_hit.go b/backend/internal/models/risk_hit.go
index abdaefb..3bdc613 100644
--- a/backend/internal/models/risk_hit.go
+++ b/backend/internal/models/risk_hit.go
@@ -10,6 +10,10 @@ type RiskHit struct {
RequestID *string `db:"request_id" json:"request_id,omitempty"`
UserID *int `db:"user_id" json:"user_id,omitempty"`
InstanceID *int `db:"instance_id" json:"instance_id,omitempty"`
+ InstanceMode *string `db:"instance_mode" json:"instance_mode,omitempty"`
+ RuntimeType *string `db:"runtime_type" json:"runtime_type,omitempty"`
+ GatewayID *string `db:"gateway_id" json:"gateway_id,omitempty"`
+ RuntimePodID *int64 `db:"runtime_pod_id" json:"runtime_pod_id,omitempty"`
InvocationID *int `db:"invocation_id" json:"invocation_id,omitempty"`
RuleID string `db:"rule_id" json:"rule_id"`
RuleName string `db:"rule_name" json:"rule_name"`
diff --git a/backend/internal/models/runtime_pod.go b/backend/internal/models/runtime_pod.go
new file mode 100644
index 0000000..69c90ef
--- /dev/null
+++ b/backend/internal/models/runtime_pod.go
@@ -0,0 +1,33 @@
+package models
+
+import "time"
+
+type RuntimePod struct {
+ ID int64 `db:"id,primarykey,autoincrement" json:"id"`
+ RuntimeType string `db:"runtime_type" json:"runtime_type"`
+ Namespace string `db:"namespace" json:"namespace"`
+ PodName string `db:"pod_name" json:"pod_name"`
+ PodUID *string `db:"pod_uid" json:"pod_uid,omitempty"`
+ PodIP *string `db:"pod_ip" json:"pod_ip,omitempty"`
+ NodeName *string `db:"node_name" json:"node_name,omitempty"`
+ DeploymentName string `db:"deployment_name" json:"deployment_name"`
+ ImageRef string `db:"image_ref" json:"image_ref"`
+ AgentEndpoint *string `db:"agent_endpoint" json:"agent_endpoint,omitempty"`
+ State string `db:"state" json:"state"`
+ Capacity int `db:"capacity" json:"capacity"`
+ UsedSlots int `db:"used_slots" json:"used_slots"`
+ Draining bool `db:"draining" json:"draining"`
+ CPUMillisUsed int64 `db:"cpu_millis_used" json:"cpu_millis_used"`
+ MemoryBytesUsed int64 `db:"memory_bytes_used" json:"memory_bytes_used"`
+ DiskBytesUsed int64 `db:"disk_bytes_used" json:"disk_bytes_used"`
+ NetworkRXBytes int64 `db:"network_rx_bytes" json:"network_rx_bytes"`
+ NetworkTXBytes int64 `db:"network_tx_bytes" json:"network_tx_bytes"`
+ MetricsJSON *string `db:"metrics_json" json:"-"`
+ LastSeenAt *time.Time `db:"last_seen_at" json:"last_seen_at,omitempty"`
+ CreatedAt time.Time `db:"created_at" json:"created_at"`
+ UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
+}
+
+func (RuntimePod) TableName() string {
+ return "runtime_pods"
+}
diff --git a/backend/internal/models/runtime_rollout.go b/backend/internal/models/runtime_rollout.go
new file mode 100644
index 0000000..5b7502e
--- /dev/null
+++ b/backend/internal/models/runtime_rollout.go
@@ -0,0 +1,22 @@
+package models
+
+import "time"
+
+type RuntimeRollout struct {
+ ID int64 `db:"id,primarykey,autoincrement" json:"id"`
+ RuntimeType string `db:"runtime_type" json:"runtime_type"`
+ TargetImageRef string `db:"target_image_ref" json:"target_image_ref"`
+ Status string `db:"status" json:"status"`
+ BatchSize int `db:"batch_size" json:"batch_size"`
+ MaxUnavailable int `db:"max_unavailable" json:"max_unavailable"`
+ StartedBy *int `db:"started_by" json:"started_by,omitempty"`
+ StartedAt *time.Time `db:"started_at" json:"started_at,omitempty"`
+ FinishedAt *time.Time `db:"finished_at" json:"finished_at,omitempty"`
+ ErrorMessage *string `db:"error_message" json:"error_message,omitempty"`
+ CreatedAt time.Time `db:"created_at" json:"created_at"`
+ UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
+}
+
+func (RuntimeRollout) TableName() string {
+ return "runtime_rollouts"
+}
diff --git a/backend/internal/models/team.go b/backend/internal/models/team.go
index 7f5fc8d..080c3df 100644
--- a/backend/internal/models/team.go
+++ b/backend/internal/models/team.go
@@ -65,6 +65,7 @@ type TeamMember struct {
DisplayName string `db:"display_name" json:"display_name"`
Role string `db:"role" json:"role"`
RuntimeType string `db:"runtime_type" json:"runtime_type"`
+ InstanceMode string `db:"instance_mode" json:"instance_mode"`
Description *string `db:"description" json:"description,omitempty"`
Status string `db:"status" json:"status"`
CurrentTaskID *int `db:"current_task_id" json:"current_task_id,omitempty"`
diff --git a/backend/internal/models/workspace_file_audit.go b/backend/internal/models/workspace_file_audit.go
new file mode 100644
index 0000000..eb28539
--- /dev/null
+++ b/backend/internal/models/workspace_file_audit.go
@@ -0,0 +1,17 @@
+package models
+
+import "time"
+
+type WorkspaceFileAudit struct {
+ ID int64 `db:"id,primarykey,autoincrement" json:"id"`
+ InstanceID int `db:"instance_id" json:"instance_id"`
+ UserID int `db:"user_id" json:"user_id"`
+ Action string `db:"action" json:"action"`
+ RelativePath string `db:"relative_path" json:"relative_path"`
+ Bytes int64 `db:"bytes" json:"bytes"`
+ CreatedAt time.Time `db:"created_at" json:"created_at"`
+}
+
+func (WorkspaceFileAudit) TableName() string {
+ return "workspace_file_audits"
+}
diff --git a/backend/internal/repository/audit_event_repository.go b/backend/internal/repository/audit_event_repository.go
index 84afac6..377a364 100644
--- a/backend/internal/repository/audit_event_repository.go
+++ b/backend/internal/repository/audit_event_repository.go
@@ -36,6 +36,10 @@ CREATE TABLE IF NOT EXISTS audit_events (
request_id VARCHAR(100) NULL,
user_id INT NULL,
instance_id INT NULL,
+ instance_mode VARCHAR(16) NULL,
+ runtime_type VARCHAR(32) NULL,
+ gateway_id VARCHAR(128) NULL,
+ runtime_pod_id BIGINT NULL,
invocation_id INT NULL,
event_type VARCHAR(100) NOT NULL,
traffic_class VARCHAR(50) NOT NULL,
@@ -46,6 +50,7 @@ CREATE TABLE IF NOT EXISTS audit_events (
INDEX idx_audit_events_trace_id (trace_id),
INDEX idx_audit_events_request_id (request_id),
INDEX idx_audit_events_user_id (user_id),
+ INDEX idx_audit_events_gateway_id (gateway_id),
INDEX idx_audit_events_invocation_id (invocation_id),
INDEX idx_audit_events_event_type (event_type),
INDEX idx_audit_events_created_at (created_at)
diff --git a/backend/internal/repository/cost_record_repository.go b/backend/internal/repository/cost_record_repository.go
index e305732..51bb713 100644
--- a/backend/internal/repository/cost_record_repository.go
+++ b/backend/internal/repository/cost_record_repository.go
@@ -37,6 +37,10 @@ CREATE TABLE IF NOT EXISTS cost_records (
request_id VARCHAR(100) NULL,
user_id INT NULL,
instance_id INT NULL,
+ instance_mode VARCHAR(16) NULL,
+ runtime_type VARCHAR(32) NULL,
+ gateway_id VARCHAR(128) NULL,
+ runtime_pod_id BIGINT NULL,
invocation_id INT NULL,
model_id INT NULL,
provider_type VARCHAR(100) NOT NULL,
@@ -53,6 +57,7 @@ CREATE TABLE IF NOT EXISTS cost_records (
recorded_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
INDEX idx_cost_records_trace_id (trace_id),
INDEX idx_cost_records_user_id (user_id),
+ INDEX idx_cost_records_gateway_id (gateway_id),
INDEX idx_cost_records_model_id (model_id),
INDEX idx_cost_records_provider_type (provider_type),
INDEX idx_cost_records_recorded_at (recorded_at)
diff --git a/backend/internal/repository/instance_external_access_repository.go b/backend/internal/repository/instance_external_access_repository.go
new file mode 100644
index 0000000..cd99504
--- /dev/null
+++ b/backend/internal/repository/instance_external_access_repository.go
@@ -0,0 +1,107 @@
+package repository
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "clawreef/internal/models"
+ "github.com/upper/db/v4"
+)
+
+type InstanceExternalAccessRepository interface {
+ GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceExternalAccess, error)
+ GetByShortCodeHash(ctx context.Context, codeHash string) (*models.InstanceExternalAccess, error)
+ Upsert(ctx context.Context, access *models.InstanceExternalAccess) error
+ Disable(ctx context.Context, instanceID int) error
+ MarkUsed(ctx context.Context, id int64) error
+}
+
+type instanceExternalAccessRepository struct {
+ sess db.Session
+}
+
+func NewInstanceExternalAccessRepository(sess db.Session) InstanceExternalAccessRepository {
+ return &instanceExternalAccessRepository{sess: sess}
+}
+
+func (r *instanceExternalAccessRepository) GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceExternalAccess, error) {
+ _ = ctx
+ var access models.InstanceExternalAccess
+ if err := r.sess.Collection(access.TableName()).Find(db.Cond{"instance_id": instanceID}).One(&access); err != nil {
+ if err == db.ErrNoMoreRows {
+ return nil, nil
+ }
+ return nil, fmt.Errorf("failed to get instance external access: %w", err)
+ }
+ return &access, nil
+}
+
+func (r *instanceExternalAccessRepository) GetByShortCodeHash(ctx context.Context, codeHash string) (*models.InstanceExternalAccess, error) {
+ _ = ctx
+ var access models.InstanceExternalAccess
+ if err := r.sess.Collection(access.TableName()).Find(db.Cond{"short_code_hash": codeHash}).One(&access); err != nil {
+ if err == db.ErrNoMoreRows {
+ return nil, nil
+ }
+ return nil, fmt.Errorf("failed to get instance external access by short code hash: %w", err)
+ }
+ return &access, nil
+}
+
+func (r *instanceExternalAccessRepository) Upsert(ctx context.Context, access *models.InstanceExternalAccess) error {
+ now := time.Now().UTC()
+ if access.CreatedAt.IsZero() {
+ access.CreatedAt = now
+ }
+ access.UpdatedAt = now
+ _, err := r.sess.SQL().ExecContext(ctx, `
+ INSERT INTO instance_external_access (
+ instance_id, enabled, auth_mode, public_slug, short_code_hash, public_token_hash,
+ api_key_hash, password_value, api_key_prefix, expires_at, created_by,
+ created_at, updated_at
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ ON DUPLICATE KEY UPDATE
+ enabled = VALUES(enabled),
+ auth_mode = VALUES(auth_mode),
+ public_slug = VALUES(public_slug),
+ short_code_hash = VALUES(short_code_hash),
+ public_token_hash = VALUES(public_token_hash),
+ api_key_hash = VALUES(api_key_hash),
+ password_value = VALUES(password_value),
+ api_key_prefix = VALUES(api_key_prefix),
+ expires_at = VALUES(expires_at),
+ created_by = VALUES(created_by),
+ updated_at = VALUES(updated_at)
+ `, access.InstanceID, access.Enabled, access.AuthMode, access.PublicSlug, access.ShortCodeHash, access.PublicTokenHash,
+ access.PasswordHash, access.PasswordValue, access.PasswordHint, access.ExpiresAt, access.CreatedBy,
+ access.CreatedAt, access.UpdatedAt)
+ if err != nil {
+ return fmt.Errorf("failed to upsert instance external access: %w", err)
+ }
+ return nil
+}
+
+func (r *instanceExternalAccessRepository) Disable(ctx context.Context, instanceID int) error {
+ _, err := r.sess.SQL().ExecContext(ctx, `
+ UPDATE instance_external_access
+ SET enabled = 0, updated_at = ?
+ WHERE instance_id = ?
+ `, time.Now().UTC(), instanceID)
+ if err != nil {
+ return fmt.Errorf("failed to disable instance external access: %w", err)
+ }
+ return nil
+}
+
+func (r *instanceExternalAccessRepository) MarkUsed(ctx context.Context, id int64) error {
+ _, err := r.sess.SQL().ExecContext(ctx, `
+ UPDATE instance_external_access
+ SET last_used_at = ?, updated_at = ?
+ WHERE id = ?
+ `, time.Now().UTC(), time.Now().UTC(), id)
+ if err != nil {
+ return fmt.Errorf("failed to mark instance external access used: %w", err)
+ }
+ return nil
+}
diff --git a/backend/internal/repository/instance_repository.go b/backend/internal/repository/instance_repository.go
index 1102afd..1b593e3 100644
--- a/backend/internal/repository/instance_repository.go
+++ b/backend/internal/repository/instance_repository.go
@@ -1,13 +1,19 @@
package repository
import (
+ "context"
+ "database/sql"
+ "errors"
"fmt"
"strings"
+ "time"
"clawreef/internal/models"
"github.com/upper/db/v4"
)
+var ErrStaleRuntimeGeneration = errors.New("stale runtime generation")
+
// InstanceRepository defines the interface for instance data operations
type InstanceRepository interface {
Create(instance *models.Instance) error
@@ -18,8 +24,14 @@ type InstanceRepository interface {
CountAll() (int, error)
GetByUserID(userID int, offset, limit int) ([]models.Instance, error)
CountByUserID(userID int) (int, error)
+ CountActiveByMode(ctx context.Context, mode string) (int, error)
ExistsByUserIDAndName(userID int, name string) (bool, error)
GetAllRunning() ([]models.Instance, error)
+ GetV2DesiredRunning(ctx context.Context, limit int) ([]models.Instance, error)
+ GetV2Creating(ctx context.Context, limit int) ([]models.Instance, error)
+ UpdateRuntimeState(ctx context.Context, id int, status string, generation int, message *string) error
+ SetWorkspacePath(ctx context.Context, id int, workspacePath string) error
+ UpdateWorkspaceUsage(ctx context.Context, id int, usageBytes int64) error
Update(instance *models.Instance) error
Delete(id int) error
}
@@ -121,6 +133,27 @@ func (r *instanceRepository) CountByUserID(userID int) (int, error) {
return int(count), nil
}
+func (r *instanceRepository) CountActiveByMode(ctx context.Context, mode string) (int, error) {
+ normalized := strings.TrimSpace(strings.ToLower(mode))
+ if normalized == "" {
+ return 0, nil
+ }
+ row, err := r.sess.SQL().QueryRowContext(ctx, `
+ SELECT COUNT(*)
+ FROM instances
+ WHERE instance_mode = ?
+ AND status IN ('creating', 'running')
+ `, normalized)
+ if err != nil {
+ return 0, fmt.Errorf("failed to count active instances by mode: %w", err)
+ }
+ var count int
+ if err := row.Scan(&count); err != nil {
+ return 0, fmt.Errorf("failed to scan active instances by mode count: %w", err)
+ }
+ return count, nil
+}
+
// ExistsByUserIDAndName checks whether the user already has an instance with the same display name.
func (r *instanceRepository) ExistsByUserIDAndName(userID int, name string) (bool, error) {
instances, err := r.GetByUserID(userID, 0, 1000)
@@ -155,6 +188,139 @@ func (r *instanceRepository) GetAllRunning() ([]models.Instance, error) {
return instances, nil
}
+func (r *instanceRepository) GetV2DesiredRunning(ctx context.Context, limit int) ([]models.Instance, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ if limit <= 0 {
+ limit = 100
+ }
+ var instances []models.Instance
+ query, args := buildV2SchedulerInstanceQuery(v2DesiredRunningStatuses(), limit)
+ iter := r.sess.SQL().IteratorContext(ctx, query, args...)
+ defer iter.Close()
+ if err := iter.All(&instances); err != nil {
+ return nil, fmt.Errorf("failed to get v2 desired running instances: %w", err)
+ }
+ return instances, nil
+}
+
+func v2DesiredRunningStatuses() []string {
+ return []string{"creating", "running", "error"}
+}
+
+func (r *instanceRepository) GetV2Creating(ctx context.Context, limit int) ([]models.Instance, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ if limit <= 0 {
+ limit = 100
+ }
+ var instances []models.Instance
+ query, args := buildV2SchedulerInstanceQuery([]string{"creating"}, limit)
+ iter := r.sess.SQL().IteratorContext(ctx, query, args...)
+ defer iter.Close()
+ if err := iter.All(&instances); err != nil {
+ return nil, fmt.Errorf("failed to get v2 creating instances: %w", err)
+ }
+ return instances, nil
+}
+
+func buildV2SchedulerInstanceQuery(statuses []string, limit int) (string, []any) {
+ if limit <= 0 {
+ limit = 100
+ }
+ if len(statuses) == 0 {
+ statuses = []string{"creating", "running"}
+ }
+ statusPlaceholders := strings.TrimRight(strings.Repeat("?, ", len(statuses)), ", ")
+ args := make([]any, 0, len(statuses)+5)
+ for _, status := range statuses {
+ args = append(args, status)
+ }
+ args = append(args, "gateway", "lite", "openclaw", "hermes", limit)
+ return fmt.Sprintf(`
+ SELECT *
+ FROM instances
+ WHERE status IN (%s)
+ AND runtime_type = ?
+ AND instance_mode = ?
+ AND workspace_path IS NOT NULL
+ AND TRIM(workspace_path) <> ''
+ AND type IN (?, ?)
+ ORDER BY id
+ LIMIT ?
+ `, statusPlaceholders), args
+}
+
+func (r *instanceRepository) UpdateRuntimeState(ctx context.Context, id int, status string, generation int, message *string) error {
+ res, err := r.sess.SQL().ExecContext(ctx, `
+ UPDATE instances
+ SET status = ?, runtime_generation = ?, runtime_error_message = ?, updated_at = ?
+ WHERE id = ? AND runtime_generation <= ?
+ `, status, generation, message, time.Now().UTC(), id, generation)
+ if err != nil {
+ return fmt.Errorf("failed to update instance runtime state: %w", err)
+ }
+ affected, err := res.RowsAffected()
+ if err != nil {
+ return fmt.Errorf("failed to inspect instance runtime state update: %w", err)
+ }
+ if affected == 0 {
+ currentGeneration, err := r.getRuntimeGeneration(ctx, id)
+ if err != nil {
+ return err
+ }
+ if currentGeneration > generation {
+ return ErrStaleRuntimeGeneration
+ }
+ }
+ return nil
+}
+
+func (r *instanceRepository) getRuntimeGeneration(ctx context.Context, id int) (int, error) {
+ var currentGeneration int
+ row, err := r.sess.SQL().QueryRowContext(ctx, `
+ SELECT runtime_generation
+ FROM instances
+ WHERE id = ?
+ `, id)
+ if err != nil {
+ return 0, fmt.Errorf("failed to query instance runtime generation: %w", err)
+ }
+ if err := row.Scan(¤tGeneration); err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return 0, ErrStaleRuntimeGeneration
+ }
+ return 0, fmt.Errorf("failed to scan instance runtime generation: %w", err)
+ }
+ return currentGeneration, nil
+}
+
+func (r *instanceRepository) SetWorkspacePath(ctx context.Context, id int, workspacePath string) error {
+ _, err := r.sess.SQL().ExecContext(ctx, `
+ UPDATE instances
+ SET workspace_path = ?, updated_at = ?
+ WHERE id = ?
+ `, workspacePath, time.Now().UTC(), id)
+ if err != nil {
+ return fmt.Errorf("failed to set instance workspace path: %w", err)
+ }
+ return nil
+}
+
+func (r *instanceRepository) UpdateWorkspaceUsage(ctx context.Context, id int, usageBytes int64) error {
+ _, err := r.sess.SQL().ExecContext(ctx, `
+ UPDATE instances
+ SET workspace_usage_bytes = ?, updated_at = ?
+ WHERE id = ?
+ `, usageBytes, time.Now().UTC(), id)
+ if err != nil {
+ return fmt.Errorf("failed to update instance workspace usage: %w", err)
+ }
+ return nil
+}
+
// Update updates an instance
func (r *instanceRepository) Update(instance *models.Instance) error {
err := r.sess.Collection("instances").Find(db.Cond{"id": instance.ID}).Update(instance)
diff --git a/backend/internal/repository/instance_repository_test.go b/backend/internal/repository/instance_repository_test.go
new file mode 100644
index 0000000..1a45955
--- /dev/null
+++ b/backend/internal/repository/instance_repository_test.go
@@ -0,0 +1,58 @@
+package repository
+
+import (
+ "reflect"
+ "strings"
+ "testing"
+)
+
+func TestBuildV2SchedulerInstanceQueryRequiresWorkspaceTypeAndStatuses(t *testing.T) {
+ query, args := buildV2SchedulerInstanceQuery([]string{"creating", "running"}, 25)
+ normalized := normalizeSQLForTest(query)
+
+ requiredFragments := []string{
+ "FROM instances",
+ "status IN (?, ?)",
+ "runtime_type = ?",
+ "instance_mode = ?",
+ "workspace_path IS NOT NULL",
+ "TRIM(workspace_path) <> ''",
+ "type IN (?, ?)",
+ "ORDER BY id",
+ "LIMIT ?",
+ }
+ for _, fragment := range requiredFragments {
+ if !strings.Contains(normalized, fragment) {
+ t.Fatalf("query %q does not contain %q", normalized, fragment)
+ }
+ }
+
+ wantArgs := []any{"creating", "running", "gateway", "lite", "openclaw", "hermes", 25}
+ if !reflect.DeepEqual(args, wantArgs) {
+ t.Fatalf("args = %#v, want %#v", args, wantArgs)
+ }
+}
+
+func TestV2DesiredRunningStatusesIncludeRecoverableErrorCandidates(t *testing.T) {
+ want := []string{"creating", "running", "error"}
+ if got := v2DesiredRunningStatuses(); !reflect.DeepEqual(got, want) {
+ t.Fatalf("desired running statuses = %#v, want %#v", got, want)
+ }
+}
+
+func TestBuildV2SchedulerInstanceQueryDefaultsLimit(t *testing.T) {
+ query, args := buildV2SchedulerInstanceQuery([]string{"creating"}, 0)
+ normalized := normalizeSQLForTest(query)
+
+ if !strings.Contains(normalized, "status IN (?)") {
+ t.Fatalf("query %q does not contain single status predicate", normalized)
+ }
+ wantArgs := []any{"creating", "gateway", "lite", "openclaw", "hermes", 100}
+ if !reflect.DeepEqual(args, wantArgs) {
+ t.Fatalf("args = %#v, want %#v", args, wantArgs)
+ }
+}
+
+func normalizeSQLForTest(query string) string {
+ return strings.Join(strings.Fields(query), " ")
+}
diff --git a/backend/internal/repository/instance_runtime_binding_repository.go b/backend/internal/repository/instance_runtime_binding_repository.go
new file mode 100644
index 0000000..f5d2969
--- /dev/null
+++ b/backend/internal/repository/instance_runtime_binding_repository.go
@@ -0,0 +1,213 @@
+package repository
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+ "time"
+
+ "clawreef/internal/models"
+
+ "github.com/upper/db/v4"
+)
+
+type InstanceRuntimeBindingRepository interface {
+ Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error
+ GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error)
+ GetRunningByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error)
+ ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error)
+ ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error)
+ UpdateRunning(ctx context.Context, instanceID int, generation int, gatewayID string, port int, pid *int) error
+ UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error
+ DeleteByInstanceID(ctx context.Context, instanceID int) error
+ DeleteByInstanceIDAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64) error
+}
+
+type instanceRuntimeBindingRepository struct {
+ sess db.Session
+}
+
+func NewInstanceRuntimeBindingRepository(sess db.Session) InstanceRuntimeBindingRepository {
+ return &instanceRuntimeBindingRepository{sess: sess}
+}
+
+func (r *instanceRuntimeBindingRepository) Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ ensureTimestamps(&binding.CreatedAt, &binding.UpdatedAt)
+ res, err := r.sess.Collection("instance_runtime_bindings").Insert(binding)
+ if err != nil {
+ return fmt.Errorf("failed to create instance runtime binding: %w", err)
+ }
+ if id, ok := res.ID().(int64); ok {
+ binding.ID = id
+ }
+ return nil
+}
+
+func (r *instanceRuntimeBindingRepository) GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ var binding models.InstanceRuntimeBinding
+ if err := r.sess.Collection("instance_runtime_bindings").Find(db.Cond{"instance_id": instanceID}).One(&binding); err != nil {
+ if err == db.ErrNoMoreRows {
+ return nil, nil
+ }
+ return nil, fmt.Errorf("failed to get instance runtime binding: %w", err)
+ }
+ return &binding, nil
+}
+
+func (r *instanceRuntimeBindingRepository) GetRunningByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ var binding models.InstanceRuntimeBinding
+ if err := r.sess.Collection("instance_runtime_bindings").Find(db.Cond{"instance_id": instanceID, "state": "running"}).One(&binding); err != nil {
+ if err == db.ErrNoMoreRows {
+ return nil, nil
+ }
+ return nil, fmt.Errorf("failed to get running instance runtime binding: %w", err)
+ }
+ return &binding, nil
+}
+
+func (r *instanceRuntimeBindingRepository) ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ var bindings []models.InstanceRuntimeBinding
+ if err := r.sess.Collection("instance_runtime_bindings").Find(db.Cond{"runtime_pod_id": runtimePodID}).OrderBy("id").All(&bindings); err != nil {
+ return nil, fmt.Errorf("failed to list instance runtime bindings: %w", err)
+ }
+ return bindings, nil
+}
+
+func (r *instanceRuntimeBindingRepository) ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ if len(runtimePodIDs) == 0 {
+ return []models.InstanceRuntimeBinding{}, nil
+ }
+ var bindings []models.InstanceRuntimeBinding
+ if err := r.sess.Collection("instance_runtime_bindings").Find(db.Cond{"runtime_pod_id IN": runtimePodIDs}).OrderBy("runtime_pod_id", "id").All(&bindings); err != nil {
+ return nil, fmt.Errorf("failed to list instance runtime bindings by pods: %w", err)
+ }
+ return bindings, nil
+}
+
+func (r *instanceRuntimeBindingRepository) UpdateRunning(ctx context.Context, instanceID int, generation int, gatewayID string, port int, pid *int) error {
+ now := time.Now().UTC()
+ res, err := r.sess.SQL().ExecContext(ctx, `
+ UPDATE instance_runtime_bindings
+ SET state = 'running', generation = ?, gateway_id = ?, gateway_port = ?, gateway_pid = ?,
+ last_health_at = ?, error_message = NULL, updated_at = ?
+ WHERE instance_id = ? AND generation <= ?
+ `, generation, gatewayID, port, pid, now, now, instanceID, generation)
+ if err != nil {
+ return fmt.Errorf("failed to update running instance runtime binding: %w", err)
+ }
+ affected, err := res.RowsAffected()
+ if err != nil {
+ return fmt.Errorf("failed to inspect running instance runtime binding update: %w", err)
+ }
+ if affected == 0 {
+ currentGeneration, err := r.getGeneration(ctx, instanceID)
+ if err != nil {
+ return err
+ }
+ if currentGeneration > generation {
+ return ErrStaleRuntimeGeneration
+ }
+ }
+ return nil
+}
+
+func (r *instanceRuntimeBindingRepository) UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error {
+ res, err := r.sess.SQL().ExecContext(ctx, `
+ UPDATE instance_runtime_bindings
+ SET state = ?, generation = ?, error_message = ?, updated_at = ?
+ WHERE instance_id = ? AND generation <= ?
+ `, state, generation, message, time.Now().UTC(), instanceID, generation)
+ if err != nil {
+ return fmt.Errorf("failed to update instance runtime binding state: %w", err)
+ }
+ affected, err := res.RowsAffected()
+ if err != nil {
+ return fmt.Errorf("failed to inspect instance runtime binding state update: %w", err)
+ }
+ if affected == 0 {
+ currentGeneration, err := r.getGeneration(ctx, instanceID)
+ if err != nil {
+ return err
+ }
+ if currentGeneration > generation {
+ return ErrStaleRuntimeGeneration
+ }
+ }
+ return nil
+}
+
+func (r *instanceRuntimeBindingRepository) getGeneration(ctx context.Context, instanceID int) (int, error) {
+ var currentGeneration int
+ row, err := r.sess.SQL().QueryRowContext(ctx, `
+ SELECT generation
+ FROM instance_runtime_bindings
+ WHERE instance_id = ?
+ `, instanceID)
+ if err != nil {
+ return 0, fmt.Errorf("failed to query instance runtime binding generation: %w", err)
+ }
+ if err := row.Scan(¤tGeneration); err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return 0, ErrStaleRuntimeGeneration
+ }
+ return 0, fmt.Errorf("failed to scan instance runtime binding generation: %w", err)
+ }
+ return currentGeneration, nil
+}
+
+func (r *instanceRuntimeBindingRepository) DeleteByInstanceID(ctx context.Context, instanceID int) error {
+ _, err := r.sess.SQL().ExecContext(ctx, `
+ DELETE FROM instance_runtime_bindings
+ WHERE instance_id = ?
+ `, instanceID)
+ if err != nil {
+ return fmt.Errorf("failed to delete instance runtime binding: %w", err)
+ }
+ return nil
+}
+
+func (r *instanceRuntimeBindingRepository) DeleteByInstanceIDAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64) error {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ return r.sess.TxContext(ctx, func(tx db.Session) error {
+ res, err := tx.SQL().ExecContext(ctx, `
+ DELETE FROM instance_runtime_bindings
+ WHERE instance_id = ? AND runtime_pod_id = ?
+ `, instanceID, runtimePodID)
+ if err != nil {
+ return fmt.Errorf("failed to delete instance runtime binding: %w", err)
+ }
+ affected, err := res.RowsAffected()
+ if err != nil {
+ return fmt.Errorf("failed to inspect instance runtime binding delete: %w", err)
+ }
+ if affected == 0 {
+ return nil
+ }
+ if _, err := tx.SQL().ExecContext(ctx, `
+ UPDATE runtime_pods
+ SET used_slots = CASE WHEN used_slots > 0 THEN used_slots - 1 ELSE 0 END, updated_at = ?
+ WHERE id = ?
+ `, time.Now().UTC(), runtimePodID); err != nil {
+ return fmt.Errorf("failed to release runtime pod slot: %w", err)
+ }
+ return nil
+ }, nil)
+}
diff --git a/backend/internal/repository/model_invocation_repository.go b/backend/internal/repository/model_invocation_repository.go
index c713f3f..a581cd7 100644
--- a/backend/internal/repository/model_invocation_repository.go
+++ b/backend/internal/repository/model_invocation_repository.go
@@ -40,6 +40,10 @@ CREATE TABLE IF NOT EXISTS model_invocations (
request_id VARCHAR(100) NOT NULL,
user_id INT NULL,
instance_id INT NULL,
+ instance_mode VARCHAR(16) NULL,
+ runtime_type VARCHAR(32) NULL,
+ gateway_id VARCHAR(128) NULL,
+ runtime_pod_id BIGINT NULL,
model_id INT NULL,
provider_type VARCHAR(100) NOT NULL,
requested_model VARCHAR(255) NOT NULL,
@@ -62,6 +66,7 @@ CREATE TABLE IF NOT EXISTS model_invocations (
INDEX idx_model_invocations_request_id (request_id),
INDEX idx_model_invocations_user_id (user_id),
INDEX idx_model_invocations_instance_id (instance_id),
+ INDEX idx_model_invocations_gateway_id (gateway_id),
INDEX idx_model_invocations_model_id (model_id),
INDEX idx_model_invocations_status (status),
INDEX idx_model_invocations_created_at (created_at)
diff --git a/backend/internal/repository/risk_hit_repository.go b/backend/internal/repository/risk_hit_repository.go
index ecad93b..864187c 100644
--- a/backend/internal/repository/risk_hit_repository.go
+++ b/backend/internal/repository/risk_hit_repository.go
@@ -35,6 +35,10 @@ CREATE TABLE IF NOT EXISTS risk_hits (
request_id VARCHAR(100) NULL,
user_id INT NULL,
instance_id INT NULL,
+ instance_mode VARCHAR(16) NULL,
+ runtime_type VARCHAR(32) NULL,
+ gateway_id VARCHAR(128) NULL,
+ runtime_pod_id BIGINT NULL,
invocation_id INT NULL,
rule_id VARCHAR(100) NOT NULL,
rule_name VARCHAR(255) NOT NULL,
@@ -45,6 +49,7 @@ CREATE TABLE IF NOT EXISTS risk_hits (
INDEX idx_risk_hits_trace_id (trace_id),
INDEX idx_risk_hits_request_id (request_id),
INDEX idx_risk_hits_user_id (user_id),
+ INDEX idx_risk_hits_gateway_id (gateway_id),
INDEX idx_risk_hits_invocation_id (invocation_id),
INDEX idx_risk_hits_severity (severity),
INDEX idx_risk_hits_created_at (created_at)
diff --git a/backend/internal/repository/runtime_pod_repository.go b/backend/internal/repository/runtime_pod_repository.go
new file mode 100644
index 0000000..e168a12
--- /dev/null
+++ b/backend/internal/repository/runtime_pod_repository.go
@@ -0,0 +1,224 @@
+package repository
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "clawreef/internal/models"
+
+ "github.com/upper/db/v4"
+)
+
+type RuntimePodMetricsUpdate struct {
+ CPUMillisUsed int64
+ MemoryBytesUsed int64
+ DiskBytesUsed int64
+ NetworkRXBytes int64
+ NetworkTXBytes int64
+ MetricsJSON *string
+ LastSeenAt *time.Time
+}
+
+type RuntimePodRepository interface {
+ UpsertFromAgent(ctx context.Context, pod *models.RuntimePod) error
+ GetByID(ctx context.Context, id int64) (*models.RuntimePod, error)
+ GetByNamespaceName(ctx context.Context, namespace, podName string) (*models.RuntimePod, error)
+ List(ctx context.Context, runtimeType string) ([]models.RuntimePod, error)
+ ListSchedulable(ctx context.Context, runtimeType string) ([]models.RuntimePod, error)
+ TryClaimSlot(ctx context.Context, podID int64) (bool, error)
+ ReleaseSlot(ctx context.Context, podID int64) error
+ MarkState(ctx context.Context, podID int64, state string, draining bool) error
+ UpdateHeartbeat(ctx context.Context, podID int64, state string, usedSlots int, capacity int, draining bool, lastSeenAt time.Time) error
+ MarkUnseenUnhealthy(ctx context.Context, cutoff time.Time) error
+ UpdateMetrics(ctx context.Context, podID int64, metrics RuntimePodMetricsUpdate) error
+}
+
+type runtimePodRepository struct {
+ sess db.Session
+}
+
+func NewRuntimePodRepository(sess db.Session) RuntimePodRepository {
+ return &runtimePodRepository{sess: sess}
+}
+
+func (r *runtimePodRepository) UpsertFromAgent(ctx context.Context, pod *models.RuntimePod) error {
+ ensureTimestamps(&pod.CreatedAt, &pod.UpdatedAt)
+ res, err := r.sess.SQL().ExecContext(ctx, `
+ INSERT INTO runtime_pods (
+ runtime_type, namespace, pod_name, pod_uid, pod_ip, node_name, deployment_name,
+ image_ref, agent_endpoint, state, capacity, used_slots, draining, cpu_millis_used,
+ memory_bytes_used, disk_bytes_used, network_rx_bytes, network_tx_bytes, metrics_json,
+ last_seen_at, created_at, updated_at
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ ON DUPLICATE KEY UPDATE
+ id = LAST_INSERT_ID(id),
+ runtime_type = VALUES(runtime_type),
+ pod_uid = VALUES(pod_uid),
+ pod_ip = VALUES(pod_ip),
+ node_name = VALUES(node_name),
+ deployment_name = VALUES(deployment_name),
+ image_ref = VALUES(image_ref),
+ agent_endpoint = VALUES(agent_endpoint),
+ capacity = VALUES(capacity),
+ cpu_millis_used = VALUES(cpu_millis_used),
+ memory_bytes_used = VALUES(memory_bytes_used),
+ disk_bytes_used = VALUES(disk_bytes_used),
+ network_rx_bytes = VALUES(network_rx_bytes),
+ network_tx_bytes = VALUES(network_tx_bytes),
+ metrics_json = VALUES(metrics_json),
+ last_seen_at = VALUES(last_seen_at),
+ updated_at = VALUES(updated_at)
+ `, pod.RuntimeType, pod.Namespace, pod.PodName, pod.PodUID, pod.PodIP, pod.NodeName, pod.DeploymentName,
+ pod.ImageRef, pod.AgentEndpoint, pod.State, pod.Capacity, pod.UsedSlots, pod.Draining, pod.CPUMillisUsed,
+ pod.MemoryBytesUsed, pod.DiskBytesUsed, pod.NetworkRXBytes, pod.NetworkTXBytes, pod.MetricsJSON,
+ pod.LastSeenAt, pod.CreatedAt, pod.UpdatedAt)
+ if err != nil {
+ return fmt.Errorf("failed to upsert runtime pod: %w", err)
+ }
+ if id, err := res.LastInsertId(); err == nil {
+ pod.ID = id
+ }
+ return nil
+}
+
+func (r *runtimePodRepository) GetByID(ctx context.Context, id int64) (*models.RuntimePod, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ var pod models.RuntimePod
+ if err := r.sess.Collection("runtime_pods").Find(db.Cond{"id": id}).One(&pod); err != nil {
+ if err == db.ErrNoMoreRows {
+ return nil, nil
+ }
+ return nil, fmt.Errorf("failed to get runtime pod: %w", err)
+ }
+ return &pod, nil
+}
+
+func (r *runtimePodRepository) GetByNamespaceName(ctx context.Context, namespace, podName string) (*models.RuntimePod, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ var pod models.RuntimePod
+ if err := r.sess.Collection("runtime_pods").Find(db.Cond{"namespace": namespace, "pod_name": podName}).One(&pod); err != nil {
+ if err == db.ErrNoMoreRows {
+ return nil, nil
+ }
+ return nil, fmt.Errorf("failed to get runtime pod by namespace/name: %w", err)
+ }
+ return &pod, nil
+}
+
+func (r *runtimePodRepository) List(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ cond := db.Cond{}
+ if runtimeType != "" {
+ cond["runtime_type"] = runtimeType
+ }
+ var pods []models.RuntimePod
+ if err := r.sess.Collection("runtime_pods").Find(cond).OrderBy("namespace", "pod_name").All(&pods); err != nil {
+ return nil, fmt.Errorf("failed to list runtime pods: %w", err)
+ }
+ return pods, nil
+}
+
+func (r *runtimePodRepository) ListSchedulable(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ var pods []models.RuntimePod
+ iter := r.sess.SQL().IteratorContext(ctx, `
+ SELECT *
+ FROM runtime_pods
+ WHERE runtime_type = ? AND state = 'ready' AND draining = 0 AND used_slots < capacity
+ ORDER BY used_slots, id
+ `, runtimeType)
+ defer iter.Close()
+ if err := iter.All(&pods); err != nil {
+ return nil, fmt.Errorf("failed to list schedulable runtime pods: %w", err)
+ }
+ return pods, nil
+}
+
+func (r *runtimePodRepository) TryClaimSlot(ctx context.Context, podID int64) (bool, error) {
+ res, err := r.sess.SQL().ExecContext(ctx, `
+ UPDATE runtime_pods
+ SET used_slots = used_slots + 1, updated_at = ?
+ WHERE id = ? AND state = 'ready' AND draining = 0 AND used_slots < capacity
+ `, time.Now().UTC(), podID)
+ if err != nil {
+ return false, fmt.Errorf("failed to claim runtime pod slot: %w", err)
+ }
+ affected, err := res.RowsAffected()
+ if err != nil {
+ return false, fmt.Errorf("failed to inspect runtime pod slot claim: %w", err)
+ }
+ return affected == 1, nil
+}
+
+func (r *runtimePodRepository) ReleaseSlot(ctx context.Context, podID int64) error {
+ _, err := r.sess.SQL().ExecContext(ctx, `
+ UPDATE runtime_pods
+ SET used_slots = CASE WHEN used_slots > 0 THEN used_slots - 1 ELSE 0 END, updated_at = ?
+ WHERE id = ?
+ `, time.Now().UTC(), podID)
+ if err != nil {
+ return fmt.Errorf("failed to release runtime pod slot: %w", err)
+ }
+ return nil
+}
+
+func (r *runtimePodRepository) MarkState(ctx context.Context, podID int64, state string, draining bool) error {
+ _, err := r.sess.SQL().ExecContext(ctx, `
+ UPDATE runtime_pods
+ SET state = ?, draining = ?, updated_at = ?
+ WHERE id = ?
+ `, state, draining, time.Now().UTC(), podID)
+ if err != nil {
+ return fmt.Errorf("failed to mark runtime pod state: %w", err)
+ }
+ return nil
+}
+
+func (r *runtimePodRepository) UpdateHeartbeat(ctx context.Context, podID int64, state string, usedSlots int, capacity int, draining bool, lastSeenAt time.Time) error {
+ _, err := r.sess.SQL().ExecContext(ctx, `
+ UPDATE runtime_pods
+ SET state = ?, used_slots = ?, capacity = ?, draining = ?, last_seen_at = ?, updated_at = ?
+ WHERE id = ?
+ `, state, usedSlots, capacity, draining, lastSeenAt, time.Now().UTC(), podID)
+ if err != nil {
+ return fmt.Errorf("failed to update runtime pod heartbeat: %w", err)
+ }
+ return nil
+}
+
+func (r *runtimePodRepository) MarkUnseenUnhealthy(ctx context.Context, cutoff time.Time) error {
+ _, err := r.sess.SQL().ExecContext(ctx, `
+ UPDATE runtime_pods
+ SET state = 'unhealthy', updated_at = ?
+ WHERE (last_seen_at IS NULL OR last_seen_at < ?) AND state <> 'unhealthy'
+ `, time.Now().UTC(), cutoff)
+ if err != nil {
+ return fmt.Errorf("failed to mark unseen runtime pods unhealthy: %w", err)
+ }
+ return nil
+}
+
+func (r *runtimePodRepository) UpdateMetrics(ctx context.Context, podID int64, metrics RuntimePodMetricsUpdate) error {
+ _, err := r.sess.SQL().ExecContext(ctx, `
+ UPDATE runtime_pods
+ SET cpu_millis_used = ?, memory_bytes_used = ?, disk_bytes_used = ?,
+ network_rx_bytes = ?, network_tx_bytes = ?, metrics_json = ?,
+ last_seen_at = ?, updated_at = ?
+ WHERE id = ?
+ `, metrics.CPUMillisUsed, metrics.MemoryBytesUsed, metrics.DiskBytesUsed,
+ metrics.NetworkRXBytes, metrics.NetworkTXBytes, metrics.MetricsJSON,
+ metrics.LastSeenAt, time.Now().UTC(), podID)
+ if err != nil {
+ return fmt.Errorf("failed to update runtime pod metrics: %w", err)
+ }
+ return nil
+}
diff --git a/backend/internal/repository/runtime_rollout_repository.go b/backend/internal/repository/runtime_rollout_repository.go
new file mode 100644
index 0000000..43c5e0d
--- /dev/null
+++ b/backend/internal/repository/runtime_rollout_repository.go
@@ -0,0 +1,82 @@
+package repository
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "clawreef/internal/models"
+
+ "github.com/upper/db/v4"
+)
+
+type RuntimeRolloutRepository interface {
+ Create(ctx context.Context, rollout *models.RuntimeRollout) error
+ GetByID(ctx context.Context, id int64) (*models.RuntimeRollout, error)
+ ListActive(ctx context.Context, runtimeType string) ([]models.RuntimeRollout, error)
+ UpdateStatus(ctx context.Context, id int64, status string, startedAt *time.Time, finishedAt *time.Time, message *string) error
+}
+
+type runtimeRolloutRepository struct {
+ sess db.Session
+}
+
+func NewRuntimeRolloutRepository(sess db.Session) RuntimeRolloutRepository {
+ return &runtimeRolloutRepository{sess: sess}
+}
+
+func (r *runtimeRolloutRepository) Create(ctx context.Context, rollout *models.RuntimeRollout) error {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ ensureTimestamps(&rollout.CreatedAt, &rollout.UpdatedAt)
+ res, err := r.sess.Collection("runtime_rollouts").Insert(rollout)
+ if err != nil {
+ return fmt.Errorf("failed to create runtime rollout: %w", err)
+ }
+ if id, ok := res.ID().(int64); ok {
+ rollout.ID = id
+ }
+ return nil
+}
+
+func (r *runtimeRolloutRepository) GetByID(ctx context.Context, id int64) (*models.RuntimeRollout, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ var rollout models.RuntimeRollout
+ if err := r.sess.Collection("runtime_rollouts").Find(db.Cond{"id": id}).One(&rollout); err != nil {
+ if err == db.ErrNoMoreRows {
+ return nil, nil
+ }
+ return nil, fmt.Errorf("failed to get runtime rollout: %w", err)
+ }
+ return &rollout, nil
+}
+
+func (r *runtimeRolloutRepository) ListActive(ctx context.Context, runtimeType string) ([]models.RuntimeRollout, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ cond := db.Cond{"status IN": []string{"pending", "running"}}
+ if runtimeType != "" {
+ cond["runtime_type"] = runtimeType
+ }
+ var rollouts []models.RuntimeRollout
+ if err := r.sess.Collection("runtime_rollouts").Find(cond).OrderBy("created_at", "id").All(&rollouts); err != nil {
+ return nil, fmt.Errorf("failed to list active runtime rollouts: %w", err)
+ }
+ return rollouts, nil
+}
+
+func (r *runtimeRolloutRepository) UpdateStatus(ctx context.Context, id int64, status string, startedAt *time.Time, finishedAt *time.Time, message *string) error {
+ _, err := r.sess.SQL().ExecContext(ctx, `
+ UPDATE runtime_rollouts
+ SET status = ?, started_at = ?, finished_at = ?, error_message = ?, updated_at = ?
+ WHERE id = ?
+ `, status, startedAt, finishedAt, message, time.Now().UTC(), id)
+ if err != nil {
+ return fmt.Errorf("failed to update runtime rollout status: %w", err)
+ }
+ return nil
+}
diff --git a/backend/internal/repository/system_image_setting_repository.go b/backend/internal/repository/system_image_setting_repository.go
index 32f6d97..f9f9e08 100644
--- a/backend/internal/repository/system_image_setting_repository.go
+++ b/backend/internal/repository/system_image_setting_repository.go
@@ -36,7 +36,7 @@ func (r *systemImageSettingRepository) ensureTable() {
CREATE TABLE IF NOT EXISTS system_image_settings (
id INT AUTO_INCREMENT PRIMARY KEY,
instance_type VARCHAR(50) NOT NULL,
- runtime_type ENUM('desktop', 'shell') NOT NULL DEFAULT 'desktop',
+ runtime_type ENUM('desktop', 'shell', 'gateway') NOT NULL DEFAULT 'desktop',
display_name VARCHAR(255) NOT NULL,
image VARCHAR(500) NOT NULL,
is_enabled BOOLEAN NOT NULL DEFAULT TRUE,
@@ -52,6 +52,7 @@ CREATE TABLE IF NOT EXISTS system_image_settings (
r.ensureIsEnabledColumn()
r.ensureRuntimeTypeColumn()
+ r.ensureRuntimeTypeAllowsGateway()
r.ensureInstanceTypeIsNotUnique()
r.ensureInstanceTypeIndex()
}
@@ -96,12 +97,35 @@ WHERE table_schema = DATABASE()
}
if count == 0 {
- if _, err := r.sess.SQL().Exec("ALTER TABLE system_image_settings ADD COLUMN runtime_type ENUM('desktop', 'shell') NOT NULL DEFAULT 'desktop' AFTER instance_type"); err != nil {
+ if _, err := r.sess.SQL().Exec("ALTER TABLE system_image_settings ADD COLUMN runtime_type ENUM('desktop', 'shell', 'gateway') NOT NULL DEFAULT 'desktop' AFTER instance_type"); err != nil {
panic(fmt.Errorf("failed to ensure system_image_settings.runtime_type column: %w", err))
}
}
}
+func (r *systemImageSettingRepository) ensureRuntimeTypeAllowsGateway() {
+ var columnType string
+ row, err := r.sess.SQL().QueryRow(`
+SELECT COLUMN_TYPE
+FROM information_schema.columns
+WHERE table_schema = DATABASE()
+ AND table_name = 'system_image_settings'
+ AND column_name = 'runtime_type'
+`)
+ if err != nil {
+ panic(fmt.Errorf("failed to inspect system_image_settings runtime_type enum: %w", err))
+ }
+ if err := row.Scan(&columnType); err != nil {
+ panic(fmt.Errorf("failed to scan system_image_settings runtime_type enum: %w", err))
+ }
+
+ if !strings.Contains(columnType, "'gateway'") {
+ if _, err := r.sess.SQL().Exec("ALTER TABLE system_image_settings MODIFY COLUMN runtime_type ENUM('desktop', 'shell', 'gateway') NOT NULL DEFAULT 'desktop'"); err != nil {
+ panic(fmt.Errorf("failed to allow system_image_settings.runtime_type gateway: %w", err))
+ }
+ }
+}
+
func (r *systemImageSettingRepository) ensureInstanceTypeIsNotUnique() {
rows, err := r.sess.SQL().Query(`
SELECT DISTINCT INDEX_NAME
diff --git a/backend/internal/repository/workspace_file_audit_repository.go b/backend/internal/repository/workspace_file_audit_repository.go
new file mode 100644
index 0000000..5cac1c1
--- /dev/null
+++ b/backend/internal/repository/workspace_file_audit_repository.go
@@ -0,0 +1,40 @@
+package repository
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "clawreef/internal/models"
+
+ "github.com/upper/db/v4"
+)
+
+type WorkspaceFileAuditRepository interface {
+ Create(ctx context.Context, audit *models.WorkspaceFileAudit) error
+}
+
+type workspaceFileAuditRepository struct {
+ sess db.Session
+}
+
+func NewWorkspaceFileAuditRepository(sess db.Session) WorkspaceFileAuditRepository {
+ return &workspaceFileAuditRepository{sess: sess}
+}
+
+func (r *workspaceFileAuditRepository) Create(ctx context.Context, audit *models.WorkspaceFileAudit) error {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ if audit.CreatedAt.IsZero() {
+ audit.CreatedAt = time.Now().UTC()
+ }
+ res, err := r.sess.Collection("workspace_file_audits").Insert(audit)
+ if err != nil {
+ return fmt.Errorf("failed to create workspace file audit: %w", err)
+ }
+ if id, ok := res.ID().(int64); ok {
+ audit.ID = id
+ }
+ return nil
+}
diff --git a/backend/internal/services/instance_env.go b/backend/internal/services/instance_env.go
index 98f5dfd..cdf3954 100644
--- a/backend/internal/services/instance_env.go
+++ b/backend/internal/services/instance_env.go
@@ -129,3 +129,21 @@ func buildInstancePodEnv(instance *models.Instance, runtimeEnv, gatewayEnv, agen
return resolved, nil
}
+
+func buildInstanceGatewayEnv(instance *models.Instance, gatewayEnv map[string]string) (map[string]string, error) {
+ if instance == nil {
+ return nil, fmt.Errorf("instance is required")
+ }
+
+ overrides, err := parseEnvironmentOverridesJSON(instance.EnvironmentOverridesJSON)
+ if err != nil {
+ return nil, err
+ }
+
+ resolved := mergeEnvMaps(gatewayEnv, nil)
+ resolved = withInstanceProxyEnv(instance.Type, instance.ID, resolved)
+ resolved["CLAWMANAGER_RUNTIME_TYPE"] = normalizeInstanceRuntimeType(instance.RuntimeType)
+ resolved = mergeEnvMaps(resolved, overrides)
+
+ return resolved, nil
+}
diff --git a/backend/internal/services/instance_external_access_service.go b/backend/internal/services/instance_external_access_service.go
new file mode 100644
index 0000000..e440118
--- /dev/null
+++ b/backend/internal/services/instance_external_access_service.go
@@ -0,0 +1,286 @@
+package services
+
+import (
+ "context"
+ "crypto/rand"
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "strings"
+ "time"
+
+ "clawreef/internal/models"
+ "clawreef/internal/repository"
+)
+
+const (
+ ExternalAccessModeShareLink = "share_link"
+ ExternalAccessModePassword = "password"
+
+ ExternalAccessExpirationPreset = "preset"
+ ExternalAccessExpirationCustom = "custom"
+ ExternalAccessExpirationPermanent = "permanent"
+
+ ExternalAccessPreset1Hour = "1h"
+ ExternalAccessPreset24Hours = "24h"
+ ExternalAccessPreset7Days = "7d"
+ ExternalAccessPreset30Days = "30d"
+)
+
+type ExternalAccessExpirationRequest struct {
+ Mode string `json:"expires_mode,omitempty"`
+ Preset string `json:"expires_preset,omitempty"`
+ ExpiresAt *time.Time `json:"expires_at,omitempty"`
+}
+
+type EnableShareLinkResult struct {
+ Access *models.InstanceExternalAccess `json:"access"`
+ ShareURL string `json:"share_url,omitempty"`
+}
+
+type PasswordExternalAccessResult struct {
+ Access *models.InstanceExternalAccess `json:"access"`
+ Password string `json:"password"`
+ ShareURL string `json:"share_url,omitempty"`
+}
+
+type InstanceExternalAccessService interface {
+ Get(ctx context.Context, instanceID int) (*models.InstanceExternalAccess, error)
+ EnableShareLink(ctx context.Context, instanceID, createdBy int, expiration ExternalAccessExpirationRequest) (*EnableShareLinkResult, error)
+ CreatePassword(ctx context.Context, instanceID, createdBy int, expiration ExternalAccessExpirationRequest) (*PasswordExternalAccessResult, error)
+ Disable(ctx context.Context, instanceID int) error
+ ResolveShortLink(ctx context.Context, code string) (*models.InstanceExternalAccess, error)
+ ValidateShortLink(ctx context.Context, code, password string) (*models.InstanceExternalAccess, error)
+}
+
+type instanceExternalAccessService struct {
+ repo repository.InstanceExternalAccessRepository
+}
+
+func NewInstanceExternalAccessService(repo repository.InstanceExternalAccessRepository) InstanceExternalAccessService {
+ return &instanceExternalAccessService{repo: repo}
+}
+
+func (s *instanceExternalAccessService) Get(ctx context.Context, instanceID int) (*models.InstanceExternalAccess, error) {
+ if s == nil || s.repo == nil {
+ return nil, fmt.Errorf("instance external access repository is not configured")
+ }
+ return s.repo.GetByInstanceID(ctx, instanceID)
+}
+
+func (s *instanceExternalAccessService) EnableShareLink(ctx context.Context, instanceID, createdBy int, expiration ExternalAccessExpirationRequest) (*EnableShareLinkResult, error) {
+ if s == nil || s.repo == nil {
+ return nil, fmt.Errorf("instance external access repository is not configured")
+ }
+ code, codeHash, err := generateShortCode()
+ if err != nil {
+ return nil, err
+ }
+ expiresAt, err := resolveExternalAccessExpiresAt(expiration)
+ if err != nil {
+ return nil, err
+ }
+ access := &models.InstanceExternalAccess{
+ InstanceID: instanceID,
+ Enabled: true,
+ AuthMode: ExternalAccessModeShareLink,
+ ShortCodeHash: &codeHash,
+ PublicSlug: &code,
+ PublicTokenHash: nil,
+ ExpiresAt: expiresAt,
+ CreatedBy: &createdBy,
+ PasswordHash: nil,
+ PasswordValue: nil,
+ PasswordHint: nil,
+ LastUsedAt: nil,
+ }
+ if err := s.repo.Upsert(ctx, access); err != nil {
+ return nil, err
+ }
+ saved, err := s.repo.GetByInstanceID(ctx, instanceID)
+ if err != nil {
+ return nil, err
+ }
+ return &EnableShareLinkResult{
+ Access: saved,
+ ShareURL: shortExternalAccessURL(code),
+ }, nil
+}
+
+func (s *instanceExternalAccessService) CreatePassword(ctx context.Context, instanceID, createdBy int, expiration ExternalAccessExpirationRequest) (*PasswordExternalAccessResult, error) {
+ if s == nil || s.repo == nil {
+ return nil, fmt.Errorf("instance external access repository is not configured")
+ }
+ code, codeHash, err := generateShortCode()
+ if err != nil {
+ return nil, err
+ }
+ password, err := randomToken("pwd", 16)
+ if err != nil {
+ return nil, err
+ }
+ expiresAt, err := resolveExternalAccessExpiresAt(expiration)
+ if err != nil {
+ return nil, err
+ }
+ passwordHash := hashExternalSecret(password)
+ hint := password
+ if len(hint) > 12 {
+ hint = hint[:12]
+ }
+ access := &models.InstanceExternalAccess{
+ InstanceID: instanceID,
+ Enabled: true,
+ AuthMode: ExternalAccessModePassword,
+ PublicSlug: &code,
+ ShortCodeHash: &codeHash,
+ PasswordHash: &passwordHash,
+ PasswordValue: &password,
+ PasswordHint: &hint,
+ ExpiresAt: expiresAt,
+ CreatedBy: &createdBy,
+ PublicTokenHash: nil,
+ LastUsedAt: nil,
+ }
+ if err := s.repo.Upsert(ctx, access); err != nil {
+ return nil, err
+ }
+ saved, err := s.repo.GetByInstanceID(ctx, instanceID)
+ if err != nil {
+ return nil, err
+ }
+ return &PasswordExternalAccessResult{Access: saved, Password: password, ShareURL: shortExternalAccessURL(code)}, nil
+}
+
+func (s *instanceExternalAccessService) Disable(ctx context.Context, instanceID int) error {
+ if s == nil || s.repo == nil {
+ return fmt.Errorf("instance external access repository is not configured")
+ }
+ return s.repo.Disable(ctx, instanceID)
+}
+
+func (s *instanceExternalAccessService) ResolveShortLink(ctx context.Context, code string) (*models.InstanceExternalAccess, error) {
+ if s == nil || s.repo == nil {
+ return nil, fmt.Errorf("instance external access repository is not configured")
+ }
+ code = strings.TrimSpace(code)
+ if code == "" {
+ return nil, fmt.Errorf("short link code is required")
+ }
+ access, err := s.repo.GetByShortCodeHash(ctx, hashExternalSecret(code))
+ if err != nil {
+ return nil, err
+ }
+ if access == nil || !access.Enabled {
+ return nil, fmt.Errorf("external access is not enabled")
+ }
+ if access.ExpiresAt != nil && time.Now().UTC().After(*access.ExpiresAt) {
+ return nil, fmt.Errorf("external access has expired")
+ }
+ return access, nil
+}
+
+func (s *instanceExternalAccessService) ValidateShortLink(ctx context.Context, code, password string) (*models.InstanceExternalAccess, error) {
+ access, err := s.ResolveShortLink(ctx, code)
+ if err != nil {
+ return nil, err
+ }
+ switch access.AuthMode {
+ case ExternalAccessModeShareLink:
+ // The short code itself is the share-link capability. A matching hash is enough.
+ case ExternalAccessModePassword:
+ if access.PasswordHash == nil || *access.PasswordHash != hashExternalSecret(strings.TrimSpace(password)) {
+ return nil, fmt.Errorf("invalid external access password")
+ }
+ default:
+ return nil, fmt.Errorf("unsupported external access mode")
+ }
+ if err := s.repo.MarkUsed(ctx, access.ID); err != nil {
+ return nil, err
+ }
+ return access, nil
+}
+
+func resolveExternalAccessExpiresAt(expiration ExternalAccessExpirationRequest) (*time.Time, error) {
+ mode := strings.TrimSpace(expiration.Mode)
+ if mode == "" {
+ mode = ExternalAccessExpirationPreset
+ }
+ switch mode {
+ case ExternalAccessExpirationPermanent:
+ return nil, nil
+ case ExternalAccessExpirationCustom:
+ if expiration.ExpiresAt == nil {
+ return nil, fmt.Errorf("custom external access expiration requires expires_at")
+ }
+ value := expiration.ExpiresAt.UTC()
+ return &value, nil
+ case ExternalAccessExpirationPreset:
+ preset := strings.TrimSpace(expiration.Preset)
+ if preset == "" {
+ preset = ExternalAccessPreset24Hours
+ }
+ duration, err := externalAccessPresetDuration(preset)
+ if err != nil {
+ return nil, err
+ }
+ value := time.Now().UTC().Add(duration)
+ return &value, nil
+ default:
+ return nil, fmt.Errorf("unsupported external access expiration mode")
+ }
+}
+
+func externalAccessPresetDuration(preset string) (time.Duration, error) {
+ switch strings.TrimSpace(preset) {
+ case ExternalAccessPreset1Hour:
+ return time.Hour, nil
+ case ExternalAccessPreset24Hours:
+ return 24 * time.Hour, nil
+ case ExternalAccessPreset7Days:
+ return 7 * 24 * time.Hour, nil
+ case ExternalAccessPreset30Days:
+ return 30 * 24 * time.Hour, nil
+ default:
+ return 0, fmt.Errorf("unsupported external access expiration preset")
+ }
+}
+
+func generateShortCode() (code, codeHash string, err error) {
+ code, err = randomToken("sl", 12)
+ if err != nil {
+ return "", "", err
+ }
+ return code, hashExternalSecret(code), nil
+}
+
+func shortExternalAccessURL(code string) string {
+ return fmt.Sprintf("/s/%s/", strings.Trim(strings.TrimSpace(code), "/"))
+}
+
+func ExternalAccessShareURL(access *models.InstanceExternalAccess) string {
+ if access == nil || !access.Enabled || access.PublicSlug == nil {
+ return ""
+ }
+ return shortExternalAccessURL(*access.PublicSlug)
+}
+
+func ExternalAccessPassword(access *models.InstanceExternalAccess) string {
+ if access == nil || !access.Enabled || access.AuthMode != ExternalAccessModePassword || access.PasswordValue == nil {
+ return ""
+ }
+ return *access.PasswordValue
+}
+
+func randomToken(prefix string, bytesLen int) (string, error) {
+ buf := make([]byte, bytesLen)
+ if _, err := rand.Read(buf); err != nil {
+ return "", fmt.Errorf("failed to generate token: %w", err)
+ }
+ return prefix + "_" + hex.EncodeToString(buf), nil
+}
+
+func hashExternalSecret(value string) string {
+ sum := sha256.Sum256([]byte(strings.TrimSpace(value)))
+ return hex.EncodeToString(sum[:])
+}
diff --git a/backend/internal/services/instance_external_access_service_test.go b/backend/internal/services/instance_external_access_service_test.go
new file mode 100644
index 0000000..5d9bb23
--- /dev/null
+++ b/backend/internal/services/instance_external_access_service_test.go
@@ -0,0 +1,304 @@
+package services
+
+import (
+ "context"
+ "strings"
+ "testing"
+ "time"
+
+ "clawreef/internal/models"
+)
+
+func TestInstanceExternalAccessShareLinkValidation(t *testing.T) {
+ repo := newFakeExternalAccessRepo()
+ service := NewInstanceExternalAccessService(repo)
+
+ result, err := service.EnableShareLink(context.Background(), 42, 7, ExternalAccessExpirationRequest{Mode: ExternalAccessExpirationPermanent})
+ if err != nil {
+ t.Fatalf("EnableShareLink failed: %v", err)
+ }
+ if !strings.HasPrefix(result.ShareURL, "/s/") || strings.Contains(result.ShareURL, "token=") || strings.Contains(result.ShareURL, "/api/v1/") || strings.Contains(result.ShareURL, "/proxy") {
+ t.Fatalf("unexpected short share URL: %q", result.ShareURL)
+ }
+ code := strings.Trim(strings.TrimPrefix(result.ShareURL, "/s/"), "/")
+ if code == "" {
+ t.Fatalf("short code missing from URL %q", result.ShareURL)
+ }
+ if result.Access.ShortCodeHash == nil || *result.Access.ShortCodeHash == code {
+ t.Fatalf("expected stored short code to be hashed")
+ }
+ if result.Access.AuthMode != ExternalAccessModeShareLink {
+ t.Fatalf("auth mode = %q, want %q", result.Access.AuthMode, ExternalAccessModeShareLink)
+ }
+
+ access, err := service.ValidateShortLink(context.Background(), code, "")
+ if err != nil {
+ t.Fatalf("ValidateShortLink failed: %v", err)
+ }
+ if access.InstanceID != 42 {
+ t.Fatalf("validated instance ID = %d, want 42", access.InstanceID)
+ }
+ if repo.markUsedCount != 1 {
+ t.Fatalf("mark used count = %d, want 1", repo.markUsedCount)
+ }
+
+ if _, err := service.ValidateShortLink(context.Background(), "wrong-code", ""); err == nil {
+ t.Fatalf("expected wrong short code to fail")
+ }
+}
+
+func TestInstanceExternalAccessShareLinkPresetExpiration(t *testing.T) {
+ repo := newFakeExternalAccessRepo()
+ service := NewInstanceExternalAccessService(repo)
+ before := time.Now().UTC().Add(24 * time.Hour)
+
+ result, err := service.EnableShareLink(context.Background(), 42, 7, ExternalAccessExpirationRequest{
+ Mode: ExternalAccessExpirationPreset,
+ Preset: ExternalAccessPreset24Hours,
+ })
+ if err != nil {
+ t.Fatalf("EnableShareLink failed: %v", err)
+ }
+ if result.Access.ExpiresAt == nil {
+ t.Fatal("preset expiration must set expires_at")
+ }
+ after := time.Now().UTC().Add(24 * time.Hour)
+ if result.Access.ExpiresAt.Before(before) || result.Access.ExpiresAt.After(after.Add(time.Second)) {
+ t.Fatalf("preset expires_at = %v, want around 24h from now", result.Access.ExpiresAt)
+ }
+}
+
+func TestInstanceExternalAccessShareLinkCustomAndPermanentExpiration(t *testing.T) {
+ repo := newFakeExternalAccessRepo()
+ service := NewInstanceExternalAccessService(repo)
+ customAt := time.Now().UTC().Add(3 * time.Hour).Truncate(time.Second)
+
+ custom, err := service.EnableShareLink(context.Background(), 42, 7, ExternalAccessExpirationRequest{
+ Mode: ExternalAccessExpirationCustom,
+ ExpiresAt: &customAt,
+ })
+ if err != nil {
+ t.Fatalf("custom EnableShareLink failed: %v", err)
+ }
+ if custom.Access.ExpiresAt == nil || !custom.Access.ExpiresAt.Equal(customAt) {
+ t.Fatalf("custom expires_at = %v, want %v", custom.Access.ExpiresAt, customAt)
+ }
+
+ permanent, err := service.EnableShareLink(context.Background(), 42, 7, ExternalAccessExpirationRequest{Mode: ExternalAccessExpirationPermanent})
+ if err != nil {
+ t.Fatalf("permanent EnableShareLink failed: %v", err)
+ }
+ if permanent.Access.ExpiresAt != nil {
+ t.Fatalf("permanent expires_at = %v, want nil", permanent.Access.ExpiresAt)
+ }
+}
+
+func TestInstanceExternalAccessPasswordDisable(t *testing.T) {
+ repo := newFakeExternalAccessRepo()
+ service := NewInstanceExternalAccessService(repo)
+
+ result, err := service.CreatePassword(context.Background(), 100, 9, ExternalAccessExpirationRequest{Mode: ExternalAccessExpirationPermanent})
+ if err != nil {
+ t.Fatalf("CreatePassword failed: %v", err)
+ }
+ if !strings.HasPrefix(result.Password, "pwd_") {
+ t.Fatalf("unexpected password prefix: %q", result.Password)
+ }
+ if result.Access.PasswordHash == nil || *result.Access.PasswordHash == result.Password {
+ t.Fatalf("expected stored password to be hashed")
+ }
+ if result.Access.PasswordHint == nil || !strings.HasPrefix(result.Password, *result.Access.PasswordHint) {
+ t.Fatalf("stored password hint does not match generated password")
+ }
+ if result.Access.AuthMode != ExternalAccessModePassword {
+ t.Fatalf("auth mode = %q, want %q", result.Access.AuthMode, ExternalAccessModePassword)
+ }
+ if !strings.HasPrefix(result.ShareURL, "/s/") || strings.Contains(result.ShareURL, "token=") || strings.Contains(result.ShareURL, "/api/v1/") {
+ t.Fatalf("unexpected password short URL: %q", result.ShareURL)
+ }
+ code := strings.Trim(strings.TrimPrefix(result.ShareURL, "/s/"), "/")
+
+ if _, err := service.ValidateShortLink(context.Background(), code, result.Password); err != nil {
+ t.Fatalf("ValidateShortLink with password failed: %v", err)
+ }
+
+ if err := service.Disable(context.Background(), 100); err != nil {
+ t.Fatalf("Disable failed: %v", err)
+ }
+ if _, err := service.ValidateShortLink(context.Background(), code, result.Password); err == nil {
+ t.Fatalf("expected disabled password access to fail")
+ }
+}
+
+func TestInstanceExternalAccessRestoresShareURLAfterRefresh(t *testing.T) {
+ repo := newFakeExternalAccessRepo()
+ service := NewInstanceExternalAccessService(repo)
+
+ result, err := service.CreatePassword(context.Background(), 100, 9, ExternalAccessExpirationRequest{Mode: ExternalAccessExpirationPermanent})
+ if err != nil {
+ t.Fatalf("CreatePassword failed: %v", err)
+ }
+ reloaded, err := service.Get(context.Background(), 100)
+ if err != nil {
+ t.Fatalf("Get failed: %v", err)
+ }
+ if got := ExternalAccessShareURL(reloaded); got != result.ShareURL {
+ t.Fatalf("ExternalAccessShareURL after reload = %q, want %q", got, result.ShareURL)
+ }
+}
+
+func TestInstanceExternalAccessRestoresPasswordAfterRefresh(t *testing.T) {
+ repo := newFakeExternalAccessRepo()
+ service := NewInstanceExternalAccessService(repo)
+
+ result, err := service.CreatePassword(context.Background(), 100, 9, ExternalAccessExpirationRequest{Mode: ExternalAccessExpirationPermanent})
+ if err != nil {
+ t.Fatalf("CreatePassword failed: %v", err)
+ }
+ reloaded, err := service.Get(context.Background(), 100)
+ if err != nil {
+ t.Fatalf("Get failed: %v", err)
+ }
+ if got := ExternalAccessPassword(reloaded); got != result.Password {
+ t.Fatalf("ExternalAccessPassword after reload = %q, want generated password", got)
+ }
+}
+
+func TestInstanceExternalAccessExpiration(t *testing.T) {
+ repo := newFakeExternalAccessRepo()
+ service := NewInstanceExternalAccessService(repo)
+ expiredAt := time.Now().UTC().Add(-time.Minute)
+
+ result, err := service.EnableShareLink(context.Background(), 77, 3, ExternalAccessExpirationRequest{
+ Mode: ExternalAccessExpirationCustom,
+ ExpiresAt: &expiredAt,
+ })
+ if err != nil {
+ t.Fatalf("EnableShareLink failed: %v", err)
+ }
+ code := strings.Trim(strings.TrimPrefix(result.ShareURL, "/s/"), "/")
+
+ if _, err := service.ValidateShortLink(context.Background(), code, ""); err == nil {
+ t.Fatalf("expected expired public access to fail")
+ }
+}
+
+type fakeExternalAccessRepo struct {
+ byInstance map[int]*models.InstanceExternalAccess
+ byCodeHash map[string]*models.InstanceExternalAccess
+ nextID int64
+ markUsedCount int
+}
+
+func newFakeExternalAccessRepo() *fakeExternalAccessRepo {
+ return &fakeExternalAccessRepo{
+ byInstance: make(map[int]*models.InstanceExternalAccess),
+ byCodeHash: make(map[string]*models.InstanceExternalAccess),
+ nextID: 1,
+ }
+}
+
+func (r *fakeExternalAccessRepo) GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceExternalAccess, error) {
+ _ = ctx
+ return cloneExternalAccess(r.byInstance[instanceID]), nil
+}
+
+func (r *fakeExternalAccessRepo) GetByShortCodeHash(ctx context.Context, codeHash string) (*models.InstanceExternalAccess, error) {
+ _ = ctx
+ return cloneExternalAccess(r.byCodeHash[codeHash]), nil
+}
+
+func (r *fakeExternalAccessRepo) Upsert(ctx context.Context, access *models.InstanceExternalAccess) error {
+ _ = ctx
+ saved := cloneExternalAccess(access)
+ if saved.ID == 0 {
+ if existing := r.byInstance[saved.InstanceID]; existing != nil {
+ saved.ID = existing.ID
+ } else {
+ saved.ID = r.nextID
+ r.nextID++
+ }
+ }
+ if existing := r.byInstance[saved.InstanceID]; existing != nil && existing.ShortCodeHash != nil {
+ delete(r.byCodeHash, *existing.ShortCodeHash)
+ }
+ r.byInstance[saved.InstanceID] = cloneExternalAccess(saved)
+ if saved.ShortCodeHash != nil {
+ r.byCodeHash[*saved.ShortCodeHash] = cloneExternalAccess(saved)
+ }
+ return nil
+}
+
+func (r *fakeExternalAccessRepo) Disable(ctx context.Context, instanceID int) error {
+ _ = ctx
+ if existing := r.byInstance[instanceID]; existing != nil {
+ existing.Enabled = false
+ if existing.ShortCodeHash != nil {
+ if byCode := r.byCodeHash[*existing.ShortCodeHash]; byCode != nil {
+ byCode.Enabled = false
+ }
+ }
+ }
+ return nil
+}
+
+func (r *fakeExternalAccessRepo) MarkUsed(ctx context.Context, id int64) error {
+ _ = ctx
+ r.markUsedCount++
+ now := time.Now().UTC()
+ for _, access := range r.byInstance {
+ if access.ID == id {
+ access.LastUsedAt = &now
+ }
+ }
+ for _, access := range r.byCodeHash {
+ if access.ID == id {
+ access.LastUsedAt = &now
+ }
+ }
+ return nil
+}
+
+func cloneExternalAccess(access *models.InstanceExternalAccess) *models.InstanceExternalAccess {
+ if access == nil {
+ return nil
+ }
+ clone := *access
+ if access.PublicSlug != nil {
+ value := *access.PublicSlug
+ clone.PublicSlug = &value
+ }
+ if access.PublicTokenHash != nil {
+ value := *access.PublicTokenHash
+ clone.PublicTokenHash = &value
+ }
+ if access.ShortCodeHash != nil {
+ value := *access.ShortCodeHash
+ clone.ShortCodeHash = &value
+ }
+ if access.PasswordHash != nil {
+ value := *access.PasswordHash
+ clone.PasswordHash = &value
+ }
+ if access.PasswordValue != nil {
+ value := *access.PasswordValue
+ clone.PasswordValue = &value
+ }
+ if access.PasswordHint != nil {
+ value := *access.PasswordHint
+ clone.PasswordHint = &value
+ }
+ if access.ExpiresAt != nil {
+ value := *access.ExpiresAt
+ clone.ExpiresAt = &value
+ }
+ if access.CreatedBy != nil {
+ value := *access.CreatedBy
+ clone.CreatedBy = &value
+ }
+ if access.LastUsedAt != nil {
+ value := *access.LastUsedAt
+ clone.LastUsedAt = &value
+ }
+ return &clone
+}
diff --git a/backend/internal/services/instance_proxy_service.go b/backend/internal/services/instance_proxy_service.go
index 8e3e05d..9143255 100644
--- a/backend/internal/services/instance_proxy_service.go
+++ b/backend/internal/services/instance_proxy_service.go
@@ -4,16 +4,20 @@ import (
"bytes"
"context"
"crypto/tls"
+ "errors"
"fmt"
"io"
+ "net"
"net/http"
"net/url"
+ "os"
"strconv"
"strings"
"sync"
"time"
"clawreef/internal/models"
+ "clawreef/internal/repository"
"clawreef/internal/services/k8s"
"github.com/gorilla/websocket"
@@ -21,14 +25,19 @@ import (
// InstanceProxyService handles proxying requests to instance pods
type InstanceProxyService struct {
- serviceService *k8s.ServiceService
- accessService *InstanceAccessService
- httpClient *http.Client
- serviceCache map[serviceCacheKey]serviceCacheEntry
- serviceLookups map[serviceCacheKey]*serviceLookupCall
- cacheMu sync.RWMutex
- lookupMu sync.Mutex
- serviceTTL time.Duration
+ serviceService *k8s.ServiceService
+ accessService *InstanceAccessService
+ instanceRepo repository.InstanceRepository
+ runtimePodRepo repository.RuntimePodRepository
+ bindingRepo repository.InstanceRuntimeBindingRepository
+ httpClient *http.Client
+ openClawGatewayToken string
+ openClawProxyOrigin string
+ serviceCache map[serviceCacheKey]serviceCacheEntry
+ serviceLookups map[serviceCacheKey]*serviceLookupCall
+ cacheMu sync.RWMutex
+ lookupMu sync.Mutex
+ serviceTTL time.Duration
}
type serviceCacheKey struct {
@@ -50,8 +59,20 @@ type serviceLookupCall struct {
const defaultServiceCacheTTL = 30 * time.Second
+var ErrInstanceGatewayUnavailable = errors.New("instance gateway is not available")
+
+type InstanceProxyServiceOption func(*InstanceProxyService)
+
+func WithInstanceProxyRuntimeRepositories(instanceRepo repository.InstanceRepository, runtimePodRepo repository.RuntimePodRepository, bindingRepo repository.InstanceRuntimeBindingRepository) InstanceProxyServiceOption {
+ return func(s *InstanceProxyService) {
+ s.instanceRepo = instanceRepo
+ s.runtimePodRepo = runtimePodRepo
+ s.bindingRepo = bindingRepo
+ }
+}
+
// NewInstanceProxyService creates a new instance proxy service
-func NewInstanceProxyService(accessService *InstanceAccessService) *InstanceProxyService {
+func NewInstanceProxyService(accessService *InstanceAccessService, options ...InstanceProxyServiceOption) *InstanceProxyService {
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
@@ -63,7 +84,7 @@ func NewInstanceProxyService(accessService *InstanceAccessService) *InstanceProx
ForceAttemptHTTP2: true,
}
- return &InstanceProxyService{
+ service := &InstanceProxyService{
serviceService: k8s.NewServiceService(),
accessService: accessService,
httpClient: &http.Client{
@@ -73,10 +94,18 @@ func NewInstanceProxyService(accessService *InstanceAccessService) *InstanceProx
return http.ErrUseLastResponse
},
},
- serviceCache: make(map[serviceCacheKey]serviceCacheEntry),
- serviceLookups: make(map[serviceCacheKey]*serviceLookupCall),
- serviceTTL: defaultServiceCacheTTL,
+ openClawGatewayToken: strings.TrimSpace(os.Getenv("OPENCLAW_GATEWAY_TOKEN")),
+ openClawProxyOrigin: resolveOpenClawProxyOriginFromEnv(),
+ serviceCache: make(map[serviceCacheKey]serviceCacheEntry),
+ serviceLookups: make(map[serviceCacheKey]*serviceLookupCall),
+ serviceTTL: defaultServiceCacheTTL,
}
+ for _, option := range options {
+ if option != nil {
+ option(service)
+ }
+ }
+ return service
}
// ProxyRequest proxies a request to an instance
@@ -102,27 +131,22 @@ func (s *InstanceProxyService) ProxyRequest(ctx context.Context, instanceID int,
return fmt.Errorf("token does not match instance")
}
- // Extract the actual path from the request (remove the proxy prefix)
- targetPath := s.extractTargetPath(r.URL.Path, instanceID, accessToken.InstanceType)
- targetPort := s.resolveTargetPort(accessToken.InstanceType, accessToken.TargetPort, targetPath)
- shouldRewriteHTML := s.shouldRewriteHTML(accessToken.InstanceType)
+ effectiveRequestPath := canonicalProxyEntryRequestPath(r.URL.Path, accessToken, instanceID)
- // Get service info for the instance (create if not exists)
- serviceInfo, err := s.getOrCreateService(ctx, accessToken.UserID, instanceID, targetPort)
- if err != nil {
- return fmt.Errorf("failed to get or create service: %w", err)
- }
+ // Extract the actual path from the request (remove the proxy prefix)
+ targetPath := s.extractTargetPath(effectiveRequestPath, instanceID, accessToken.InstanceType)
+ targetPort := s.resolveTargetPort(accessToken.InstanceType, accessToken.TargetPort, targetPath)
+ shouldRewriteHTML := s.shouldRewriteHTMLForProxy(instanceID, accessToken.InstanceType)
// Build target URL
- targetURL := &url.URL{
- Scheme: s.resolveTargetScheme(accessToken.InstanceType, false),
- Host: s.resolveProxyHost(ctx, accessToken.UserID, instanceID, serviceInfo),
- Path: targetPath,
+ targetURL, err := s.resolveHTTPProxyTarget(ctx, accessToken, instanceID, targetPort, targetPath, effectiveRequestPath)
+ if err != nil {
+ return err
}
// Copy query parameters (excluding token)
queryParams := r.URL.Query()
- queryParams.Del("token")
+ removeProxyAccessTokenQuery(queryParams, token)
if len(queryParams) > 0 {
targetURL.RawQuery = queryParams.Encode()
}
@@ -148,6 +172,9 @@ func (s *InstanceProxyService) ProxyRequest(ctx context.Context, instanceID int,
proxyReq.Header.Set("X-Forwarded-Host", r.Host)
proxyReq.Header.Set("X-Forwarded-Proto", requestScheme(r))
proxyReq.Header.Set("X-Forwarded-Prefix", fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID))
+ if token := s.managedRuntimeGatewayBearerToken(ctx, instanceID, accessToken.InstanceType); token != "" {
+ proxyReq.Header.Set("Authorization", "Bearer "+token)
+ }
if shouldRewriteHTML {
proxyReq.Header.Del("Accept-Encoding")
}
@@ -179,7 +206,7 @@ func (s *InstanceProxyService) ProxyRequest(ctx context.Context, instanceID int,
return fmt.Errorf("failed to close upstream html body: %w", closeErr)
}
- modifiedBody := injectProxyBase(string(body), fmt.Sprintf("/api/v1/instances/%d/proxy/", instanceID))
+ modifiedBody := injectProxyBase(string(body), proxyBaseForRequestPath(effectiveRequestPath, instanceID))
resp.Body = io.NopCloser(bytes.NewReader([]byte(modifiedBody)))
resp.ContentLength = int64(len(modifiedBody))
resp.Header.Set("Content-Length", strconv.Itoa(len(modifiedBody)))
@@ -238,22 +265,14 @@ func (s *InstanceProxyService) ProxyWebSocket(ctx context.Context, instanceID in
targetPath := s.extractTargetPath(r.URL.Path, instanceID, accessToken.InstanceType)
targetPort := s.resolveTargetPort(accessToken.InstanceType, accessToken.TargetPort, targetPath)
- // Get service info for the instance
- serviceInfo, err := s.getOrCreateService(ctx, accessToken.UserID, instanceID, targetPort)
+ targetURL, err := s.resolveWebSocketProxyTarget(ctx, accessToken, instanceID, targetPort, targetPath, r.URL.Path)
if err != nil {
- return fmt.Errorf("failed to get or create service: %w", err)
- }
-
- // WebSocket upstream uses ws/wss explicitly.
- targetURL := &url.URL{
- Scheme: s.resolveTargetScheme(accessToken.InstanceType, true),
- Host: s.resolveProxyHost(ctx, accessToken.UserID, instanceID, serviceInfo),
- Path: targetPath,
+ return err
}
// Copy query parameters (excluding token)
queryParams := r.URL.Query()
- queryParams.Del("token")
+ removeProxyAccessTokenQuery(queryParams, token)
if len(queryParams) > 0 {
targetURL.RawQuery = queryParams.Encode()
}
@@ -270,8 +289,14 @@ func (s *InstanceProxyService) ProxyWebSocket(ctx context.Context, instanceID in
upstreamHeader.Del("Sec-Websocket-Key")
upstreamHeader.Del("Sec-Websocket-Version")
upstreamHeader.Del("Sec-Websocket-Extensions")
+ upstreamHeader.Set("X-Forwarded-For", r.RemoteAddr)
+ upstreamHeader.Set("X-Forwarded-Host", r.Host)
upstreamHeader.Set("X-Forwarded-Proto", requestScheme(r))
upstreamHeader.Set("X-Forwarded-Prefix", fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID))
+ if token := s.managedRuntimeGatewayBearerToken(ctx, instanceID, accessToken.InstanceType); token != "" {
+ upstreamHeader.Set("Authorization", "Bearer "+token)
+ upstreamHeader.Set("Origin", s.openClawWebSocketOrigin(targetURL))
+ }
dialer := websocket.Dialer{
Proxy: http.ProxyFromEnvironment,
@@ -364,6 +389,112 @@ func (s *InstanceProxyService) removeHopByHopHeaders(header http.Header) {
}
}
+func (s *InstanceProxyService) managedRuntimeGatewayBearerToken(ctx context.Context, instanceID int, instanceType string) string {
+ if s == nil || s.instanceRepo == nil {
+ return ""
+ }
+ normalizedType, managedType := NormalizeV2RuntimeType(instanceType)
+ if !managedType {
+ return ""
+ }
+ instance, err := s.instanceRepo.GetByID(instanceID)
+ if err != nil || instance == nil {
+ return ""
+ }
+ if runtimeType, ok := v2RuntimeTypeForInstance(instance); ok && runtimeType == normalizedType {
+ if instance.AccessToken != nil && strings.TrimSpace(*instance.AccessToken) != "" {
+ return strings.TrimSpace(*instance.AccessToken)
+ }
+ }
+ if normalizedType == RuntimeTypeOpenClaw && s.openClawGatewayToken != "" {
+ return s.openClawGatewayToken
+ }
+ return ""
+}
+
+func (s *InstanceProxyService) resolveHTTPProxyTarget(ctx context.Context, accessToken *AccessToken, instanceID int, targetPort int32, targetPath, requestPath string) (*url.URL, error) {
+ if targetURL, ok, err := s.resolveV2ProxyTarget(ctx, accessToken, instanceID, targetPath, requestPath, false); ok || err != nil {
+ return targetURL, err
+ }
+ serviceInfo, err := s.getOrCreateService(ctx, accessToken.UserID, instanceID, targetPort)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get or create service: %w", err)
+ }
+ return &url.URL{
+ Scheme: s.resolveTargetScheme(accessToken.InstanceType, false),
+ Host: s.resolveProxyHost(ctx, accessToken.UserID, instanceID, serviceInfo),
+ Path: targetPath,
+ }, nil
+}
+
+func (s *InstanceProxyService) resolveWebSocketProxyTarget(ctx context.Context, accessToken *AccessToken, instanceID int, targetPort int32, targetPath, requestPath string) (*url.URL, error) {
+ if targetURL, ok, err := s.resolveV2ProxyTarget(ctx, accessToken, instanceID, targetPath, requestPath, true); ok || err != nil {
+ return targetURL, err
+ }
+ serviceInfo, err := s.getOrCreateService(ctx, accessToken.UserID, instanceID, targetPort)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get or create service: %w", err)
+ }
+ return &url.URL{
+ Scheme: s.resolveTargetScheme(accessToken.InstanceType, true),
+ Host: s.resolveProxyHost(ctx, accessToken.UserID, instanceID, serviceInfo),
+ Path: targetPath,
+ }, nil
+}
+
+func (s *InstanceProxyService) resolveV2ProxyTarget(ctx context.Context, accessToken *AccessToken, instanceID int, targetPath, requestPath string, websocket bool) (*url.URL, bool, error) {
+ if s.instanceRepo == nil || s.bindingRepo == nil || s.runtimePodRepo == nil {
+ return nil, false, nil
+ }
+ instance, err := s.instanceRepo.GetByID(instanceID)
+ if err != nil {
+ return nil, false, fmt.Errorf("failed to get instance for proxy: %w", err)
+ }
+ if instance == nil {
+ return nil, false, ErrInstanceGatewayUnavailable
+ }
+ if instance.UserID != accessToken.UserID {
+ return nil, false, fmt.Errorf("token does not match instance owner")
+ }
+ if _, ok := v2RuntimeTypeForInstance(instance); !ok {
+ return nil, false, nil
+ }
+ if !strings.EqualFold(strings.TrimSpace(instance.Status), "running") {
+ return nil, true, ErrInstanceGatewayUnavailable
+ }
+
+ binding, err := s.bindingRepo.GetRunningByInstanceID(ctx, instanceID)
+ if err != nil {
+ return nil, true, fmt.Errorf("%w: %v", ErrInstanceGatewayUnavailable, err)
+ }
+ if binding == nil {
+ return nil, true, ErrInstanceGatewayUnavailable
+ }
+ if binding.Generation != instance.RuntimeGeneration {
+ return nil, true, ErrInstanceGatewayUnavailable
+ }
+ pod, err := s.runtimePodRepo.GetByID(ctx, binding.RuntimePodID)
+ if err != nil {
+ return nil, true, fmt.Errorf("%w: %v", ErrInstanceGatewayUnavailable, err)
+ }
+ if pod == nil || pod.PodIP == nil || strings.TrimSpace(*pod.PodIP) == "" || binding.GatewayPort <= 0 {
+ return nil, true, ErrInstanceGatewayUnavailable
+ }
+ scheme := "http"
+ if websocket {
+ scheme = "ws"
+ }
+ upstreamPath := stripInstanceProxyPrefix(targetPath, instanceID)
+ if shouldPreserveOpenClawControlUIPath(instance) {
+ upstreamPath = openClawControlUIRequestPath(requestPath, instanceID)
+ }
+ return &url.URL{
+ Scheme: scheme,
+ Host: net.JoinHostPort(strings.TrimSpace(*pod.PodIP), strconv.Itoa(binding.GatewayPort)),
+ Path: upstreamPath,
+ }, true, nil
+}
+
// getOrCreateService gets service info or creates the service if it doesn't exist
func (s *InstanceProxyService) getOrCreateService(ctx context.Context, userID, instanceID int, targetPort int32) (*k8s.ServiceInfo, error) {
cacheKey := serviceCacheKey{
@@ -445,13 +576,78 @@ func (s *InstanceProxyService) extractTargetPath(requestPath string, instanceID
return requestPath
}
+func stripInstanceProxyPrefix(requestPath string, instanceID int) string {
+ prefix := fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID)
+ if strings.HasPrefix(requestPath, prefix) {
+ path := strings.TrimPrefix(requestPath, prefix)
+ if path == "" {
+ return "/"
+ }
+ return path
+ }
+ return requestPath
+}
+
+func canonicalProxyEntryRequestPath(requestPath string, accessToken *AccessToken, instanceID int) string {
+ prefix := fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID)
+ path := strings.TrimSpace(requestPath)
+ if path != prefix && path != prefix+"/" {
+ return requestPath
+ }
+ if accessToken == nil || strings.TrimSpace(accessToken.AccessURL) == "" {
+ return requestPath
+ }
+ parsed, err := url.Parse(accessToken.AccessURL)
+ if err != nil {
+ return requestPath
+ }
+ entryPath := strings.TrimSpace(parsed.Path)
+ if entryPath == "" || entryPath == prefix || entryPath == prefix+"/" {
+ return requestPath
+ }
+ if strings.HasPrefix(entryPath, prefix+"/") {
+ return entryPath
+ }
+ return requestPath
+}
+
+func shouldPreserveOpenClawControlUIPath(instance *models.Instance) bool {
+ if instance == nil || !strings.EqualFold(strings.TrimSpace(instance.Type), RuntimeTypeOpenClaw) {
+ return false
+ }
+ _, ok := v2RuntimeTypeForInstance(instance)
+ return ok
+}
+
+func openClawControlUIRequestPath(requestPath string, instanceID int) string {
+ prefix := fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID)
+ path := strings.TrimSpace(requestPath)
+ if path == "" || path == prefix {
+ return prefix + "/"
+ }
+ if strings.HasPrefix(path, prefix) {
+ return path
+ }
+ if strings.HasPrefix(path, "/") {
+ return prefix + path
+ }
+ return prefix + "/" + path
+}
+
// GetProxyURL generates a proxy URL for frontend
func (s *InstanceProxyService) GetProxyURL(instanceID int, token string) string {
- if token == "" {
- return fmt.Sprintf("/api/v1/instances/%d/proxy/", instanceID)
- }
+ return proxyURLWithPath(instanceID, "/", token)
+}
- return fmt.Sprintf("/api/v1/instances/%d/proxy/?token=%s", instanceID, token)
+// GetProxyURLForInstance generates the best frontend entry URL for an instance.
+func (s *InstanceProxyService) GetProxyURLForInstance(instance *models.Instance, token string) string {
+ if instance == nil {
+ return ""
+ }
+ if runtimeType, ok := v2RuntimeTypeForInstance(instance); ok && runtimeType == RuntimeTypeHermes {
+ return proxyURLWithPath(instance.ID, "/chat", token)
+ }
+ return proxyURLWithPath(instance.ID, "/", token)
}
// GetTargetPortForInstance returns the service target port used by the instance type.
@@ -526,6 +722,18 @@ func (s *InstanceProxyService) shouldRewriteHTML(instanceType string) bool {
return !usesWebtopImage(instanceType)
}
+func (s *InstanceProxyService) shouldRewriteHTMLForProxy(instanceID int, instanceType string) bool {
+ if s != nil && s.instanceRepo != nil && strings.EqualFold(strings.TrimSpace(instanceType), RuntimeTypeHermes) {
+ instance, err := s.instanceRepo.GetByID(instanceID)
+ if err == nil && instance != nil {
+ if runtimeType, ok := v2RuntimeTypeForInstance(instance); ok && runtimeType == RuntimeTypeHermes {
+ return true
+ }
+ }
+ }
+ return s.shouldRewriteHTML(instanceType)
+}
+
func (s *InstanceProxyService) getCachedService(key serviceCacheKey) *k8s.ServiceInfo {
s.cacheMu.RLock()
entry, ok := s.serviceCache[key]
@@ -593,6 +801,114 @@ func injectProxyBase(html, proxyBase string) string {
return baseTag + html
}
+func proxyURLWithPath(instanceID int, targetPath, token string) string {
+ path := strings.TrimSpace(targetPath)
+ if path == "" || path == "/" {
+ path = "/"
+ } else {
+ path = "/" + strings.TrimLeft(path, "/")
+ if !strings.HasSuffix(path, "/") {
+ path += "/"
+ }
+ }
+
+ raw := fmt.Sprintf("/api/v1/instances/%d/proxy%s", instanceID, path)
+ if token == "" {
+ return raw
+ }
+ return fmt.Sprintf("%s?token=%s", raw, url.QueryEscape(token))
+}
+
+func removeProxyAccessTokenQuery(query url.Values, accessToken string) {
+ values := query["token"]
+ if len(values) == 0 {
+ return
+ }
+ filtered := values[:0]
+ for _, value := range values {
+ if value != accessToken {
+ filtered = append(filtered, value)
+ }
+ }
+ if len(filtered) == 0 {
+ query.Del("token")
+ return
+ }
+ query["token"] = filtered
+}
+
+func proxyBaseForRequestPath(requestPath string, instanceID int) string {
+ prefix := fmt.Sprintf("/api/v1/instances/%d/proxy", instanceID)
+ path := strings.TrimSpace(requestPath)
+ if strings.HasPrefix(path, prefix) {
+ path = strings.TrimPrefix(path, prefix)
+ }
+ if path == "" || path == "/" {
+ return fmt.Sprintf("%s/", prefix)
+ }
+ if !strings.HasPrefix(path, "/") {
+ path = "/" + path
+ }
+ if !strings.HasSuffix(path, "/") {
+ lastSlash := strings.LastIndex(path, "/")
+ if lastSlash >= 0 {
+ path = path[:lastSlash+1]
+ } else {
+ path = "/"
+ }
+ }
+ return prefix + path
+}
+
+func websocketUpstreamOrigin(targetURL *url.URL) string {
+ if targetURL == nil {
+ return ""
+ }
+ scheme := targetURL.Scheme
+ switch scheme {
+ case "ws":
+ scheme = "http"
+ case "wss":
+ scheme = "https"
+ }
+ if scheme == "" || targetURL.Host == "" {
+ return ""
+ }
+ return scheme + "://" + targetURL.Host
+}
+
+func (s *InstanceProxyService) openClawWebSocketOrigin(targetURL *url.URL) string {
+ if s != nil && s.openClawProxyOrigin != "" {
+ return s.openClawProxyOrigin
+ }
+ return websocketUpstreamOrigin(targetURL)
+}
+
+func resolveOpenClawProxyOriginFromEnv() string {
+ for _, key := range []string{
+ "OPENCLAW_PROXY_ORIGIN",
+ "CLAWMANAGER_TEAM_MANAGER_BASE_URL",
+ "CLAWMANAGER_BACKEND_URL",
+ } {
+ if origin := originFromURLString(os.Getenv(key)); origin != "" {
+ return origin
+ }
+ }
+ return ""
+}
+
+func originFromURLString(rawURL string) string {
+ value := strings.TrimSpace(rawURL)
+ if value == "" {
+ return ""
+ }
+ parsed, err := url.Parse(value)
+ if err != nil || parsed.Scheme == "" || parsed.Host == "" {
+ return ""
+ }
+ return parsed.Scheme + "://" + parsed.Host
+}
+
func (s *InstanceProxyService) rewriteRedirectLocation(instanceID int, location string) string {
if strings.HasPrefix(location, "/") && !strings.HasPrefix(location, "/api/v1/instances/") {
return fmt.Sprintf("/api/v1/instances/%d/proxy%s", instanceID, location)
diff --git a/backend/internal/services/instance_proxy_service_v2_test.go b/backend/internal/services/instance_proxy_service_v2_test.go
new file mode 100644
index 0000000..d497eeb
--- /dev/null
+++ b/backend/internal/services/instance_proxy_service_v2_test.go
@@ -0,0 +1,692 @@
+package services
+
+import (
+ "errors"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+
+ "clawreef/internal/models"
+ "clawreef/internal/repository"
+
+ "github.com/gorilla/websocket"
+)
+
+func TestInstanceProxyServiceUsesRuntimeBindingForV2(t *testing.T) {
+ upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/api/v1/instances/123/proxy/apps/openclaw" {
+ t.Fatalf("unexpected upstream path %s", r.URL.Path)
+ }
+ if got := r.Header.Get("X-Forwarded-Prefix"); got != "/api/v1/instances/123/proxy" {
+ t.Fatalf("X-Forwarded-Prefix = %q", got)
+ }
+ if got := r.Header.Get("Authorization"); got != "Bearer internal-openclaw-token" {
+ t.Fatalf("Authorization = %q", got)
+ }
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte("ok"))
+ }))
+ defer upstream.Close()
+
+ podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL)
+ workspacePath := "/workspaces/openclaw/user-45/instance-123"
+ instanceRepo := newV2LifecycleInstanceRepo()
+ instanceRepo.byID[123] = &models.Instance{
+ ID: 123,
+ UserID: 45,
+ Type: "openclaw",
+ RuntimeType: "gateway",
+ Status: "running",
+ WorkspacePath: &workspacePath,
+ RuntimeGeneration: 3,
+ }
+ bindingRepo := newFakeRuntimeBindingRepo()
+ bindingRepo.bindings[123] = &models.InstanceRuntimeBinding{
+ InstanceID: 123,
+ RuntimePodID: 9,
+ GatewayPort: gatewayPort,
+ State: "running",
+ Generation: 3,
+ }
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 9: {ID: 9, PodIP: &podIP, State: "ready"},
+ },
+ }
+ accessService := NewInstanceAccessService()
+ defer accessService.Stop()
+ token, err := accessService.GenerateToken(45, 123, "openclaw", "/api/v1/instances/123/proxy/", 3000, time.Hour)
+ if err != nil {
+ t.Fatalf("GenerateToken returned error: %v", err)
+ }
+ service := NewInstanceProxyService(accessService)
+ service.instanceRepo = instanceRepo
+ service.bindingRepo = bindingRepo
+ service.runtimePodRepo = podRepo
+ service.httpClient = upstream.Client()
+ service.openClawGatewayToken = "internal-openclaw-token"
+
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/123/proxy/apps/openclaw?token="+url.QueryEscape(token.Token), nil)
+ rec := httptest.NewRecorder()
+
+ if err := service.ProxyRequest(req.Context(), 123, token.Token, rec, req); err != nil {
+ t.Fatalf("ProxyRequest returned error: %v", err)
+ }
+ if rec.Code != http.StatusOK || rec.Body.String() != "ok" {
+ t.Fatalf("unexpected proxy response %d %q", rec.Code, rec.Body.String())
+ }
+}
+
+func TestInstanceProxyServiceInjectsInstanceTokenForHermesLite(t *testing.T) {
+ instanceToken := "igt_hermes_instance"
+ upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/chat" {
+ t.Fatalf("unexpected upstream path %s", r.URL.Path)
+ }
+ if r.URL.RawQuery != "" {
+ t.Fatalf("unexpected upstream query %q", r.URL.RawQuery)
+ }
+ if got := r.Header.Get("Authorization"); got != "Bearer "+instanceToken {
+ t.Fatalf("Authorization = %q", got)
+ }
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte("ok"))
+ }))
+ defer upstream.Close()
+
+ podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL)
+ workspacePath := "/workspaces/hermes/user-45/instance-127"
+ instanceRepo := newV2LifecycleInstanceRepo()
+ instanceRepo.byID[127] = &models.Instance{
+ ID: 127,
+ UserID: 45,
+ Type: "hermes",
+ RuntimeType: "gateway",
+ InstanceMode: InstanceModeLite,
+ Status: "running",
+ AccessToken: &instanceToken,
+ WorkspacePath: &workspacePath,
+ RuntimeGeneration: 5,
+ }
+ bindingRepo := newFakeRuntimeBindingRepo()
+ bindingRepo.bindings[127] = &models.InstanceRuntimeBinding{
+ InstanceID: 127,
+ RuntimePodID: 10,
+ GatewayPort: gatewayPort,
+ State: "running",
+ Generation: 5,
+ }
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 10: {ID: 10, PodIP: &podIP, State: "ready"},
+ },
+ }
+ accessService := NewInstanceAccessService()
+ defer accessService.Stop()
+ token, err := accessService.GenerateToken(45, 127, "hermes", "/api/v1/instances/127/proxy/", 3000, time.Hour)
+ if err != nil {
+ t.Fatalf("GenerateToken returned error: %v", err)
+ }
+ service := NewInstanceProxyService(accessService)
+ service.instanceRepo = instanceRepo
+ service.bindingRepo = bindingRepo
+ service.runtimePodRepo = podRepo
+ service.httpClient = upstream.Client()
+
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/127/proxy/chat?token="+url.QueryEscape(token.Token), nil)
+ rec := httptest.NewRecorder()
+
+ if err := service.ProxyRequest(req.Context(), 127, token.Token, rec, req); err != nil {
+ t.Fatalf("ProxyRequest returned error: %v", err)
+ }
+ if rec.Code != http.StatusOK || rec.Body.String() != "ok" {
+ t.Fatalf("unexpected proxy response %d %q", rec.Code, rec.Body.String())
+ }
+}
+
+func TestInstanceProxyServiceUsesHermesLiteAccessURLForRootEntry(t *testing.T) {
+ instanceToken := "igt_hermes_instance"
+ upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/chat/" {
+ t.Fatalf("unexpected upstream path %s", r.URL.Path)
+ }
+ if got := r.Header.Get("Authorization"); got != "Bearer "+instanceToken {
+ t.Fatalf("Authorization = %q", got)
+ }
+ w.Header().Set("Content-Type", "text/html; charset=utf-8")
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte("Hermesok"))
+ }))
+ defer upstream.Close()
+
+ podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL)
+ workspacePath := "/workspaces/hermes/user-45/instance-131"
+ instanceRepo := newV2LifecycleInstanceRepo()
+ instanceRepo.byID[131] = &models.Instance{
+ ID: 131,
+ UserID: 45,
+ Type: "hermes",
+ RuntimeType: "gateway",
+ InstanceMode: InstanceModeLite,
+ Status: "running",
+ AccessToken: &instanceToken,
+ WorkspacePath: &workspacePath,
+ RuntimeGeneration: 5,
+ }
+ bindingRepo := newFakeRuntimeBindingRepo()
+ bindingRepo.bindings[131] = &models.InstanceRuntimeBinding{
+ InstanceID: 131,
+ RuntimePodID: 14,
+ GatewayPort: gatewayPort,
+ State: "running",
+ Generation: 5,
+ }
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 14: {ID: 14, PodIP: &podIP, State: "ready"},
+ },
+ }
+ accessService := NewInstanceAccessService()
+ defer accessService.Stop()
+ token, err := accessService.GenerateToken(45, 131, "hermes", "/api/v1/instances/131/proxy/chat/", 3000, time.Hour)
+ if err != nil {
+ t.Fatalf("GenerateToken returned error: %v", err)
+ }
+ service := NewInstanceProxyService(accessService)
+ service.instanceRepo = instanceRepo
+ service.bindingRepo = bindingRepo
+ service.runtimePodRepo = podRepo
+ service.httpClient = upstream.Client()
+
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/131/proxy/?token="+url.QueryEscape(token.Token), nil)
+ rec := httptest.NewRecorder()
+
+ if err := service.ProxyRequest(req.Context(), 131, token.Token, rec, req); err != nil {
+ t.Fatalf("ProxyRequest returned error: %v", err)
+ }
+ if rec.Code != http.StatusOK {
+ t.Fatalf("unexpected proxy response %d %q", rec.Code, rec.Body.String())
+ }
+ if !strings.Contains(rec.Body.String(), ``) {
+ t.Fatalf("expected Hermes Lite HTML to include chat proxy base, got %q", rec.Body.String())
+ }
+}
+
+func TestInstanceProxyServicePreservesHermesRuntimeQueryToken(t *testing.T) {
+ instanceToken := "igt_hermes_instance"
+ upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/api/ws" {
+ t.Fatalf("unexpected upstream path %s", r.URL.Path)
+ }
+ if got := r.URL.Query().Get("token"); got != "hermes-session-token" {
+ t.Fatalf("upstream token query = %q", got)
+ }
+ if got := r.Header.Get("Authorization"); got != "Bearer "+instanceToken {
+ t.Fatalf("Authorization = %q", got)
+ }
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte("ok"))
+ }))
+ defer upstream.Close()
+
+ podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL)
+ workspacePath := "/workspaces/hermes/user-45/instance-130"
+ instanceRepo := newV2LifecycleInstanceRepo()
+ instanceRepo.byID[130] = &models.Instance{
+ ID: 130,
+ UserID: 45,
+ Type: "hermes",
+ RuntimeType: "gateway",
+ InstanceMode: InstanceModeLite,
+ Status: "running",
+ AccessToken: &instanceToken,
+ WorkspacePath: &workspacePath,
+ RuntimeGeneration: 5,
+ }
+ bindingRepo := newFakeRuntimeBindingRepo()
+ bindingRepo.bindings[130] = &models.InstanceRuntimeBinding{
+ InstanceID: 130,
+ RuntimePodID: 13,
+ GatewayPort: gatewayPort,
+ State: "running",
+ Generation: 5,
+ }
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 13: {ID: 13, PodIP: &podIP, State: "ready"},
+ },
+ }
+ accessService := NewInstanceAccessService()
+ defer accessService.Stop()
+ token, err := accessService.GenerateToken(45, 130, "hermes", "/api/v1/instances/130/proxy/chat/", 3000, time.Hour)
+ if err != nil {
+ t.Fatalf("GenerateToken returned error: %v", err)
+ }
+ service := NewInstanceProxyService(accessService)
+ service.instanceRepo = instanceRepo
+ service.bindingRepo = bindingRepo
+ service.runtimePodRepo = podRepo
+ service.httpClient = upstream.Client()
+
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/130/proxy/api/ws?token=hermes-session-token", nil)
+ rec := httptest.NewRecorder()
+
+ if err := service.ProxyRequest(req.Context(), 130, token.Token, rec, req); err != nil {
+ t.Fatalf("ProxyRequest returned error: %v", err)
+ }
+ if rec.Code != http.StatusOK || rec.Body.String() != "ok" {
+ t.Fatalf("unexpected proxy response %d %q", rec.Code, rec.Body.String())
+ }
+}
+
+func TestInstanceProxyServiceRewritesHermesLiteHTMLBase(t *testing.T) {
+ instanceToken := "igt_hermes_instance"
+ upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/" {
+ t.Fatalf("unexpected upstream path %s", r.URL.Path)
+ }
+ if got := r.Header.Get("Authorization"); got != "Bearer "+instanceToken {
+ t.Fatalf("Authorization = %q", got)
+ }
+ w.Header().Set("Content-Type", "text/html; charset=utf-8")
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte("Hermesok"))
+ }))
+ defer upstream.Close()
+
+ podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL)
+ workspacePath := "/workspaces/hermes/user-45/instance-128"
+ instanceRepo := newV2LifecycleInstanceRepo()
+ instanceRepo.byID[128] = &models.Instance{
+ ID: 128,
+ UserID: 45,
+ Type: "hermes",
+ RuntimeType: "gateway",
+ InstanceMode: InstanceModeLite,
+ Status: "running",
+ AccessToken: &instanceToken,
+ WorkspacePath: &workspacePath,
+ RuntimeGeneration: 5,
+ }
+ bindingRepo := newFakeRuntimeBindingRepo()
+ bindingRepo.bindings[128] = &models.InstanceRuntimeBinding{
+ InstanceID: 128,
+ RuntimePodID: 11,
+ GatewayPort: gatewayPort,
+ State: "running",
+ Generation: 5,
+ }
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 11: {ID: 11, PodIP: &podIP, State: "ready"},
+ },
+ }
+ accessService := NewInstanceAccessService()
+ defer accessService.Stop()
+ token, err := accessService.GenerateToken(45, 128, "hermes", "/api/v1/instances/128/proxy/", 3000, time.Hour)
+ if err != nil {
+ t.Fatalf("GenerateToken returned error: %v", err)
+ }
+ service := NewInstanceProxyService(accessService)
+ service.instanceRepo = instanceRepo
+ service.bindingRepo = bindingRepo
+ service.runtimePodRepo = podRepo
+ service.httpClient = upstream.Client()
+
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/128/proxy/?token="+url.QueryEscape(token.Token), nil)
+ rec := httptest.NewRecorder()
+
+ if err := service.ProxyRequest(req.Context(), 128, token.Token, rec, req); err != nil {
+ t.Fatalf("ProxyRequest returned error: %v", err)
+ }
+ if rec.Code != http.StatusOK {
+ t.Fatalf("unexpected proxy response %d %q", rec.Code, rec.Body.String())
+ }
+ if !strings.Contains(rec.Body.String(), ``) {
+ t.Fatalf("expected Hermes Lite HTML to include proxy base, got %q", rec.Body.String())
+ }
+}
+
+func TestInstanceProxyServiceProxiesHermesLiteWebSocket(t *testing.T) {
+ instanceToken := "igt_hermes_instance"
+ upgrader := websocket.Upgrader{CheckOrigin: func(r *http.Request) bool { return true }}
+ upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/ws" {
+ t.Fatalf("unexpected upstream websocket path %s", r.URL.Path)
+ }
+ if got := r.URL.Query().Get("token"); got != "hermes-ws-token" {
+ t.Fatalf("upstream websocket token query = %q", got)
+ }
+ if got := r.Header.Get("Authorization"); got != "Bearer "+instanceToken {
+ t.Fatalf("Authorization = %q", got)
+ }
+ if got := r.Header.Get("X-Forwarded-Prefix"); got != "/api/v1/instances/129/proxy" {
+ t.Fatalf("X-Forwarded-Prefix = %q", got)
+ }
+ if got := r.Header.Get("X-Forwarded-Host"); got == "" {
+ t.Fatalf("X-Forwarded-Host is empty")
+ }
+ if got := r.Header.Get("X-Forwarded-For"); got == "" {
+ t.Fatalf("X-Forwarded-For is empty")
+ }
+ if got, want := r.Header.Get("Origin"), "http://"+r.Host; got != want {
+ t.Fatalf("Origin = %q, want %q", got, want)
+ }
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ t.Fatalf("upstream websocket upgrade failed: %v", err)
+ }
+ defer conn.Close()
+ messageType, message, err := conn.ReadMessage()
+ if err != nil {
+ t.Fatalf("upstream websocket read failed: %v", err)
+ }
+ if string(message) != "ping" {
+ t.Fatalf("upstream websocket message = %q", message)
+ }
+ if err := conn.WriteMessage(messageType, []byte("pong")); err != nil {
+ t.Fatalf("upstream websocket write failed: %v", err)
+ }
+ }))
+ defer upstream.Close()
+
+ podIP, gatewayPort := splitURLHostPortForProxyTest(t, upstream.URL)
+ workspacePath := "/workspaces/hermes/user-45/instance-129"
+ instanceRepo := newV2LifecycleInstanceRepo()
+ instanceRepo.byID[129] = &models.Instance{
+ ID: 129,
+ UserID: 45,
+ Type: "hermes",
+ RuntimeType: "gateway",
+ InstanceMode: InstanceModeLite,
+ Status: "running",
+ AccessToken: &instanceToken,
+ WorkspacePath: &workspacePath,
+ RuntimeGeneration: 5,
+ }
+ bindingRepo := newFakeRuntimeBindingRepo()
+ bindingRepo.bindings[129] = &models.InstanceRuntimeBinding{
+ InstanceID: 129,
+ RuntimePodID: 12,
+ GatewayPort: gatewayPort,
+ State: "running",
+ Generation: 5,
+ }
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 12: {ID: 12, PodIP: &podIP, State: "ready"},
+ },
+ }
+ accessService := NewInstanceAccessService()
+ defer accessService.Stop()
+ token, err := accessService.GenerateToken(45, 129, "hermes", "/api/v1/instances/129/proxy/", 3000, time.Hour)
+ if err != nil {
+ t.Fatalf("GenerateToken returned error: %v", err)
+ }
+ service := NewInstanceProxyService(accessService)
+ service.instanceRepo = instanceRepo
+ service.bindingRepo = bindingRepo
+ service.runtimePodRepo = podRepo
+
+ proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if err := service.ProxyWebSocket(r.Context(), 129, token.Token, w, r); err != nil {
+ http.Error(w, err.Error(), http.StatusBadGateway)
+ }
+ }))
+ defer proxy.Close()
+
+ wsURL := "ws" + strings.TrimPrefix(proxy.URL, "http") + "/api/v1/instances/129/proxy/ws?token=hermes-ws-token"
+ clientConn, _, err := websocket.DefaultDialer.Dial(wsURL, nil)
+ if err != nil {
+ t.Fatalf("client websocket dial failed: %v", err)
+ }
+ defer clientConn.Close()
+ if err := clientConn.WriteMessage(websocket.TextMessage, []byte("ping")); err != nil {
+ t.Fatalf("client websocket write failed: %v", err)
+ }
+ _, message, err := clientConn.ReadMessage()
+ if err != nil {
+ t.Fatalf("client websocket read failed: %v", err)
+ }
+ if string(message) != "pong" {
+ t.Fatalf("client websocket message = %q", message)
+ }
+}
+
+func TestInstanceProxyServiceUsesBaseProxyEntryForV2OpenClaw(t *testing.T) {
+ workspacePath := "/workspaces/openclaw/user-45/instance-123"
+ accessService := NewInstanceAccessService()
+ t.Cleanup(accessService.Stop)
+ service := NewInstanceProxyService(accessService)
+ got := service.GetProxyURLForInstance(&models.Instance{
+ ID: 123,
+ Type: "openclaw",
+ RuntimeType: "gateway",
+ WorkspacePath: &workspacePath,
+ }, "token+with/slash")
+
+ want := "/api/v1/instances/123/proxy/?token=token%2Bwith%2Fslash"
+ if got != want {
+ t.Fatalf("GetProxyURLForInstance() = %q, want %q", got, want)
+ }
+}
+
+func TestInstanceProxyServiceUsesChatEntryForHermesLite(t *testing.T) {
+ workspacePath := "/workspaces/hermes/user-45/instance-123"
+ accessService := NewInstanceAccessService()
+ t.Cleanup(accessService.Stop)
+ service := NewInstanceProxyService(accessService)
+ got := service.GetProxyURLForInstance(&models.Instance{
+ ID: 123,
+ Type: "hermes",
+ RuntimeType: "gateway",
+ InstanceMode: InstanceModeLite,
+ WorkspacePath: &workspacePath,
+ }, "token+with/slash")
+
+ want := "/api/v1/instances/123/proxy/chat/?token=token%2Bwith%2Fslash"
+ if got != want {
+ t.Fatalf("GetProxyURLForInstance() = %q, want %q", got, want)
+ }
+}
+
+func TestProxyBaseForRequestPathUsesSubdirectory(t *testing.T) {
+ got := proxyBaseForRequestPath("/api/v1/instances/123/proxy/__openclaw__/canvas/", 123)
+ want := "/api/v1/instances/123/proxy/__openclaw__/canvas/"
+ if got != want {
+ t.Fatalf("proxyBaseForRequestPath() = %q, want %q", got, want)
+ }
+
+ got = proxyBaseForRequestPath("/api/v1/instances/123/proxy/__openclaw__/canvas/app.js", 123)
+ if got != want {
+ t.Fatalf("proxyBaseForRequestPath(file) = %q, want %q", got, want)
+ }
+}
+
+func TestResolveOpenClawProxyOriginFromEnvUsesInternalOrigin(t *testing.T) {
+ t.Setenv("OPENCLAW_PROXY_ORIGIN", "")
+ t.Setenv("CLAWMANAGER_BACKEND_URL", "")
+ t.Setenv("CLAWMANAGER_TEAM_MANAGER_BASE_URL", "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001/api")
+
+ got := resolveOpenClawProxyOriginFromEnv()
+ want := "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001"
+ if got != want {
+ t.Fatalf("resolveOpenClawProxyOriginFromEnv() = %q, want %q", got, want)
+ }
+}
+
+func TestOpenClawWebSocketOriginPrefersConfiguredProxyOrigin(t *testing.T) {
+ accessService := NewInstanceAccessService()
+ t.Cleanup(accessService.Stop)
+ service := NewInstanceProxyService(accessService)
+ service.openClawProxyOrigin = "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001"
+
+ got := service.openClawWebSocketOrigin(&url.URL{Scheme: "ws", Host: "10.42.0.63:20000"})
+ want := "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001"
+ if got != want {
+ t.Fatalf("openClawWebSocketOrigin() = %q, want %q", got, want)
+ }
+}
+
+func TestWebSocketUpstreamOriginFallsBackToGatewayHost(t *testing.T) {
+ tests := []struct {
+ name string
+ url *url.URL
+ want string
+ }{
+ {
+ name: "plain websocket",
+ url: &url.URL{Scheme: "ws", Host: "10.42.0.63:20000"},
+ want: "http://10.42.0.63:20000",
+ },
+ {
+ name: "tls websocket",
+ url: &url.URL{Scheme: "wss", Host: "gateway.example.test"},
+ want: "https://gateway.example.test",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := websocketUpstreamOrigin(tt.url); got != tt.want {
+ t.Fatalf("websocketUpstreamOrigin() = %q, want %q", got, tt.want)
+ }
+ })
+ }
+}
+
+func TestInstanceProxyServiceRejectsStoppedV2InstanceWithStaleBinding(t *testing.T) {
+ workspacePath := "/workspaces/openclaw/user-45/instance-125"
+ instanceRepo := newV2LifecycleInstanceRepo()
+ instanceRepo.byID[125] = &models.Instance{
+ ID: 125,
+ UserID: 45,
+ Type: "openclaw",
+ RuntimeType: "gateway",
+ Status: "stopped",
+ WorkspacePath: &workspacePath,
+ RuntimeGeneration: 4,
+ }
+ bindingRepo := newFakeRuntimeBindingRepo()
+ bindingRepo.bindings[125] = &models.InstanceRuntimeBinding{
+ InstanceID: 125,
+ RuntimePodID: 9,
+ GatewayPort: 20025,
+ State: "running",
+ Generation: 4,
+ }
+ podIP := "10.42.0.125"
+ service, token := newV2ProxyTestService(t, instanceRepo, bindingRepo, &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 9: {ID: 9, PodIP: &podIP},
+ },
+ }, 45, 125, "openclaw")
+
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/125/proxy/", nil)
+ err := service.ProxyRequest(req.Context(), 125, token.Token, httptest.NewRecorder(), req)
+ if !errors.Is(err, ErrInstanceGatewayUnavailable) {
+ t.Fatalf("ProxyRequest error = %v, want ErrInstanceGatewayUnavailable", err)
+ }
+}
+
+func TestInstanceProxyServiceRejectsV2BindingGenerationMismatch(t *testing.T) {
+ workspacePath := "/workspaces/hermes/user-45/instance-126"
+ instanceRepo := newV2LifecycleInstanceRepo()
+ instanceRepo.byID[126] = &models.Instance{
+ ID: 126,
+ UserID: 45,
+ Type: "hermes",
+ RuntimeType: "gateway",
+ Status: "running",
+ WorkspacePath: &workspacePath,
+ RuntimeGeneration: 8,
+ }
+ bindingRepo := newFakeRuntimeBindingRepo()
+ bindingRepo.bindings[126] = &models.InstanceRuntimeBinding{
+ InstanceID: 126,
+ RuntimePodID: 10,
+ GatewayPort: 20026,
+ State: "running",
+ Generation: 7,
+ }
+ podIP := "10.42.0.126"
+ service, token := newV2ProxyTestService(t, instanceRepo, bindingRepo, &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 10: {ID: 10, PodIP: &podIP},
+ },
+ }, 45, 126, "hermes")
+
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/126/proxy/", nil)
+ err := service.ProxyRequest(req.Context(), 126, token.Token, httptest.NewRecorder(), req)
+ if !errors.Is(err, ErrInstanceGatewayUnavailable) {
+ t.Fatalf("ProxyRequest error = %v, want ErrInstanceGatewayUnavailable", err)
+ }
+}
+
+func TestInstanceProxyServiceReturnsUnavailableWhenV2BindingMissing(t *testing.T) {
+ workspacePath := "/workspaces/hermes/user-45/instance-124"
+ instanceRepo := newV2LifecycleInstanceRepo()
+ instanceRepo.byID[124] = &models.Instance{
+ ID: 124,
+ UserID: 45,
+ Type: "hermes",
+ RuntimeType: "gateway",
+ Status: "running",
+ WorkspacePath: &workspacePath,
+ }
+ accessService := NewInstanceAccessService()
+ defer accessService.Stop()
+ token, err := accessService.GenerateToken(45, 124, "hermes", "/api/v1/instances/124/proxy/", 3000, time.Hour)
+ if err != nil {
+ t.Fatalf("GenerateToken returned error: %v", err)
+ }
+ service := NewInstanceProxyService(accessService)
+ service.instanceRepo = instanceRepo
+ service.bindingRepo = newFakeRuntimeBindingRepo()
+ service.runtimePodRepo = &fakeRuntimePodRepo{}
+
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/124/proxy/", nil)
+ rec := httptest.NewRecorder()
+
+ err = service.ProxyRequest(req.Context(), 124, token.Token, rec, req)
+ if !errors.Is(err, ErrInstanceGatewayUnavailable) {
+ t.Fatalf("ProxyRequest error = %v, want ErrInstanceGatewayUnavailable", err)
+ }
+}
+
+func newV2ProxyTestService(t *testing.T, instanceRepo repository.InstanceRepository, bindingRepo repository.InstanceRuntimeBindingRepository, podRepo repository.RuntimePodRepository, userID int, instanceID int, instanceType string) (*InstanceProxyService, *AccessToken) {
+ t.Helper()
+ accessService := NewInstanceAccessService()
+ t.Cleanup(accessService.Stop)
+ token, err := accessService.GenerateToken(userID, instanceID, instanceType, "/api/v1/instances/"+strconv.Itoa(instanceID)+"/proxy/", 3000, time.Hour)
+ if err != nil {
+ t.Fatalf("GenerateToken returned error: %v", err)
+ }
+ service := NewInstanceProxyService(accessService)
+ service.instanceRepo = instanceRepo
+ service.bindingRepo = bindingRepo
+ service.runtimePodRepo = podRepo
+ return service, token
+}
+
+func splitURLHostPortForProxyTest(t *testing.T, rawURL string) (string, int) {
+ t.Helper()
+ parsed, err := url.Parse(rawURL)
+ if err != nil {
+ t.Fatalf("parse upstream URL: %v", err)
+ }
+ host, portText, err := net.SplitHostPort(parsed.Host)
+ if err != nil {
+ t.Fatalf("split upstream host port: %v", err)
+ }
+ port, err := strconv.Atoi(portText)
+ if err != nil {
+ t.Fatalf("parse upstream port: %v", err)
+ }
+ return host, port
+}
diff --git a/backend/internal/services/instance_service.go b/backend/internal/services/instance_service.go
index db53c9d..0729a9b 100644
--- a/backend/internal/services/instance_service.go
+++ b/backend/internal/services/instance_service.go
@@ -6,6 +6,8 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
+ "os"
+ "path"
"strconv"
"strings"
"time"
@@ -79,11 +81,13 @@ func (s *instanceService) ValidateCreateRequests(userID int, requests []CreateIn
currentGPU := 0
existingNames := map[string]struct{}{}
for _, existing := range existingInstances {
- currentCPU += existing.CPUCores
- currentMemory += existing.MemoryGB
- currentStorage += existing.DiskGB
- if existing.GPUEnabled {
- currentGPU += existing.GPUCount
+ if instanceModeUsesDedicatedResources(modeForExistingInstance(&existing)) {
+ currentCPU += existing.CPUCores
+ currentMemory += existing.MemoryGB
+ currentStorage += existing.DiskGB
+ if existing.GPUEnabled {
+ currentGPU += existing.GPUCount
+ }
}
existingNames[strings.TrimSpace(strings.ToLower(existing.Name))] = struct{}{}
}
@@ -102,11 +106,13 @@ func (s *instanceService) ValidateCreateRequests(userID int, requests []CreateIn
return fmt.Errorf("instance name already exists")
}
requestNames[normalizedName] = struct{}{}
- requestedCPU += req.CPUCores
- requestedMemory += req.MemoryGB
- requestedStorage += req.DiskGB
- if req.GPUEnabled {
- requestedGPU += req.GPUCount
+ if instanceModeUsesDedicatedResources(resolveCreateInstanceMode(req)) {
+ requestedCPU += req.CPUCores
+ requestedMemory += req.MemoryGB
+ requestedStorage += req.DiskGB
+ if req.GPUEnabled {
+ requestedGPU += req.GPUCount
+ }
}
}
@@ -130,8 +136,10 @@ func (s *instanceService) ValidateCreateRequests(userID int, requests []CreateIn
type CreateInstanceRequest struct {
Name string `json:"name" validate:"required,min=3,max=50"`
Description *string `json:"description,omitempty"`
- Type string `json:"type" validate:"required,oneof=openclaw ubuntu debian centos custom webtop hermes"`
- RuntimeType string `json:"runtime_type" validate:"omitempty,oneof=desktop shell"`
+ Type string `json:"type" validate:"required,oneof=openclaw hermes"`
+ Mode string `json:"mode" validate:"omitempty,oneof=lite pro"`
+ InstanceMode string `json:"instance_mode" validate:"omitempty,oneof=lite pro"`
+ RuntimeType string `json:"runtime_type" validate:"omitempty,oneof=gateway desktop shell"`
CPUCores float64 `json:"cpu_cores" validate:"required,min=0.1,max=32"`
MemoryGB int `json:"memory_gb" validate:"required,min=1,max=128"`
DiskGB int `json:"disk_gb" validate:"required,min=10,max=1000"`
@@ -159,6 +167,14 @@ type TeamInstanceConfig struct {
SharedUmask string
}
+type instanceModeLimitConfig struct {
+ Capacity *int
+ MaxCPU *float64
+ MaxMemoryGB *int
+ MaxStorageGB *int
+ MaxGPUCount *int
+}
+
// UpdateInstanceRequest holds data for updating an instance
type UpdateInstanceRequest struct {
Name *string `json:"name,omitempty" validate:"omitempty,min=3,max=50"`
@@ -167,14 +183,17 @@ type UpdateInstanceRequest struct {
// InstanceStatus holds the status of an instance
type InstanceStatus struct {
- InstanceID int `json:"instance_id"`
- Status string `json:"status"`
- PodName *string `json:"pod_name,omitempty"`
- PodNamespace *string `json:"pod_namespace,omitempty"`
- PodIP *string `json:"pod_ip,omitempty"`
- PodStatus string `json:"pod_status,omitempty"`
- CreatedAt time.Time `json:"created_at"`
- StartedAt *time.Time `json:"started_at,omitempty"`
+ InstanceID int `json:"instance_id"`
+ Status string `json:"status"`
+ Availability string `json:"availability,omitempty"`
+ AgentType string `json:"agent_type,omitempty"`
+ WorkspaceUsageBytes int64 `json:"workspace_usage_bytes,omitempty"`
+ PodName *string `json:"pod_name,omitempty"`
+ PodNamespace *string `json:"pod_namespace,omitempty"`
+ PodIP *string `json:"pod_ip,omitempty"`
+ PodStatus string `json:"pod_status,omitempty"`
+ CreatedAt time.Time `json:"created_at"`
+ StartedAt *time.Time `json:"started_at,omitempty"`
}
// instanceService implements InstanceService
@@ -184,7 +203,12 @@ type instanceService struct {
llmModelRepo repository.LLMModelRepository
openClawConfigService OpenClawConfigService
allowPrivilegedPods bool
+ runtimePodRepo repository.RuntimePodRepository
+ bindingRepo repository.InstanceRuntimeBindingRepository
+ agentClient RuntimeAgentClient
+ workspaceRoot string
podService *k8s.PodService
+ deploymentService *k8s.InstanceDeploymentService
pvcService *k8s.PVCService
serviceService *k8s.ServiceService
networkPolicyService *k8s.NetworkPolicyService
@@ -203,6 +227,17 @@ func WithPrivilegedInstancePods(allowed bool) InstanceServiceOption {
}
}
+func WithV2RuntimeLifecycle(runtimePodRepo repository.RuntimePodRepository, bindingRepo repository.InstanceRuntimeBindingRepository, agentClient RuntimeAgentClient, workspaceRoot string) InstanceServiceOption {
+ return func(s *instanceService) {
+ s.runtimePodRepo = runtimePodRepo
+ s.bindingRepo = bindingRepo
+ s.agentClient = agentClient
+ if strings.TrimSpace(workspaceRoot) != "" {
+ s.workspaceRoot = strings.TrimSpace(workspaceRoot)
+ }
+ }
+}
+
// NewInstanceService creates a new instance service
func NewInstanceService(instanceRepo repository.InstanceRepository, quotaRepo repository.QuotaRepository, llmModelRepo repository.LLMModelRepository, openClawConfigService OpenClawConfigService, options ...InstanceServiceOption) InstanceService {
service := &instanceService{
@@ -210,7 +245,9 @@ func NewInstanceService(instanceRepo repository.InstanceRepository, quotaRepo re
quotaRepo: quotaRepo,
llmModelRepo: llmModelRepo,
openClawConfigService: openClawConfigService,
+ workspaceRoot: "/workspaces",
podService: k8s.NewPodService(),
+ deploymentService: k8s.NewInstanceDeploymentService(),
pvcService: k8s.NewPVCService(),
serviceService: k8s.NewServiceService(),
networkPolicyService: k8s.NewNetworkPolicyService(),
@@ -227,6 +264,7 @@ func NewInstanceService(instanceRepo repository.InstanceRepository, quotaRepo re
func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models.Instance, error) {
ctx := context.Background()
req.Name = strings.TrimSpace(req.Name)
+ req.Type = strings.ToLower(strings.TrimSpace(req.Type))
environmentOverrides, err := normalizeEnvironmentOverrides(req.EnvironmentOverrides)
if err != nil {
return nil, err
@@ -245,6 +283,11 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models
if quota == nil {
return nil, fmt.Errorf("user quota not found")
}
+ instanceMode := resolveCreateInstanceMode(req)
+ modeRuntimeType, _ := RuntimeTypeForInstanceMode(instanceMode)
+ if !hasExplicitCreateInstanceMode(req) && normalizeInstanceRuntimeType(req.RuntimeType) == RuntimeBackendShell {
+ modeRuntimeType = RuntimeBackendShell
+ }
// Check instance count limit
currentCount, err := s.instanceRepo.CountByUserID(userID)
@@ -266,11 +309,13 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models
currentStorage := 0
currentGPU := 0
for _, existing := range existingInstances {
- currentCPU += existing.CPUCores
- currentMemory += existing.MemoryGB
- currentStorage += existing.DiskGB
- if existing.GPUEnabled {
- currentGPU += existing.GPUCount
+ if instanceModeUsesDedicatedResources(modeForExistingInstance(&existing)) {
+ currentCPU += existing.CPUCores
+ currentMemory += existing.MemoryGB
+ currentStorage += existing.DiskGB
+ if existing.GPUEnabled {
+ currentGPU += existing.GPUCount
+ }
}
}
@@ -282,43 +327,58 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models
return nil, fmt.Errorf("instance name already exists")
}
- // Check CPU limit
- if currentCPU+req.CPUCores > quota.MaxCPUCores {
- return nil, fmt.Errorf("CPU cores exceed quota: current %v, requested %v, max %v", currentCPU, req.CPUCores, quota.MaxCPUCores)
- }
-
- // Check memory limit
- if currentMemory+req.MemoryGB > quota.MaxMemoryGB {
- return nil, fmt.Errorf("memory exceed quota: current %dGB, requested %dGB, max %dGB", currentMemory, req.MemoryGB, quota.MaxMemoryGB)
- }
-
- // Check storage limit
- if currentStorage+req.DiskGB > quota.MaxStorageGB {
- return nil, fmt.Errorf("storage exceed quota: current %dGB, requested %dGB, max %dGB", currentStorage, req.DiskGB, quota.MaxStorageGB)
- }
-
- // Check GPU limit
requestedGPU := 0
if req.GPUEnabled {
requestedGPU = req.GPUCount
}
- if currentGPU+requestedGPU > quota.MaxGPUCount {
- return nil, fmt.Errorf("GPU count exceed quota: current %d, requested %d, max %d", currentGPU, requestedGPU, quota.MaxGPUCount)
+ if instanceModeUsesDedicatedResources(instanceMode) {
+ // Check CPU limit
+ if currentCPU+req.CPUCores > quota.MaxCPUCores {
+ return nil, fmt.Errorf("CPU cores exceed quota: current %v, requested %v, max %v", currentCPU, req.CPUCores, quota.MaxCPUCores)
+ }
+
+ // Check memory limit
+ if currentMemory+req.MemoryGB > quota.MaxMemoryGB {
+ return nil, fmt.Errorf("memory exceed quota: current %dGB, requested %dGB, max %dGB", currentMemory, req.MemoryGB, quota.MaxMemoryGB)
+ }
+
+ // Check storage limit
+ if currentStorage+req.DiskGB > quota.MaxStorageGB {
+ return nil, fmt.Errorf("storage exceed quota: current %dGB, requested %dGB, max %dGB", currentStorage, req.DiskGB, quota.MaxStorageGB)
+ }
+
+ // Check GPU limit
+ if currentGPU+requestedGPU > quota.MaxGPUCount {
+ return nil, fmt.Errorf("GPU count exceed quota: current %d, requested %d, max %d", currentGPU, requestedGPU, quota.MaxGPUCount)
+ }
+ }
+ if err := s.enforceInstanceModeLimits(ctx, instanceMode, req.CPUCores, req.MemoryGB, req.DiskGB, requestedGPU); err != nil {
+ return nil, err
+ }
+ if runtimeType, isV2 := NormalizeV2RuntimeType(req.Type); isV2 && instanceMode == InstanceModeLite {
+ return s.createV2Instance(ctx, userID, req, runtimeType, environmentOverridesJSON)
}
runtimeConfig := buildRuntimeConfig(req.Type, req.OSType, req.OSVersion, req.ImageRegistry, req.ImageTag)
runtimeType := normalizeInstanceRuntimeType(req.RuntimeType)
+ if modeRuntimeType != "" {
+ runtimeType = modeRuntimeType
+ }
if (req.ImageRegistry == nil || strings.TrimSpace(*req.ImageRegistry) == "") && (req.ImageTag == nil || strings.TrimSpace(*req.ImageTag) == "") {
if selection, ok := runtimeImageOverride(req.Type); ok {
image := selection.Image
req.ImageRegistry = &image
req.ImageTag = nil
- runtimeType = normalizeInstanceRuntimeType(selection.RuntimeType)
+ if modeRuntimeType == "" {
+ runtimeType = normalizeInstanceRuntimeType(selection.RuntimeType)
+ }
runtimeConfig = buildRuntimeConfig(req.Type, req.OSType, req.OSVersion, req.ImageRegistry, req.ImageTag)
}
} else if req.ImageRegistry != nil {
if selection, ok := runtimeImageOverrideForImage(req.Type, *req.ImageRegistry); ok {
- runtimeType = normalizeInstanceRuntimeType(selection.RuntimeType)
+ if modeRuntimeType == "" {
+ runtimeType = normalizeInstanceRuntimeType(selection.RuntimeType)
+ }
}
}
@@ -334,6 +394,7 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models
Description: req.Description,
Type: req.Type,
RuntimeType: runtimeType,
+ InstanceMode: InstanceModeForRuntimeType(runtimeType),
Status: "creating",
CPUCores: req.CPUCores,
MemoryGB: req.MemoryGB,
@@ -506,18 +567,29 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models
SecurityMode: s.securityModeForInstance(instance.Type),
}
- pod, err := s.podService.CreatePod(ctx, podConfig)
- if err != nil {
- // Rollback: delete PVC and instance record
- s.pvcService.DeletePVC(ctx, userID, instance.ID)
- if bootstrapSnapshot != nil {
- _ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err)
- }
- s.instanceRepo.Delete(instance.ID)
- return nil, fmt.Errorf("failed to create pod: %w", err)
- }
-
+ var workloadNamespace string
+ var workloadName string
if instanceUsesDesktopRuntime(instance) {
+ if s.deploymentService == nil {
+ s.pvcService.DeletePVC(ctx, userID, instance.ID)
+ if bootstrapSnapshot != nil {
+ _ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, fmt.Errorf("instance deployment service is not configured"))
+ }
+ s.instanceRepo.Delete(instance.ID)
+ return nil, fmt.Errorf("instance deployment service is not configured")
+ }
+ deployment, err := s.deploymentService.EnsureDeployment(ctx, podConfig, 1)
+ if err != nil {
+ s.pvcService.DeletePVC(ctx, userID, instance.ID)
+ if bootstrapSnapshot != nil {
+ _ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err)
+ }
+ s.instanceRepo.Delete(instance.ID)
+ return nil, fmt.Errorf("failed to create deployment: %w", err)
+ }
+ workloadNamespace = deployment.Namespace
+ workloadName = deployment.Name
+
// Create Service for browser desktop access.
serviceConfig := k8s.ServiceConfig{
InstanceID: instance.ID,
@@ -529,8 +601,8 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models
serviceInfo, err := s.serviceService.CreateService(ctx, serviceConfig)
if err != nil {
- // Rollback: delete pod, PVC and instance record
- s.podService.DeletePod(ctx, userID, instance.ID)
+ // Rollback: delete Deployment, PVC and instance record.
+ _ = s.deploymentService.DeleteDeployment(ctx, userID, instance.ID)
s.pvcService.DeletePVC(ctx, userID, instance.ID)
if bootstrapSnapshot != nil {
_ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err)
@@ -541,12 +613,25 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models
fmt.Printf("Instance %d: Service created successfully (ClusterIP: %s)\n", instance.ID, serviceInfo.ClusterIP)
} else {
+ pod, err := s.podService.CreatePod(ctx, podConfig)
+ if err != nil {
+ // Rollback: delete PVC and instance record.
+ s.pvcService.DeletePVC(ctx, userID, instance.ID)
+ if bootstrapSnapshot != nil {
+ _ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err)
+ }
+ s.instanceRepo.Delete(instance.ID)
+ return nil, fmt.Errorf("failed to create pod: %w", err)
+ }
+ workloadNamespace = pod.Namespace
+ workloadName = pod.Name
fmt.Printf("Instance %d: Shell runtime selected, skipping desktop service creation\n", instance.ID)
}
- // Update instance with pod info
- podNamespace := pod.Namespace
- podName := pod.Name
+ // Update instance with initial workload info. For Pro instances this is the
+ // stable Deployment name; sync later records the active Pod name/IP.
+ podNamespace := workloadNamespace
+ podName := workloadName
instance.PodNamespace = &podNamespace
instance.PodName = &podName
instance.Status = "creating"
@@ -576,6 +661,54 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models
return instance, nil
}
+func (s *instanceService) createV2Instance(ctx context.Context, userID int, req CreateInstanceRequest, runtimeType string, environmentOverridesJSON *string) (*models.Instance, error) {
+ now := time.Now()
+ workspaceRoot := s.runtimeWorkspaceRoot()
+ instance := &models.Instance{
+ UserID: userID,
+ Name: strings.TrimSpace(req.Name),
+ Description: trimOptionalString(req.Description),
+ Type: runtimeType,
+ RuntimeType: RuntimeBackendGateway,
+ InstanceMode: InstanceModeLite,
+ Status: "creating",
+ CPUCores: req.CPUCores,
+ MemoryGB: req.MemoryGB,
+ DiskGB: req.DiskGB,
+ GPUEnabled: req.GPUEnabled,
+ GPUCount: req.GPUCount,
+ OSType: req.OSType,
+ OSVersion: req.OSVersion,
+ ImageRegistry: req.ImageRegistry,
+ ImageTag: req.ImageTag,
+ EnvironmentOverridesJSON: environmentOverridesJSON,
+ StorageClass: strings.TrimSpace(req.StorageClass),
+ MountPath: workspaceRoot,
+ RuntimeGeneration: 1,
+ CreatedAt: now,
+ UpdatedAt: now,
+ StartedAt: &now,
+ }
+
+ if err := s.instanceRepo.Create(instance); err != nil {
+ return nil, fmt.Errorf("failed to create instance record: %w", err)
+ }
+
+ workspacePath, err := ensureRuntimeWorkspaceDirectories(workspaceRoot, runtimeType, userID, instance.ID)
+ if err != nil {
+ _ = s.instanceRepo.Delete(instance.ID)
+ return nil, fmt.Errorf("failed to create instance workspace: %w", err)
+ }
+ if err := s.instanceRepo.SetWorkspacePath(ctx, instance.ID, workspacePath); err != nil {
+ _ = s.instanceRepo.Delete(instance.ID)
+ return nil, fmt.Errorf("failed to persist instance workspace path: %w", err)
+ }
+ instance.WorkspacePath = &workspacePath
+
+ GetHub().BroadcastInstanceStatus(userID, instance)
+ return instance, nil
+}
+
// GetByID gets an instance by ID
func (s *instanceService) GetByID(id int) (*models.Instance, error) {
return s.instanceRepo.GetByID(id)
@@ -626,6 +759,13 @@ func (s *instanceService) Start(instanceID int) error {
if instance.Status == "running" {
return fmt.Errorf("instance is already running")
}
+ if err := s.enforceInstanceModeLimits(ctx, modeForExistingInstance(instance), instance.CPUCores, instance.MemoryGB, instance.DiskGB, instance.GPUCount); err != nil {
+ return err
+ }
+
+ if runtimeType, ok := v2RuntimeTypeForInstance(instance); ok {
+ return s.startV2Instance(ctx, instance, runtimeType)
+ }
if _, err := s.ensureGatewayToken(instance); err != nil {
return fmt.Errorf("failed to provision instance gateway token: %w", err)
@@ -686,12 +826,19 @@ func (s *instanceService) Start(instanceID int) error {
SecurityMode: s.securityModeForInstance(instance.Type),
}
- pod, err := s.podService.CreatePod(ctx, podConfig)
- if err != nil {
- return fmt.Errorf("failed to create pod: %w", err)
- }
-
+ var workloadNamespace string
+ var workloadName string
if instanceUsesDesktopRuntime(instance) {
+ if s.deploymentService == nil {
+ return fmt.Errorf("instance deployment service is not configured")
+ }
+ deployment, err := s.deploymentService.EnsureDeployment(ctx, podConfig, 1)
+ if err != nil {
+ return fmt.Errorf("failed to ensure deployment: %w", err)
+ }
+ workloadNamespace = deployment.Namespace
+ workloadName = deployment.Name
+
// Ensure Service exists (create if not exists)
serviceExists, _ := s.serviceService.ServiceExists(ctx, instance.UserID, instance.ID)
if !serviceExists {
@@ -708,12 +855,19 @@ func (s *instanceService) Start(instanceID int) error {
// Don't fail if service creation fails, pod is already running
}
}
+ } else {
+ pod, err := s.podService.CreatePod(ctx, podConfig)
+ if err != nil {
+ return fmt.Errorf("failed to create pod: %w", err)
+ }
+ workloadNamespace = pod.Namespace
+ workloadName = pod.Name
}
// Update instance status
now := time.Now()
- podNamespace := pod.Namespace
- podName := pod.Name
+ podNamespace := workloadNamespace
+ podName := workloadName
instance.PodNamespace = &podNamespace
instance.PodName = &podName
instance.Status = "creating"
@@ -792,6 +946,25 @@ func (s *instanceService) buildGatewayEnv(instance *models.Instance) (map[string
}, nil
}
+func (s *instanceService) BuildGatewayEnv(instance *models.Instance) (map[string]string, error) {
+ if instance == nil || !supportsManagedRuntimeIntegration(instance.Type) {
+ return s.buildGatewayEnv(instance)
+ }
+ if instance.AccessToken == nil || strings.TrimSpace(*instance.AccessToken) == "" {
+ if s == nil || s.instanceRepo == nil {
+ return nil, fmt.Errorf("instance repository is not configured")
+ }
+ if _, err := s.ensureGatewayToken(instance); err != nil {
+ return nil, err
+ }
+ }
+ gatewayEnv, err := s.buildGatewayEnv(instance)
+ if err != nil {
+ return nil, err
+ }
+ return buildInstanceGatewayEnv(instance, gatewayEnv)
+}
+
func (s *instanceService) ensureAgentBootstrapToken(instance *models.Instance) (string, error) {
if instance.AgentBootstrapToken != nil && strings.TrimSpace(*instance.AgentBootstrapToken) != "" {
return strings.TrimSpace(*instance.AgentBootstrapToken), nil
@@ -964,6 +1137,130 @@ func mergeEnvMaps(base map[string]string, overlay map[string]string) map[string]
return merged
}
+func (s *instanceService) startV2Instance(ctx context.Context, instance *models.Instance, runtimeType string) error {
+ if err := s.ensureV2Workspace(ctx, instance, runtimeType); err != nil {
+ return err
+ }
+ nextGeneration := instance.RuntimeGeneration + 1
+ if nextGeneration <= 0 {
+ nextGeneration = 1
+ }
+ if err := s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "creating", nextGeneration, nil); err != nil {
+ return fmt.Errorf("failed to mark v2 instance creating: %w", err)
+ }
+ instance.Status = "creating"
+ instance.RuntimeGeneration = nextGeneration
+ instance.RuntimeErrorMessage = nil
+ now := time.Now()
+ instance.StartedAt = &now
+ instance.UpdatedAt = now
+ GetHub().BroadcastInstanceStatus(instance.UserID, instance)
+ return nil
+}
+
+func (s *instanceService) stopV2Instance(ctx context.Context, instance *models.Instance) error {
+ if err := s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "stopped", instance.RuntimeGeneration, nil); err != nil {
+ return fmt.Errorf("failed to mark v2 instance stopped: %w", err)
+ }
+ now := time.Now()
+ instance.Status = "stopped"
+ instance.StoppedAt = &now
+ instance.PodName = nil
+ instance.PodNamespace = nil
+ instance.PodIP = nil
+ instance.UpdatedAt = now
+ GetHub().BroadcastInstanceStatus(instance.UserID, instance)
+ return s.cleanupV2GatewayBinding(ctx, instance)
+}
+
+func (s *instanceService) deleteV2Instance(ctx context.Context, instance *models.Instance) error {
+ if instance.Status != "deleting" {
+ now := time.Now()
+ instance.Status = "deleting"
+ instance.UpdatedAt = now
+ if err := s.instanceRepo.Update(instance); err != nil {
+ return fmt.Errorf("failed to mark v2 instance as deleting: %w", err)
+ }
+ GetHub().BroadcastInstanceStatus(instance.UserID, instance)
+ }
+
+ cleanupErr := s.cleanupV2GatewayBinding(ctx, instance)
+ if cleanupErr != nil {
+ return cleanupErr
+ }
+ if err := s.instanceRepo.Delete(instance.ID); err != nil {
+ return fmt.Errorf("failed to delete v2 instance record: %w", err)
+ }
+ return nil
+}
+
+func (s *instanceService) cleanupV2GatewayBinding(ctx context.Context, instance *models.Instance) error {
+ if s.bindingRepo == nil {
+ return nil
+ }
+ binding, err := s.bindingRepo.GetByInstanceID(ctx, instance.ID)
+ if err != nil {
+ return fmt.Errorf("failed to get v2 runtime binding: %w", err)
+ }
+ if binding == nil {
+ return nil
+ }
+
+ if s.runtimePodRepo != nil {
+ pod, podErr := s.runtimePodRepo.GetByID(ctx, binding.RuntimePodID)
+ if podErr != nil {
+ return fmt.Errorf("failed to get runtime pod %d for v2 cleanup: %w", binding.RuntimePodID, podErr)
+ } else if pod == nil {
+ return fmt.Errorf("runtime pod %d is not available for v2 cleanup", binding.RuntimePodID)
+ } else if pod != nil && pod.AgentEndpoint != nil && strings.TrimSpace(*pod.AgentEndpoint) != "" && s.agentClient != nil && binding.GatewayID != "" {
+ if err := s.agentClient.DeleteGateway(ctx, strings.TrimSpace(*pod.AgentEndpoint), binding.GatewayID); err != nil {
+ return fmt.Errorf("failed to delete v2 gateway: %w", err)
+ }
+ }
+ }
+
+ if err := s.bindingRepo.DeleteByInstanceIDAndReleaseSlot(ctx, instance.ID, binding.RuntimePodID); err != nil {
+ return fmt.Errorf("failed to delete v2 runtime binding and release slot: %w", err)
+ }
+ return nil
+}
+
+func (s *instanceService) ensureV2Workspace(ctx context.Context, instance *models.Instance, runtimeType string) error {
+ if instance.WorkspacePath != nil && strings.TrimSpace(*instance.WorkspacePath) != "" {
+ return nil
+ }
+ workspacePath, err := ensureRuntimeWorkspaceDirectories(s.runtimeWorkspaceRoot(), runtimeType, instance.UserID, instance.ID)
+ if err != nil {
+ return fmt.Errorf("failed to create instance workspace: %w", err)
+ }
+ if err := s.instanceRepo.SetWorkspacePath(ctx, instance.ID, workspacePath); err != nil {
+ return fmt.Errorf("failed to persist instance workspace path: %w", err)
+ }
+ instance.WorkspacePath = &workspacePath
+ return nil
+}
+
+func ensureRuntimeWorkspaceDirectories(root, runtimeType string, userID, instanceID int) (string, error) {
+ workspacePath := RuntimeWorkspacePathWithRoot(root, runtimeType, userID, instanceID)
+ if err := os.MkdirAll(workspacePath, 0750); err != nil {
+ return "", err
+ }
+
+ // Allow the isolated gateway UID to traverse to its own workspace without
+ // granting read/list access to sibling user or instance directories.
+ userRoot := path.Dir(workspacePath)
+ runtimeRoot := path.Dir(userRoot)
+ for _, dir := range []string{runtimeRoot, userRoot} {
+ if err := os.Chmod(dir, 0711); err != nil {
+ return "", err
+ }
+ }
+ if err := os.Chmod(workspacePath, 0750); err != nil {
+ return "", err
+ }
+ return workspacePath, nil
+}
+
// Stop stops an instance
func (s *instanceService) Stop(instanceID int) error {
ctx := context.Background()
@@ -977,13 +1274,29 @@ func (s *instanceService) Stop(instanceID int) error {
return fmt.Errorf("instance not found")
}
+ if _, ok := v2RuntimeTypeForInstance(instance); ok {
+ return s.stopV2Instance(ctx, instance)
+ }
+
if instance.Status != "running" {
return fmt.Errorf("instance is not running")
}
- // Delete pod
- if err := s.podService.DeletePod(ctx, instance.UserID, instance.ID); err != nil {
- return fmt.Errorf("failed to delete pod: %w", err)
+ if instanceUsesDesktopRuntime(instance) {
+ if s.deploymentService == nil {
+ return fmt.Errorf("instance deployment service is not configured")
+ }
+ if err := s.deploymentService.ScaleDeployment(ctx, instance.UserID, instance.ID, 0); err != nil {
+ fmt.Printf("Warning: failed to stop deployment for instance %d, falling back to pod delete: %v\n", instance.ID, err)
+ if podErr := s.podService.DeletePod(ctx, instance.UserID, instance.ID); podErr != nil {
+ return fmt.Errorf("failed to stop deployment: %w", err)
+ }
+ }
+ } else {
+ // Delete shell pod
+ if err := s.podService.DeletePod(ctx, instance.UserID, instance.ID); err != nil {
+ return fmt.Errorf("failed to delete pod: %w", err)
+ }
}
// Update instance status
@@ -1029,6 +1342,10 @@ func (s *instanceService) Delete(instanceID int) error {
return fmt.Errorf("instance not found")
}
+ if _, ok := v2RuntimeTypeForInstance(instance); ok {
+ return s.deleteV2Instance(context.Background(), instance)
+ }
+
if instance.Status != "deleting" {
now := time.Now()
instance.Status = "deleting"
@@ -1300,6 +1617,18 @@ func (s *instanceService) GetInstanceStatus(instanceID int) (*InstanceStatus, er
return nil, fmt.Errorf("instance not found")
}
+ if runtimeType, ok := v2RuntimeTypeForInstance(instance); ok {
+ return &InstanceStatus{
+ InstanceID: instance.ID,
+ Status: instance.Status,
+ Availability: s.v2InstanceAvailability(ctx, instance),
+ AgentType: runtimeType,
+ WorkspaceUsageBytes: instance.WorkspaceUsageBytes,
+ CreatedAt: instance.CreatedAt,
+ StartedAt: instance.StartedAt,
+ }, nil
+ }
+
status := &InstanceStatus{
InstanceID: instance.ID,
Status: instance.Status,
@@ -1334,6 +1663,13 @@ func (s *instanceService) ForceSyncInstance(instanceID int) error {
return fmt.Errorf("instance not found")
}
+ if _, ok := v2RuntimeTypeForInstance(instance); ok {
+ return nil
+ }
+ if instanceUsesDesktopRuntime(instance) {
+ return s.forceSyncDeploymentInstance(ctx, instance)
+ }
+
fmt.Printf("Force syncing instance %d (current status: %s, user: %d)\n", instanceID, instance.Status, instance.UserID)
// First try direct lookup by instance ID
@@ -1452,6 +1788,60 @@ func (s *instanceService) ForceSyncInstance(instanceID int) error {
return nil
}
+func (s *instanceService) forceSyncDeploymentInstance(ctx context.Context, instance *models.Instance) error {
+ if s.deploymentService == nil {
+ return fmt.Errorf("instance deployment service is not configured")
+ }
+ deployment, err := s.deploymentService.GetDeployment(ctx, instance.UserID, instance.ID)
+ if err != nil {
+ if instance.Status == "running" || instance.Status == "creating" {
+ nextStatus := "stopped"
+ if instance.Status == "creating" {
+ nextStatus = "error"
+ }
+ instance.Status = nextStatus
+ instance.PodName = nil
+ instance.PodNamespace = nil
+ instance.PodIP = nil
+ instance.UpdatedAt = time.Now()
+ if err := s.instanceRepo.Update(instance); err != nil {
+ return fmt.Errorf("failed to update instance status: %w", err)
+ }
+ GetHub().BroadcastInstanceStatus(instance.UserID, instance)
+ }
+ return nil
+ }
+
+ needsUpdate := false
+ desiredStatus := mapDeploymentToInstanceStatus(deployment)
+ if instance.Status != desiredStatus {
+ instance.Status = desiredStatus
+ needsUpdate = true
+ }
+ if pod, podErr := s.deploymentService.GetActivePod(ctx, instance.UserID, instance.ID); podErr == nil && pod != nil {
+ if pod.Status.PodIP != "" && (instance.PodIP == nil || *instance.PodIP != pod.Status.PodIP) {
+ instance.PodIP = &pod.Status.PodIP
+ needsUpdate = true
+ }
+ if instance.PodName == nil || *instance.PodName != pod.Name {
+ instance.PodName = &pod.Name
+ needsUpdate = true
+ }
+ if instance.PodNamespace == nil || *instance.PodNamespace != pod.Namespace {
+ instance.PodNamespace = &pod.Namespace
+ needsUpdate = true
+ }
+ }
+ if needsUpdate {
+ instance.UpdatedAt = time.Now()
+ if err := s.instanceRepo.Update(instance); err != nil {
+ return fmt.Errorf("failed to update instance: %w", err)
+ }
+ GetHub().BroadcastInstanceStatus(instance.UserID, instance)
+ }
+ return nil
+}
+
func additionalServicePorts(primaryPort int32) []int32 {
if primaryPort == 3000 || primaryPort == 8082 {
return []int32{3000, 8082}
@@ -1462,10 +1852,12 @@ func additionalServicePorts(primaryPort int32) []int32 {
func normalizeInstanceRuntimeType(runtimeType string) string {
switch strings.ToLower(strings.TrimSpace(runtimeType)) {
+ case RuntimeBackendGateway:
+ return RuntimeBackendGateway
case "shell":
- return "shell"
+ return RuntimeBackendShell
default:
- return "desktop"
+ return RuntimeBackendDesktop
}
}
@@ -1473,5 +1865,182 @@ func instanceUsesDesktopRuntime(instance *models.Instance) bool {
if instance == nil {
return true
}
- return normalizeInstanceRuntimeType(instance.RuntimeType) == "desktop"
+ return normalizeInstanceRuntimeType(instance.RuntimeType) == RuntimeBackendDesktop
+}
+
+func resolveCreateInstanceMode(req CreateInstanceRequest) string {
+ if mode, ok := NormalizeInstanceMode(req.Mode); ok {
+ return mode
+ }
+ if mode, ok := NormalizeInstanceMode(req.InstanceMode); ok {
+ return mode
+ }
+ if strings.TrimSpace(req.RuntimeType) == "" {
+ return InstanceModeLite
+ }
+ return InstanceModeForRuntimeType(normalizeInstanceRuntimeType(req.RuntimeType))
+}
+
+func hasExplicitCreateInstanceMode(req CreateInstanceRequest) bool {
+ if _, ok := NormalizeInstanceMode(req.Mode); ok {
+ return true
+ }
+ if _, ok := NormalizeInstanceMode(req.InstanceMode); ok {
+ return true
+ }
+ return false
+}
+
+func modeForExistingInstance(instance *models.Instance) string {
+ if instance == nil {
+ return InstanceModeLite
+ }
+ if mode, ok := NormalizeInstanceMode(instance.InstanceMode); ok {
+ return mode
+ }
+ return InstanceModeForRuntimeType(normalizeInstanceRuntimeType(instance.RuntimeType))
+}
+
+func instanceModeUsesDedicatedResources(mode string) bool {
+ normalized, ok := NormalizeInstanceMode(mode)
+ return ok && normalized == InstanceModePro
+}
+
+func (s *instanceService) enforceInstanceModeLimits(ctx context.Context, mode string, cpuCores float64, memoryGB, storageGB, gpuCount int) error {
+ normalizedMode, ok := NormalizeInstanceMode(mode)
+ if !ok {
+ return fmt.Errorf("unsupported instance mode %q", mode)
+ }
+ limits := loadInstanceModeLimitConfig(normalizedMode)
+ if limits.Capacity != nil {
+ if *limits.Capacity <= 0 {
+ return fmt.Errorf("%s instance mode is disabled", normalizedMode)
+ }
+ if s == nil || s.instanceRepo == nil {
+ return fmt.Errorf("instance repository is not configured")
+ }
+ count, err := s.instanceRepo.CountActiveByMode(ctx, normalizedMode)
+ if err != nil {
+ return err
+ }
+ if count >= *limits.Capacity {
+ return fmt.Errorf("%s instance capacity reached: %d/%d", normalizedMode, count, *limits.Capacity)
+ }
+ }
+ if !instanceModeUsesDedicatedResources(normalizedMode) {
+ return nil
+ }
+ if limits.MaxCPU != nil && cpuCores > *limits.MaxCPU {
+ return fmt.Errorf("%s CPU cores exceed mode limit: requested %g, max %g", normalizedMode, cpuCores, *limits.MaxCPU)
+ }
+ if limits.MaxMemoryGB != nil && memoryGB > *limits.MaxMemoryGB {
+ return fmt.Errorf("%s memory exceeds mode limit: requested %dGB, max %dGB", normalizedMode, memoryGB, *limits.MaxMemoryGB)
+ }
+ if limits.MaxStorageGB != nil && storageGB > *limits.MaxStorageGB {
+ return fmt.Errorf("%s storage exceeds mode limit: requested %dGB, max %dGB", normalizedMode, storageGB, *limits.MaxStorageGB)
+ }
+ if limits.MaxGPUCount != nil && gpuCount > *limits.MaxGPUCount {
+ return fmt.Errorf("%s GPU count exceeds mode limit: requested %d, max %d", normalizedMode, gpuCount, *limits.MaxGPUCount)
+ }
+ return nil
+}
+
+func loadInstanceModeLimitConfig(mode string) instanceModeLimitConfig {
+ prefix := "CLAWMANAGER_" + strings.ToUpper(mode) + "_"
+ return instanceModeLimitConfig{
+ Capacity: optionalIntEnv(prefix + "CAPACITY"),
+ MaxCPU: optionalFloatEnv(prefix + "MAX_CPU_CORES"),
+ MaxMemoryGB: optionalIntEnv(prefix + "MAX_MEMORY_GB"),
+ MaxStorageGB: optionalIntEnv(prefix + "MAX_STORAGE_GB"),
+ MaxGPUCount: optionalIntEnv(prefix + "MAX_GPU_COUNT"),
+ }
+}
+
+func optionalIntEnv(key string) *int {
+ value := strings.TrimSpace(os.Getenv(key))
+ if value == "" {
+ return nil
+ }
+ parsed, err := strconv.Atoi(value)
+ if err != nil {
+ return nil
+ }
+ return &parsed
+}
+
+func optionalFloatEnv(key string) *float64 {
+ value := strings.TrimSpace(os.Getenv(key))
+ if value == "" {
+ return nil
+ }
+ parsed, err := strconv.ParseFloat(value, 64)
+ if err != nil {
+ return nil
+ }
+ return &parsed
+}
+
+func (s *instanceService) runtimeWorkspaceRoot() string {
+ if s != nil && strings.TrimSpace(s.workspaceRoot) != "" {
+ return strings.TrimSpace(s.workspaceRoot)
+ }
+ return "/workspaces"
+}
+
+func v2RuntimeTypeForInstance(instance *models.Instance) (string, bool) {
+ if instance == nil {
+ return "", false
+ }
+ runtimeType, ok := NormalizeV2RuntimeType(instance.Type)
+ if !ok {
+ return "", false
+ }
+ if strings.EqualFold(strings.TrimSpace(instance.RuntimeType), RuntimeBackendGateway) {
+ return runtimeType, true
+ }
+ if mode, ok := NormalizeInstanceMode(instance.InstanceMode); ok && mode == InstanceModeLite {
+ return runtimeType, true
+ }
+ return "", false
+}
+
+func availabilityForStatus(status string) string {
+ switch strings.ToLower(strings.TrimSpace(status)) {
+ case "running":
+ return "available"
+ case "creating":
+ return "starting"
+ default:
+ return "unavailable"
+ }
+}
+
+func (s *instanceService) v2InstanceAvailability(ctx context.Context, instance *models.Instance) string {
+ base := availabilityForStatus(instance.Status)
+ if base != "available" {
+ return base
+ }
+ if s == nil || s.bindingRepo == nil || s.runtimePodRepo == nil {
+ return "unavailable"
+ }
+ binding, err := s.bindingRepo.GetRunningByInstanceID(ctx, instance.ID)
+ if err != nil || binding == nil || binding.GatewayPort <= 0 {
+ return "unavailable"
+ }
+ pod, err := s.runtimePodRepo.GetByID(ctx, binding.RuntimePodID)
+ if err != nil || pod == nil || pod.PodIP == nil || strings.TrimSpace(*pod.PodIP) == "" {
+ return "unavailable"
+ }
+ return "available"
+}
+
+func trimOptionalString(value *string) *string {
+ if value == nil {
+ return nil
+ }
+ trimmed := strings.TrimSpace(*value)
+ if trimmed == "" {
+ return nil
+ }
+ return &trimmed
}
diff --git a/backend/internal/services/instance_service_test.go b/backend/internal/services/instance_service_test.go
index a91656c..1064c0d 100644
--- a/backend/internal/services/instance_service_test.go
+++ b/backend/internal/services/instance_service_test.go
@@ -80,6 +80,79 @@ func TestBuildGatewayEnvInjectsGatewayModelCatalog(t *testing.T) {
}
}
+func TestBuildGatewayEnvEnsuresMissingGatewayToken(t *testing.T) {
+ t.Setenv("CLAWMANAGER_LLM_GATEWAY_BASE_URL", "http://gateway.example/api/v1/gateway/llm")
+
+ instanceRepo := &stubGatewayEnvInstanceRepository{fakeRuntimeInstanceRepo: newFakeRuntimeInstanceRepo()}
+ service := &instanceService{
+ instanceRepo: instanceRepo,
+ llmModelRepo: &stubLLMModelRepository{
+ active: []models.LLMModel{{DisplayName: "auto"}},
+ },
+ }
+ instance := &models.Instance{
+ ID: 68,
+ UserID: 1,
+ Type: "openclaw",
+ }
+
+ env, err := service.BuildGatewayEnv(instance)
+ if err != nil {
+ t.Fatalf("BuildGatewayEnv returned error: %v", err)
+ }
+ if instance.AccessToken == nil || *instance.AccessToken == "" {
+ t.Fatal("BuildGatewayEnv did not provision instance access token")
+ }
+ if env["CLAWMANAGER_LLM_API_KEY"] != *instance.AccessToken {
+ t.Fatalf("CLAWMANAGER_LLM_API_KEY = %q, want provisioned token %q", env["CLAWMANAGER_LLM_API_KEY"], *instance.AccessToken)
+ }
+ if got := instanceRepo.updated[68]; got == nil || got.AccessToken == nil || *got.AccessToken != *instance.AccessToken {
+ t.Fatalf("repository update did not persist provisioned token: %#v", instanceRepo.updated[68])
+ }
+}
+
+func TestBuildGatewayEnvMergesEnvironmentOverrides(t *testing.T) {
+ t.Setenv("CLAWMANAGER_LLM_GATEWAY_BASE_URL", "http://gateway.example/api/v1/gateway/llm")
+ token := "igt_test_token"
+ raw, err := marshalEnvironmentOverrides(map[string]string{
+ "CLAWMANAGER_TEAM_ENABLED": "true",
+ "CLAWMANAGER_TEAM_MEMBER_ID": "lite-worker",
+ "CUSTOM_GATEWAY_ENV": "enabled",
+ })
+ if err != nil {
+ t.Fatalf("marshalEnvironmentOverrides returned error: %v", err)
+ }
+ service := &instanceService{
+ llmModelRepo: &stubLLMModelRepository{
+ active: []models.LLMModel{{DisplayName: "auto"}},
+ },
+ }
+
+ env, err := service.BuildGatewayEnv(&models.Instance{
+ ID: 88,
+ Type: "openclaw",
+ RuntimeType: RuntimeBackendGateway,
+ AccessToken: &token,
+ EnvironmentOverridesJSON: raw,
+ })
+ if err != nil {
+ t.Fatalf("BuildGatewayEnv returned error: %v", err)
+ }
+
+ if env["CLAWMANAGER_TEAM_ENABLED"] != "true" || env["CLAWMANAGER_TEAM_MEMBER_ID"] != "lite-worker" {
+ t.Fatalf("expected Team environment overrides to be merged into gateway env, got %#v", env)
+ }
+ if env["CUSTOM_GATEWAY_ENV"] != "enabled" {
+ t.Fatalf("expected custom gateway environment override to be merged, got %#v", env)
+ }
+ if env["CLAWMANAGER_LLM_API_KEY"] != token {
+ t.Fatalf("expected gateway token env to remain available")
+ }
+ if env["CLAWMANAGER_RUNTIME_TYPE"] != RuntimeBackendGateway {
+ t.Fatalf("expected runtime type marker %q, got %q", RuntimeBackendGateway, env["CLAWMANAGER_RUNTIME_TYPE"])
+ }
+}
+
func TestBuildGatewayEnvSkipsUnmanagedRuntime(t *testing.T) {
token := "igt_test_token"
service := &instanceService{}
@@ -96,6 +169,20 @@ func TestBuildGatewayEnvSkipsUnmanagedRuntime(t *testing.T) {
}
}
+type stubGatewayEnvInstanceRepository struct {
+ *fakeRuntimeInstanceRepo
+ updated map[int]*models.Instance
+}
+
+func (r *stubGatewayEnvInstanceRepository) Update(instance *models.Instance) error {
+ if r.updated == nil {
+ r.updated = map[int]*models.Instance{}
+ }
+ copy := *instance
+ r.updated[instance.ID] = ©
+ return nil
+}
+
func TestBuildAgentEnvInjectsHermesAgentConfig(t *testing.T) {
t.Setenv("CLAWMANAGER_AGENT_CONTROL_BASE_URL", "http://agent-control.example")
diff --git a/backend/internal/services/instance_service_v2_test.go b/backend/internal/services/instance_service_v2_test.go
new file mode 100644
index 0000000..479790f
--- /dev/null
+++ b/backend/internal/services/instance_service_v2_test.go
@@ -0,0 +1,751 @@
+package services
+
+import (
+ "context"
+ "errors"
+ "os"
+ "path"
+ "strings"
+ "testing"
+
+ "clawreef/internal/models"
+)
+
+func TestInstanceServiceCreateV2CreatesWorkspaceOnly(t *testing.T) {
+ workspaceRoot := strings.ReplaceAll(t.TempDir(), "\\", "/")
+ instanceRepo := newV2LifecycleInstanceRepo()
+ service := &instanceService{
+ instanceRepo: instanceRepo,
+ quotaRepo: v2LifecycleQuotaRepo{},
+ workspaceRoot: workspaceRoot,
+ }
+
+ instance, err := service.Create(45, CreateInstanceRequest{
+ Name: "OpenClaw Dev",
+ Type: " OpenClaw ",
+ CPUCores: 2,
+ MemoryGB: 4,
+ DiskGB: 20,
+ OSType: "openclaw",
+ OSVersion: "latest",
+ })
+ if err != nil {
+ t.Fatalf("Create returned error: %v", err)
+ }
+
+ expectedWorkspacePath := RuntimeWorkspacePathWithRoot(workspaceRoot, "openclaw", 45, instance.ID)
+ if instance.Type != "openclaw" {
+ t.Fatalf("instance type = %q, want openclaw", instance.Type)
+ }
+ if instance.RuntimeType != "gateway" {
+ t.Fatalf("runtime type = %q, want gateway", instance.RuntimeType)
+ }
+ if instance.InstanceMode != InstanceModeLite {
+ t.Fatalf("instance mode = %q, want lite", instance.InstanceMode)
+ }
+ if instance.Status != "creating" {
+ t.Fatalf("status = %q, want creating", instance.Status)
+ }
+ if instance.RuntimeGeneration != 1 {
+ t.Fatalf("runtime generation = %d, want 1", instance.RuntimeGeneration)
+ }
+ if instance.WorkspacePath == nil || *instance.WorkspacePath != expectedWorkspacePath {
+ t.Fatalf("workspace path = %v, want %q", instance.WorkspacePath, expectedWorkspacePath)
+ }
+ if _, err := os.Stat(expectedWorkspacePath); err != nil {
+ t.Fatalf("expected workspace directory to exist: %v", err)
+ }
+ if os.PathSeparator == '/' {
+ assertDirMode(t, path.Dir(path.Dir(expectedWorkspacePath)), 0711)
+ assertDirMode(t, path.Dir(expectedWorkspacePath), 0711)
+ assertDirMode(t, expectedWorkspacePath, 0750)
+ }
+ if instance.PodName != nil || instance.PodNamespace != nil || instance.PodIP != nil {
+ t.Fatalf("V2 create must not set per-instance pod fields: %#v", instance)
+ }
+ if len(instanceRepo.created) != 1 {
+ t.Fatalf("created instance records = %d, want 1", len(instanceRepo.created))
+ }
+}
+
+func TestInstanceServiceCreateLiteSkipsPerInstanceResourceQuota(t *testing.T) {
+ workspaceRoot := strings.ReplaceAll(t.TempDir(), "\\", "/")
+ instanceRepo := newV2LifecycleInstanceRepo()
+ service := &instanceService{
+ instanceRepo: instanceRepo,
+ quotaRepo: fixedV2QuotaRepo{cpu: 1, memory: 1, storage: 10, gpu: 0},
+ workspaceRoot: workspaceRoot,
+ }
+
+ instance, err := service.Create(45, CreateInstanceRequest{
+ Name: "Lite Gateway",
+ Type: "openclaw",
+ Mode: InstanceModeLite,
+ RuntimeType: RuntimeBackendGateway,
+ CPUCores: 8,
+ MemoryGB: 32,
+ DiskGB: 200,
+ GPUEnabled: true,
+ GPUCount: 1,
+ OSType: "openclaw",
+ OSVersion: "latest",
+ StorageClass: "manual",
+ })
+ if err != nil {
+ t.Fatalf("Create lite returned error: %v", err)
+ }
+ if instance.InstanceMode != InstanceModeLite || instance.RuntimeType != RuntimeBackendGateway {
+ t.Fatalf("created runtime = mode %q type %q, want lite gateway", instance.InstanceMode, instance.RuntimeType)
+ }
+}
+
+func TestInstanceServiceCreateProEnforcesPerInstanceResourceQuota(t *testing.T) {
+ instanceRepo := newV2LifecycleInstanceRepo()
+ service := &instanceService{
+ instanceRepo: instanceRepo,
+ quotaRepo: fixedV2QuotaRepo{cpu: 1, memory: 1, storage: 10, gpu: 0},
+ pvcService: nil,
+ deploymentService: nil,
+ serviceService: nil,
+ networkPolicyService: nil,
+ openClawConfigService: nil,
+ }
+
+ _, err := service.Create(45, CreateInstanceRequest{
+ Name: "Pro Desktop",
+ Type: "openclaw",
+ Mode: InstanceModePro,
+ RuntimeType: RuntimeBackendDesktop,
+ CPUCores: 8,
+ MemoryGB: 32,
+ DiskGB: 200,
+ OSType: "openclaw",
+ OSVersion: "latest",
+ })
+ if err == nil || !strings.Contains(err.Error(), "CPU cores exceed quota") {
+ t.Fatalf("Create pro error = %v, want CPU quota failure", err)
+ }
+}
+
+func assertDirMode(t *testing.T, dir string, want os.FileMode) {
+ t.Helper()
+ info, err := os.Stat(dir)
+ if err != nil {
+ t.Fatalf("stat %s: %v", dir, err)
+ }
+ if got := info.Mode().Perm(); got != want {
+ t.Fatalf("mode %s = %04o, want %04o", dir, got, want)
+ }
+}
+
+func TestInstanceServiceStartV2MarksCreatingWithNextGeneration(t *testing.T) {
+ workspacePath := "/workspaces/hermes/user-45/instance-77"
+ instanceRepo := newV2LifecycleInstanceRepo()
+ instanceRepo.byID[77] = &models.Instance{
+ ID: 77,
+ UserID: 45,
+ Type: "hermes",
+ RuntimeType: "gateway",
+ Status: "stopped",
+ WorkspacePath: &workspacePath,
+ RuntimeGeneration: 4,
+ }
+ service := &instanceService{instanceRepo: instanceRepo}
+
+ if err := service.Start(77); err != nil {
+ t.Fatalf("Start returned error: %v", err)
+ }
+
+ state := instanceRepo.runtimeStates[77]
+ if state.status != "creating" || state.generation != 5 {
+ t.Fatalf("runtime state = %#v, want creating generation 5", state)
+ }
+}
+
+func TestInstanceServiceStopV2DeletesGatewayBindingAndReleasesSlot(t *testing.T) {
+ workspacePath := "/workspaces/openclaw/user-45/instance-88"
+ instanceRepo := newV2LifecycleInstanceRepo()
+ instanceRepo.byID[88] = &models.Instance{
+ ID: 88,
+ UserID: 45,
+ Type: "openclaw",
+ RuntimeType: "gateway",
+ Status: "running",
+ WorkspacePath: &workspacePath,
+ RuntimeGeneration: 3,
+ }
+ endpoint := "http://agent.local:19090"
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 9: {ID: 9, AgentEndpoint: &endpoint},
+ },
+ }
+ bindingRepo := newFakeRuntimeBindingRepo()
+ bindingRepo.bindings[88] = &models.InstanceRuntimeBinding{
+ InstanceID: 88,
+ RuntimePodID: 9,
+ GatewayID: "gw-88",
+ GatewayPort: 20018,
+ State: "running",
+ Generation: 3,
+ }
+ agent := &fakeRuntimeAgentClient{}
+ service := &instanceService{
+ instanceRepo: instanceRepo,
+ runtimePodRepo: podRepo,
+ bindingRepo: bindingRepo,
+ agentClient: agent,
+ workspaceRoot: "/workspaces",
+ }
+
+ if err := service.Stop(88); err != nil {
+ t.Fatalf("Stop returned error: %v", err)
+ }
+
+ if len(agent.deleteRequests) != 1 || agent.deleteRequests[0].endpoint != endpoint || agent.deleteRequests[0].gatewayID != "gw-88" {
+ t.Fatalf("delete gateway requests = %#v", agent.deleteRequests)
+ }
+ if bindingRepo.deleteAndReleaseCalls[88] != 1 {
+ t.Fatalf("binding delete and release calls = %d, want 1", bindingRepo.deleteAndReleaseCalls[88])
+ }
+ state := instanceRepo.runtimeStates[88]
+ if state.status != "stopped" || state.generation != 3 {
+ t.Fatalf("runtime state = %#v, want stopped generation 3", state)
+ }
+}
+
+func TestInstanceServiceStopV2KeepsBindingWhenAgentDeleteFails(t *testing.T) {
+ workspacePath := "/workspaces/openclaw/user-45/instance-188"
+ instanceRepo := newV2LifecycleInstanceRepo()
+ instanceRepo.byID[188] = &models.Instance{
+ ID: 188,
+ UserID: 45,
+ Type: "openclaw",
+ RuntimeType: "gateway",
+ Status: "running",
+ WorkspacePath: &workspacePath,
+ RuntimeGeneration: 6,
+ }
+ endpoint := "http://agent.local:19090"
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 19: {ID: 19, AgentEndpoint: &endpoint},
+ },
+ }
+ bindingRepo := newFakeRuntimeBindingRepo()
+ bindingRepo.bindings[188] = &models.InstanceRuntimeBinding{
+ InstanceID: 188,
+ RuntimePodID: 19,
+ GatewayID: "gw-188",
+ GatewayPort: 20018,
+ State: "running",
+ Generation: 6,
+ }
+ agent := &fakeRuntimeAgentClient{deleteErr: errors.New("agent delete failed")}
+ service := &instanceService{
+ instanceRepo: instanceRepo,
+ runtimePodRepo: podRepo,
+ bindingRepo: bindingRepo,
+ agentClient: agent,
+ }
+
+ err := service.Stop(188)
+ if err == nil || !strings.Contains(err.Error(), "failed to delete v2 gateway") {
+ t.Fatalf("Stop error = %v, want gateway delete failure", err)
+ }
+ if bindingRepo.bindings[188] == nil {
+ t.Fatal("binding was deleted after gateway delete failed")
+ }
+ if bindingRepo.deleteAndReleaseCalls[188] != 0 {
+ t.Fatalf("delete and release calls = %d, want 0", bindingRepo.deleteAndReleaseCalls[188])
+ }
+ state := instanceRepo.runtimeStates[188]
+ if state.status != "stopped" || state.generation != 6 {
+ t.Fatalf("runtime state = %#v, want stopped generation 6", state)
+ }
+}
+
+func TestInstanceServiceStopV2KeepsBindingWhenRuntimePodLookupFails(t *testing.T) {
+ workspacePath := "/workspaces/openclaw/user-45/instance-189"
+ instanceRepo := newV2LifecycleInstanceRepo()
+ instanceRepo.byID[189] = &models.Instance{
+ ID: 189,
+ UserID: 45,
+ Type: "openclaw",
+ RuntimeType: "gateway",
+ Status: "running",
+ WorkspacePath: &workspacePath,
+ RuntimeGeneration: 6,
+ }
+ bindingRepo := newFakeRuntimeBindingRepo()
+ bindingRepo.bindings[189] = &models.InstanceRuntimeBinding{
+ InstanceID: 189,
+ RuntimePodID: 29,
+ GatewayID: "gw-189",
+ GatewayPort: 20019,
+ State: "running",
+ Generation: 6,
+ }
+ service := &instanceService{
+ instanceRepo: instanceRepo,
+ runtimePodRepo: &fakeRuntimePodRepo{getErr: errors.New("pod lookup failed")},
+ bindingRepo: bindingRepo,
+ agentClient: &fakeRuntimeAgentClient{},
+ }
+
+ err := service.Stop(189)
+ if err == nil || !strings.Contains(err.Error(), "failed to get runtime pod") {
+ t.Fatalf("Stop error = %v, want pod lookup failure", err)
+ }
+ if bindingRepo.bindings[189] == nil {
+ t.Fatal("binding was deleted after pod lookup failed")
+ }
+ if bindingRepo.deleteAndReleaseCalls[189] != 0 {
+ t.Fatalf("delete and release calls = %d, want 0", bindingRepo.deleteAndReleaseCalls[189])
+ }
+ state := instanceRepo.runtimeStates[189]
+ if state.status != "stopped" || state.generation != 6 {
+ t.Fatalf("runtime state = %#v, want stopped generation 6", state)
+ }
+}
+
+func TestInstanceServiceRestartV2RecreatesGatewayViaScheduler(t *testing.T) {
+ workspacePath := "/workspaces/openclaw/user-45/instance-89"
+ instanceRepo := newV2LifecycleInstanceRepo()
+ instanceRepo.byID[89] = &models.Instance{
+ ID: 89,
+ UserID: 45,
+ Type: "openclaw",
+ RuntimeType: "gateway",
+ Status: "running",
+ WorkspacePath: &workspacePath,
+ RuntimeGeneration: 8,
+ }
+ endpoint := "http://agent.local:19090"
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 11: {ID: 11, AgentEndpoint: &endpoint},
+ },
+ }
+ bindingRepo := newFakeRuntimeBindingRepo()
+ bindingRepo.bindings[89] = &models.InstanceRuntimeBinding{
+ InstanceID: 89,
+ RuntimePodID: 11,
+ GatewayID: "gw-89",
+ GatewayPort: 20019,
+ State: "running",
+ Generation: 8,
+ }
+ agent := &fakeRuntimeAgentClient{}
+ service := &instanceService{
+ instanceRepo: instanceRepo,
+ runtimePodRepo: podRepo,
+ bindingRepo: bindingRepo,
+ agentClient: agent,
+ }
+
+ if err := service.Restart(89); err != nil {
+ t.Fatalf("Restart returned error: %v", err)
+ }
+
+ if len(agent.deleteRequests) != 1 || agent.deleteRequests[0].gatewayID != "gw-89" {
+ t.Fatalf("delete gateway requests = %#v", agent.deleteRequests)
+ }
+ if bindingRepo.deleteAndReleaseCalls[89] != 1 {
+ t.Fatalf("binding delete and release calls = %d, want 1", bindingRepo.deleteAndReleaseCalls[89])
+ }
+ state := instanceRepo.runtimeStates[89]
+ if state.status != "creating" || state.generation != 9 {
+ t.Fatalf("runtime state = %#v, want creating generation 9", state)
+ }
+}
+
+func TestInstanceServiceDeleteV2MissingBindingStillDeletesInstance(t *testing.T) {
+ workspacePath := "/workspaces/hermes/user-45/instance-90"
+ instanceRepo := newV2LifecycleInstanceRepo()
+ instanceRepo.byID[90] = &models.Instance{
+ ID: 90,
+ UserID: 45,
+ Type: "hermes",
+ RuntimeType: "gateway",
+ Status: "stopped",
+ WorkspacePath: &workspacePath,
+ RuntimeGeneration: 2,
+ }
+ service := &instanceService{
+ instanceRepo: instanceRepo,
+ runtimePodRepo: &fakeRuntimePodRepo{},
+ bindingRepo: newFakeRuntimeBindingRepo(),
+ agentClient: &fakeRuntimeAgentClient{},
+ }
+
+ if err := service.Delete(90); err != nil {
+ t.Fatalf("Delete returned error: %v", err)
+ }
+ if len(instanceRepo.deleted) != 1 || instanceRepo.deleted[0] != 90 {
+ t.Fatalf("deleted instances = %#v, want [90]", instanceRepo.deleted)
+ }
+}
+
+func TestInstanceServiceDeleteV2KeepsInstanceAndBindingWhenAgentDeleteFails(t *testing.T) {
+ workspacePath := "/workspaces/hermes/user-45/instance-190"
+ instanceRepo := newV2LifecycleInstanceRepo()
+ instanceRepo.byID[190] = &models.Instance{
+ ID: 190,
+ UserID: 45,
+ Type: "hermes",
+ RuntimeType: "gateway",
+ Status: "running",
+ WorkspacePath: &workspacePath,
+ RuntimeGeneration: 2,
+ }
+ endpoint := "http://agent.local:19090"
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 39: {ID: 39, AgentEndpoint: &endpoint},
+ },
+ }
+ bindingRepo := newFakeRuntimeBindingRepo()
+ bindingRepo.bindings[190] = &models.InstanceRuntimeBinding{
+ InstanceID: 190,
+ RuntimePodID: 39,
+ GatewayID: "gw-190",
+ GatewayPort: 20090,
+ State: "running",
+ Generation: 2,
+ }
+ service := &instanceService{
+ instanceRepo: instanceRepo,
+ runtimePodRepo: podRepo,
+ bindingRepo: bindingRepo,
+ agentClient: &fakeRuntimeAgentClient{deleteErr: errors.New("agent delete failed")},
+ }
+
+ err := service.Delete(190)
+ if err == nil || !strings.Contains(err.Error(), "failed to delete v2 gateway") {
+ t.Fatalf("Delete error = %v, want gateway delete failure", err)
+ }
+ if len(instanceRepo.deleted) != 0 {
+ t.Fatalf("deleted instances = %#v, want none", instanceRepo.deleted)
+ }
+ if instanceRepo.byID[190] == nil {
+ t.Fatal("instance was removed after cleanup failure")
+ }
+ if bindingRepo.bindings[190] == nil {
+ t.Fatal("binding was removed after cleanup failure")
+ }
+ if bindingRepo.deleteAndReleaseCalls[190] != 0 {
+ t.Fatalf("delete and release calls = %d, want 0", bindingRepo.deleteAndReleaseCalls[190])
+ }
+}
+
+func TestInstanceServiceV2StatusRequiresRunningBindingAndPodIP(t *testing.T) {
+ workspacePath := "/workspaces/openclaw/user-45/instance-92"
+ instanceRepo := newV2LifecycleInstanceRepo()
+ instanceRepo.byID[92] = &models.Instance{
+ ID: 92,
+ UserID: 45,
+ Type: "openclaw",
+ RuntimeType: "gateway",
+ Status: "running",
+ WorkspacePath: &workspacePath,
+ WorkspaceUsageBytes: 2048,
+ RuntimeGeneration: 1,
+ }
+ bindingRepo := newFakeRuntimeBindingRepo()
+ bindingRepo.bindings[92] = &models.InstanceRuntimeBinding{
+ InstanceID: 92,
+ RuntimePodID: 12,
+ GatewayPort: 20020,
+ State: "running",
+ }
+ service := &instanceService{
+ instanceRepo: instanceRepo,
+ runtimePodRepo: &fakeRuntimePodRepo{pods: map[int64]*models.RuntimePod{12: {ID: 12}}},
+ bindingRepo: bindingRepo,
+ }
+
+ status, err := service.GetInstanceStatus(92)
+ if err != nil {
+ t.Fatalf("GetInstanceStatus returned error: %v", err)
+ }
+ if status.Availability != "unavailable" {
+ t.Fatalf("availability without pod IP = %q, want unavailable", status.Availability)
+ }
+
+ podIP := "10.42.0.92"
+ service.runtimePodRepo = &fakeRuntimePodRepo{pods: map[int64]*models.RuntimePod{12: {ID: 12, PodIP: &podIP}}}
+ status, err = service.GetInstanceStatus(92)
+ if err != nil {
+ t.Fatalf("GetInstanceStatus returned error after pod IP: %v", err)
+ }
+ if status.Availability != "available" {
+ t.Fatalf("availability with running binding and pod IP = %q, want available", status.Availability)
+ }
+}
+
+func TestSyncInstanceSkipsV2RuntimeGatewayInstances(t *testing.T) {
+ workspacePath := "/workspaces/openclaw/user-45/instance-91"
+ instanceRepo := newV2LifecycleInstanceRepo()
+ service := &SyncService{instanceRepo: instanceRepo}
+ instance := &models.Instance{
+ ID: 91,
+ UserID: 45,
+ Type: "openclaw",
+ RuntimeType: "gateway",
+ InstanceMode: InstanceModeLite,
+ Status: "running",
+ WorkspacePath: &workspacePath,
+ }
+
+ service.syncInstance(context.Background(), instance)
+
+ if len(instanceRepo.updated) != 0 {
+ t.Fatalf("sync should not update V2 gateway instance via pod status, got %#v", instanceRepo.updated)
+ }
+}
+
+func TestV2RuntimeTypeForInstanceRequiresLiteGatewayBoundary(t *testing.T) {
+ workspacePath := "/workspaces/openclaw/user-45/instance-191"
+ lite := &models.Instance{
+ ID: 191,
+ Type: "openclaw",
+ RuntimeType: RuntimeBackendGateway,
+ InstanceMode: InstanceModeLite,
+ WorkspacePath: &workspacePath,
+ }
+ if runtimeType, ok := v2RuntimeTypeForInstance(lite); !ok || runtimeType != RuntimeTypeOpenClaw {
+ t.Fatalf("lite gateway instance was not recognized: %q %v", runtimeType, ok)
+ }
+
+ pro := &models.Instance{
+ ID: 192,
+ Type: "openclaw",
+ RuntimeType: RuntimeBackendDesktop,
+ InstanceMode: InstanceModePro,
+ WorkspacePath: &workspacePath,
+ }
+ if runtimeType, ok := v2RuntimeTypeForInstance(pro); ok {
+ t.Fatalf("pro desktop instance must not be scheduler-managed, got %q", runtimeType)
+ }
+}
+
+func TestInstanceModeCapacityCanDisablePro(t *testing.T) {
+ t.Setenv("CLAWMANAGER_PRO_CAPACITY", "0")
+ service := &instanceService{instanceRepo: newV2LifecycleInstanceRepo()}
+
+ err := service.enforceInstanceModeLimits(context.Background(), InstanceModePro, 2, 4, 20, 0)
+ if err == nil || !strings.Contains(err.Error(), "pro instance mode is disabled") {
+ t.Fatalf("capacity error = %v, want pro disabled", err)
+ }
+}
+
+func TestInstanceModeResourceLimitRejectsOversizedPro(t *testing.T) {
+ t.Setenv("CLAWMANAGER_PRO_MAX_CPU_CORES", "1.5")
+ service := &instanceService{instanceRepo: newV2LifecycleInstanceRepo()}
+
+ err := service.enforceInstanceModeLimits(context.Background(), InstanceModePro, 2, 4, 20, 0)
+ if err == nil || !strings.Contains(err.Error(), "pro CPU cores exceed mode limit") {
+ t.Fatalf("resource limit error = %v, want pro CPU limit", err)
+ }
+}
+
+func TestInstanceModeResourceLimitAllowsOversizedLite(t *testing.T) {
+ t.Setenv("CLAWMANAGER_LITE_MAX_CPU_CORES", "1.5")
+ service := &instanceService{instanceRepo: newV2LifecycleInstanceRepo()}
+
+ err := service.enforceInstanceModeLimits(context.Background(), InstanceModeLite, 2, 4, 20, 0)
+ if err != nil {
+ t.Fatalf("resource limit error = %v, want lite to skip dedicated resource limits", err)
+ }
+}
+
+type v2LifecycleInstanceRepo struct {
+ byID map[int]*models.Instance
+ created []models.Instance
+ updated []models.Instance
+ deleted []int
+ workspacePath map[int]string
+ runtimeStates map[int]v2RuntimeStateRecord
+ nextID int
+}
+
+type v2RuntimeStateRecord struct {
+ status string
+ generation int
+ message *string
+}
+
+func newV2LifecycleInstanceRepo() *v2LifecycleInstanceRepo {
+ return &v2LifecycleInstanceRepo{
+ byID: map[int]*models.Instance{},
+ workspacePath: map[int]string{},
+ runtimeStates: map[int]v2RuntimeStateRecord{},
+ nextID: 1,
+ }
+}
+
+func (r *v2LifecycleInstanceRepo) Create(instance *models.Instance) error {
+ if instance.ID == 0 {
+ instance.ID = r.nextID
+ r.nextID++
+ }
+ copy := *instance
+ r.byID[instance.ID] = ©
+ r.created = append(r.created, copy)
+ return nil
+}
+
+func (r *v2LifecycleInstanceRepo) GetByID(id int) (*models.Instance, error) {
+ return r.byID[id], nil
+}
+
+func (r *v2LifecycleInstanceRepo) GetByAccessToken(string) (*models.Instance, error) {
+ return nil, nil
+}
+
+func (r *v2LifecycleInstanceRepo) GetByAgentBootstrapToken(string) (*models.Instance, error) {
+ return nil, nil
+}
+
+func (r *v2LifecycleInstanceRepo) GetAll(offset, limit int) ([]models.Instance, error) {
+ return nil, nil
+}
+
+func (r *v2LifecycleInstanceRepo) CountAll() (int, error) { return len(r.byID), nil }
+
+func (r *v2LifecycleInstanceRepo) GetByUserID(userID int, offset, limit int) ([]models.Instance, error) {
+ instances := make([]models.Instance, 0)
+ for _, instance := range r.byID {
+ if instance.UserID == userID {
+ instances = append(instances, *instance)
+ }
+ }
+ return instances, nil
+}
+
+func (r *v2LifecycleInstanceRepo) CountByUserID(userID int) (int, error) {
+ count := 0
+ for _, instance := range r.byID {
+ if instance.UserID == userID {
+ count++
+ }
+ }
+ return count, nil
+}
+
+func (r *v2LifecycleInstanceRepo) CountActiveByMode(ctx context.Context, mode string) (int, error) {
+ count := 0
+ for _, instance := range r.byID {
+ if instance.InstanceMode == mode && (instance.Status == "creating" || instance.Status == "running") {
+ count++
+ }
+ }
+ return count, nil
+}
+
+func (r *v2LifecycleInstanceRepo) ExistsByUserIDAndName(userID int, name string) (bool, error) {
+ normalized := strings.TrimSpace(strings.ToLower(name))
+ for _, instance := range r.byID {
+ if instance.UserID == userID && strings.TrimSpace(strings.ToLower(instance.Name)) == normalized {
+ return true, nil
+ }
+ }
+ return false, nil
+}
+
+func (r *v2LifecycleInstanceRepo) GetAllRunning() ([]models.Instance, error) {
+ instances := make([]models.Instance, 0, len(r.byID))
+ for _, instance := range r.byID {
+ instances = append(instances, *instance)
+ }
+ return instances, nil
+}
+
+func (r *v2LifecycleInstanceRepo) GetV2DesiredRunning(context.Context, int) ([]models.Instance, error) {
+ return nil, nil
+}
+
+func (r *v2LifecycleInstanceRepo) GetV2Creating(context.Context, int) ([]models.Instance, error) {
+ return nil, nil
+}
+
+func (r *v2LifecycleInstanceRepo) UpdateRuntimeState(ctx context.Context, id int, status string, generation int, message *string) error {
+ instance := r.byID[id]
+ if instance != nil {
+ instance.Status = status
+ instance.RuntimeGeneration = generation
+ instance.RuntimeErrorMessage = message
+ }
+ r.runtimeStates[id] = v2RuntimeStateRecord{status: status, generation: generation, message: message}
+ return nil
+}
+
+func (r *v2LifecycleInstanceRepo) SetWorkspacePath(ctx context.Context, id int, workspacePath string) error {
+ instance := r.byID[id]
+ if instance != nil {
+ instance.WorkspacePath = &workspacePath
+ }
+ r.workspacePath[id] = workspacePath
+ return nil
+}
+
+func (r *v2LifecycleInstanceRepo) UpdateWorkspaceUsage(context.Context, int, int64) error {
+ return nil
+}
+
+func (r *v2LifecycleInstanceRepo) Update(instance *models.Instance) error {
+ copy := *instance
+ r.byID[instance.ID] = ©
+ r.updated = append(r.updated, copy)
+ return nil
+}
+
+func (r *v2LifecycleInstanceRepo) Delete(id int) error {
+ delete(r.byID, id)
+ r.deleted = append(r.deleted, id)
+ return nil
+}
+
+type v2LifecycleQuotaRepo struct{}
+
+func (v2LifecycleQuotaRepo) Create(*models.UserQuota) error { return nil }
+func (v2LifecycleQuotaRepo) GetByUserID(userID int) (*models.UserQuota, error) {
+ return &models.UserQuota{
+ UserID: userID,
+ MaxInstances: 100,
+ MaxCPUCores: 100,
+ MaxMemoryGB: 1000,
+ MaxStorageGB: 1000,
+ MaxGPUCount: 8,
+ }, nil
+}
+func (v2LifecycleQuotaRepo) Update(*models.UserQuota) error { return nil }
+func (v2LifecycleQuotaRepo) DeleteByUserID(int) error { return nil }
+func (v2LifecycleQuotaRepo) CreateDefaultQuota(userID int) (*models.UserQuota, error) {
+ return v2LifecycleQuotaRepo{}.GetByUserID(userID)
+}
+
+type fixedV2QuotaRepo struct {
+ cpu float64
+ memory int
+ storage int
+ gpu int
+}
+
+func (fixedV2QuotaRepo) Create(*models.UserQuota) error { return nil }
+func (r fixedV2QuotaRepo) GetByUserID(userID int) (*models.UserQuota, error) {
+ return &models.UserQuota{
+ UserID: userID,
+ MaxInstances: 100,
+ MaxCPUCores: r.cpu,
+ MaxMemoryGB: r.memory,
+ MaxStorageGB: r.storage,
+ MaxGPUCount: r.gpu,
+ }, nil
+}
+func (fixedV2QuotaRepo) Update(*models.UserQuota) error { return nil }
+func (fixedV2QuotaRepo) DeleteByUserID(int) error { return nil }
+func (r fixedV2QuotaRepo) CreateDefaultQuota(userID int) (*models.UserQuota, error) {
+ return r.GetByUserID(userID)
+}
diff --git a/backend/internal/services/instance_visibility_test.go b/backend/internal/services/instance_visibility_test.go
index 7c9e289..7dd9981 100644
--- a/backend/internal/services/instance_visibility_test.go
+++ b/backend/internal/services/instance_visibility_test.go
@@ -1,159 +1,178 @@
-package services
-
-import (
- "testing"
-
- "clawreef/internal/models"
-)
-
-// stubInstanceVisibilityRepo is a minimal InstanceRepository used by the
-// visibility tests below. It only implements the read paths exercised by
-// GetByUserID / GetAllInstances; every other method panics so that any
-// accidental call surfaces as a test failure instead of a silent stub.
-type stubInstanceVisibilityRepo struct {
- all []models.Instance
-}
-
-func (r *stubInstanceVisibilityRepo) byUser(userID int) []models.Instance {
- out := make([]models.Instance, 0, len(r.all))
- for _, inst := range r.all {
- if inst.UserID == userID {
- out = append(out, inst)
- }
- }
- return out
-}
-
-func (r *stubInstanceVisibilityRepo) Create(*models.Instance) error { panic("not used") }
-func (r *stubInstanceVisibilityRepo) GetByID(int) (*models.Instance, error) {
- panic("not used")
-}
-func (r *stubInstanceVisibilityRepo) GetByAccessToken(string) (*models.Instance, error) {
- panic("not used")
-}
-func (r *stubInstanceVisibilityRepo) GetByAgentBootstrapToken(string) (*models.Instance, error) {
- panic("not used")
-}
-func (r *stubInstanceVisibilityRepo) GetAll(offset, limit int) ([]models.Instance, error) {
- return paginate(r.all, offset, limit), nil
-}
-func (r *stubInstanceVisibilityRepo) CountAll() (int, error) { return len(r.all), nil }
-func (r *stubInstanceVisibilityRepo) GetByUserID(userID, offset, limit int) ([]models.Instance, error) {
- return paginate(r.byUser(userID), offset, limit), nil
-}
-func (r *stubInstanceVisibilityRepo) CountByUserID(userID int) (int, error) {
- return len(r.byUser(userID)), nil
-}
-func (r *stubInstanceVisibilityRepo) ExistsByUserIDAndName(int, string) (bool, error) {
- return false, nil
-}
-func (r *stubInstanceVisibilityRepo) GetAllRunning() ([]models.Instance, error) {
- return nil, nil
-}
-func (r *stubInstanceVisibilityRepo) Update(*models.Instance) error { panic("not used") }
-func (r *stubInstanceVisibilityRepo) Delete(int) error { panic("not used") }
-
-func paginate(items []models.Instance, offset, limit int) []models.Instance {
- if offset >= len(items) {
- return []models.Instance{}
- }
- end := offset + limit
- if end > len(items) {
- end = len(items)
- }
- out := make([]models.Instance, end-offset)
- copy(out, items[offset:end])
- return out
-}
-
-// TestGetByUserIDFiltersByCaller locks in the workspace-view contract: the
-// caller-scoped listing must return only the caller's own instances,
-// regardless of any role the caller may hold elsewhere. Regression guard
-// for the admin-cross-user-leakage bug in which role=admin widened this
-// endpoint to every user's instances.
-func TestGetByUserIDFiltersByCaller(t *testing.T) {
- t.Parallel()
-
- repo := &stubInstanceVisibilityRepo{
- all: []models.Instance{
- {ID: 1, UserID: 10, Name: "alice-1"},
- {ID: 2, UserID: 10, Name: "alice-2"},
- {ID: 3, UserID: 20, Name: "bob-1"},
- },
- }
- svc := &instanceService{instanceRepo: repo}
-
- instances, total, err := svc.GetByUserID(10, 0, 50)
- if err != nil {
- t.Fatalf("GetByUserID returned error: %v", err)
- }
- if total != 2 {
- t.Fatalf("expected total=2 for alice, got %d", total)
- }
- if len(instances) != 2 || instances[0].Name != "alice-1" || instances[1].Name != "alice-2" {
- t.Fatalf("expected alice's instances only, got %+v", instances)
- }
-
- instances, total, err = svc.GetByUserID(20, 0, 50)
- if err != nil {
- t.Fatalf("GetByUserID returned error: %v", err)
- }
- if total != 1 {
- t.Fatalf("expected total=1 for bob, got %d", total)
- }
- if len(instances) != 1 || instances[0].Name != "bob-1" {
- t.Fatalf("expected bob's instance only, got %+v", instances)
- }
-
- // A user with no instances sees an empty list, never someone else's.
- instances, total, err = svc.GetByUserID(99, 0, 50)
- if err != nil {
- t.Fatalf("GetByUserID returned error: %v", err)
- }
- if total != 0 || len(instances) != 0 {
- t.Fatalf("expected empty listing for user 99, got total=%d instances=%+v", total, instances)
- }
-}
-
-// TestGetAllInstancesReturnsEveryUser verifies the admin-console surface:
-// cross-user listing, with pagination, independent of caller identity (the
-// admin middleware gates reachability at the route layer, not here).
-func TestGetAllInstancesReturnsEveryUser(t *testing.T) {
- t.Parallel()
-
- repo := &stubInstanceVisibilityRepo{
- all: []models.Instance{
- {ID: 1, UserID: 10, Name: "alice-1"},
- {ID: 2, UserID: 10, Name: "alice-2"},
- {ID: 3, UserID: 20, Name: "bob-1"},
- },
- }
- svc := &instanceService{instanceRepo: repo}
-
- instances, total, err := svc.GetAllInstances(0, 50)
- if err != nil {
- t.Fatalf("GetAllInstances returned error: %v", err)
- }
- if total != 3 {
- t.Fatalf("expected total=3 across users, got %d", total)
- }
- if len(instances) != 3 {
- t.Fatalf("expected 3 instances, got %d", len(instances))
- }
-
- // Pagination still respected.
- page1, _, err := svc.GetAllInstances(0, 2)
- if err != nil {
- t.Fatalf("GetAllInstances page1 error: %v", err)
- }
- if len(page1) != 2 {
- t.Fatalf("expected page1 size=2, got %d", len(page1))
- }
- page2, _, err := svc.GetAllInstances(2, 2)
- if err != nil {
- t.Fatalf("GetAllInstances page2 error: %v", err)
- }
- if len(page2) != 1 || page2[0].Name != "bob-1" {
- t.Fatalf("expected page2=[bob-1], got %+v", page2)
- }
-}
+package services
+
+import (
+ "context"
+ "testing"
+
+ "clawreef/internal/models"
+)
+
+// stubInstanceVisibilityRepo is a minimal InstanceRepository used by the
+// visibility tests below. It only implements the read paths exercised by
+// GetByUserID / GetAllInstances; every other method panics so that any
+// accidental call surfaces as a test failure instead of a silent stub.
+type stubInstanceVisibilityRepo struct {
+ all []models.Instance
+}
+
+func (r *stubInstanceVisibilityRepo) byUser(userID int) []models.Instance {
+ out := make([]models.Instance, 0, len(r.all))
+ for _, inst := range r.all {
+ if inst.UserID == userID {
+ out = append(out, inst)
+ }
+ }
+ return out
+}
+
+func (r *stubInstanceVisibilityRepo) Create(*models.Instance) error { panic("not used") }
+func (r *stubInstanceVisibilityRepo) GetByID(int) (*models.Instance, error) {
+ panic("not used")
+}
+func (r *stubInstanceVisibilityRepo) GetByAccessToken(string) (*models.Instance, error) {
+ panic("not used")
+}
+func (r *stubInstanceVisibilityRepo) GetByAgentBootstrapToken(string) (*models.Instance, error) {
+ panic("not used")
+}
+func (r *stubInstanceVisibilityRepo) GetAll(offset, limit int) ([]models.Instance, error) {
+ return paginate(r.all, offset, limit), nil
+}
+func (r *stubInstanceVisibilityRepo) CountAll() (int, error) { return len(r.all), nil }
+func (r *stubInstanceVisibilityRepo) GetByUserID(userID, offset, limit int) ([]models.Instance, error) {
+ return paginate(r.byUser(userID), offset, limit), nil
+}
+func (r *stubInstanceVisibilityRepo) CountByUserID(userID int) (int, error) {
+ return len(r.byUser(userID)), nil
+}
+func (r *stubInstanceVisibilityRepo) CountActiveByMode(context.Context, string) (int, error) {
+ return 0, nil
+}
+func (r *stubInstanceVisibilityRepo) ExistsByUserIDAndName(int, string) (bool, error) {
+ return false, nil
+}
+func (r *stubInstanceVisibilityRepo) GetAllRunning() ([]models.Instance, error) {
+ return nil, nil
+}
+func (r *stubInstanceVisibilityRepo) GetV2DesiredRunning(context.Context, int) ([]models.Instance, error) {
+ panic("not used")
+}
+func (r *stubInstanceVisibilityRepo) GetV2Creating(context.Context, int) ([]models.Instance, error) {
+ panic("not used")
+}
+func (r *stubInstanceVisibilityRepo) UpdateRuntimeState(context.Context, int, string, int, *string) error {
+ panic("not used")
+}
+func (r *stubInstanceVisibilityRepo) SetWorkspacePath(context.Context, int, string) error {
+ panic("not used")
+}
+func (r *stubInstanceVisibilityRepo) UpdateWorkspaceUsage(context.Context, int, int64) error {
+ panic("not used")
+}
+func (r *stubInstanceVisibilityRepo) Update(*models.Instance) error { panic("not used") }
+func (r *stubInstanceVisibilityRepo) Delete(int) error { panic("not used") }
+
+func paginate(items []models.Instance, offset, limit int) []models.Instance {
+ if offset >= len(items) {
+ return []models.Instance{}
+ }
+ end := offset + limit
+ if end > len(items) {
+ end = len(items)
+ }
+ out := make([]models.Instance, end-offset)
+ copy(out, items[offset:end])
+ return out
+}
+
+// TestGetByUserIDFiltersByCaller locks in the workspace-view contract: the
+// caller-scoped listing must return only the caller's own instances,
+// regardless of any role the caller may hold elsewhere. Regression guard
+// for the admin-cross-user-leakage bug in which role=admin widened this
+// endpoint to every user's instances.
+func TestGetByUserIDFiltersByCaller(t *testing.T) {
+ t.Parallel()
+
+ repo := &stubInstanceVisibilityRepo{
+ all: []models.Instance{
+ {ID: 1, UserID: 10, Name: "alice-1"},
+ {ID: 2, UserID: 10, Name: "alice-2"},
+ {ID: 3, UserID: 20, Name: "bob-1"},
+ },
+ }
+ svc := &instanceService{instanceRepo: repo}
+
+ instances, total, err := svc.GetByUserID(10, 0, 50)
+ if err != nil {
+ t.Fatalf("GetByUserID returned error: %v", err)
+ }
+ if total != 2 {
+ t.Fatalf("expected total=2 for alice, got %d", total)
+ }
+ if len(instances) != 2 || instances[0].Name != "alice-1" || instances[1].Name != "alice-2" {
+ t.Fatalf("expected alice's instances only, got %+v", instances)
+ }
+
+ instances, total, err = svc.GetByUserID(20, 0, 50)
+ if err != nil {
+ t.Fatalf("GetByUserID returned error: %v", err)
+ }
+ if total != 1 {
+ t.Fatalf("expected total=1 for bob, got %d", total)
+ }
+ if len(instances) != 1 || instances[0].Name != "bob-1" {
+ t.Fatalf("expected bob's instance only, got %+v", instances)
+ }
+
+ // A user with no instances sees an empty list, never someone else's.
+ instances, total, err = svc.GetByUserID(99, 0, 50)
+ if err != nil {
+ t.Fatalf("GetByUserID returned error: %v", err)
+ }
+ if total != 0 || len(instances) != 0 {
+ t.Fatalf("expected empty listing for user 99, got total=%d instances=%+v", total, instances)
+ }
+}
+
+// TestGetAllInstancesReturnsEveryUser verifies the admin-console surface:
+// cross-user listing, with pagination, independent of caller identity (the
+// admin middleware gates reachability at the route layer, not here).
+func TestGetAllInstancesReturnsEveryUser(t *testing.T) {
+ t.Parallel()
+
+ repo := &stubInstanceVisibilityRepo{
+ all: []models.Instance{
+ {ID: 1, UserID: 10, Name: "alice-1"},
+ {ID: 2, UserID: 10, Name: "alice-2"},
+ {ID: 3, UserID: 20, Name: "bob-1"},
+ },
+ }
+ svc := &instanceService{instanceRepo: repo}
+
+ instances, total, err := svc.GetAllInstances(0, 50)
+ if err != nil {
+ t.Fatalf("GetAllInstances returned error: %v", err)
+ }
+ if total != 3 {
+ t.Fatalf("expected total=3 across users, got %d", total)
+ }
+ if len(instances) != 3 {
+ t.Fatalf("expected 3 instances, got %d", len(instances))
+ }
+
+ // Pagination still respected.
+ page1, _, err := svc.GetAllInstances(0, 2)
+ if err != nil {
+ t.Fatalf("GetAllInstances page1 error: %v", err)
+ }
+ if len(page1) != 2 {
+ t.Fatalf("expected page1 size=2, got %d", len(page1))
+ }
+ page2, _, err := svc.GetAllInstances(2, 2)
+ if err != nil {
+ t.Fatalf("GetAllInstances page2 error: %v", err)
+ }
+ if len(page2) != 1 || page2[0].Name != "bob-1" {
+ t.Fatalf("expected page2=[bob-1], got %+v", page2)
+ }
+}
diff --git a/backend/internal/services/k8s/client.go b/backend/internal/services/k8s/client.go
index 8c8bd20..23f41cd 100644
--- a/backend/internal/services/k8s/client.go
+++ b/backend/internal/services/k8s/client.go
@@ -29,12 +29,15 @@ const (
// Client wraps the Kubernetes client
type Client struct {
- Clientset kubernetes.Interface
- Config *rest.Config
- Namespace string
- StorageClass string
- HostPathPrefix string
- Mode ConnectionMode
+ Clientset kubernetes.Interface
+ Config *rest.Config
+ Namespace string
+ StorageClass string
+ HostPathPrefix string
+ WorkspaceRoot string
+ WorkspaceNFSServer string
+ WorkspaceNFSPath string
+ Mode ConnectionMode
}
var (
@@ -97,12 +100,15 @@ func Initialize(cfg *config.Config) error {
}
globalClient = &Client{
- Clientset: clientset,
- Config: restConfig,
- Namespace: cfg.GetNamespace(),
- StorageClass: cfg.GetStorageClass(),
- HostPathPrefix: cfg.GetHostPathPrefix(),
- Mode: detectedMode,
+ Clientset: clientset,
+ Config: restConfig,
+ Namespace: cfg.GetNamespace(),
+ StorageClass: cfg.GetStorageClass(),
+ HostPathPrefix: cfg.GetHostPathPrefix(),
+ WorkspaceRoot: cfg.Runtime.WorkspaceRoot,
+ WorkspaceNFSServer: cfg.Runtime.WorkspaceNFSServer,
+ WorkspaceNFSPath: cfg.Runtime.WorkspaceNFSPath,
+ Mode: detectedMode,
}
return nil
diff --git a/backend/internal/services/k8s/instance_deployment_service.go b/backend/internal/services/k8s/instance_deployment_service.go
new file mode 100644
index 0000000..12186bc
--- /dev/null
+++ b/backend/internal/services/k8s/instance_deployment_service.go
@@ -0,0 +1,376 @@
+package k8s
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "reflect"
+ "time"
+
+ appsv1 "k8s.io/api/apps/v1"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/errors"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
+)
+
+const instanceDeploymentRestartedAtAnnotation = "clawmanager.io/restarted-at"
+
+type InstanceDeploymentService struct {
+ client *Client
+ namespaceService *NamespaceService
+}
+
+func NewInstanceDeploymentService() *InstanceDeploymentService {
+ return &InstanceDeploymentService{
+ client: globalClient,
+ namespaceService: NewNamespaceService(),
+ }
+}
+
+func InstanceDeploymentName(instanceID int) string {
+ return sanitizeK8sName(fmt.Sprintf("clawreef-%d-deployment", instanceID))
+}
+
+func (s *InstanceDeploymentService) EnsureDeployment(ctx context.Context, config PodConfig, replicas int32) (*appsv1.Deployment, error) {
+ if s.client == nil {
+ return nil, fmt.Errorf("k8s client not initialized")
+ }
+ if _, err := s.namespaceService.EnsureNamespace(ctx, config.UserID); err != nil {
+ return nil, fmt.Errorf("failed to ensure namespace: %w", err)
+ }
+
+ desired := BuildInstanceDeployment(s.client, config, replicas)
+ deployments := s.client.Clientset.AppsV1().Deployments(desired.Namespace)
+ existing, err := deployments.Get(ctx, desired.Name, metav1.GetOptions{})
+ if errors.IsNotFound(err) {
+ created, createErr := deployments.Create(ctx, desired, metav1.CreateOptions{})
+ if createErr != nil {
+ return nil, fmt.Errorf("failed to create instance deployment %s/%s: %w", desired.Namespace, desired.Name, createErr)
+ }
+ return created, nil
+ }
+ if err != nil {
+ return nil, fmt.Errorf("failed to get instance deployment %s/%s: %w", desired.Namespace, desired.Name, err)
+ }
+ if !reflect.DeepEqual(existing.Spec.Selector, desired.Spec.Selector) {
+ return nil, fmt.Errorf("instance deployment %s/%s selector mismatch; delete and recreate the deployment to change immutable selector", desired.Namespace, desired.Name)
+ }
+
+ updated := existing.DeepCopy()
+ if updated.Labels == nil {
+ updated.Labels = map[string]string{}
+ }
+ for key, value := range desired.Labels {
+ updated.Labels[key] = value
+ }
+ updated.Spec.Replicas = desired.Spec.Replicas
+ updated.Spec.Template = desired.Spec.Template
+
+ result, updateErr := deployments.Update(ctx, updated, metav1.UpdateOptions{})
+ if updateErr != nil {
+ return nil, fmt.Errorf("failed to update instance deployment %s/%s: %w", desired.Namespace, desired.Name, updateErr)
+ }
+ return result, nil
+}
+
+func (s *InstanceDeploymentService) ScaleDeployment(ctx context.Context, userID, instanceID int, replicas int32) error {
+ if s.client == nil {
+ return fmt.Errorf("k8s client not initialized")
+ }
+ namespace := s.client.GetNamespace(userID)
+ name := InstanceDeploymentName(instanceID)
+ patch, err := json.Marshal(map[string]any{
+ "spec": map[string]any{"replicas": replicas},
+ })
+ if err != nil {
+ return fmt.Errorf("failed to build instance deployment scale patch: %w", err)
+ }
+ if _, err := s.client.Clientset.AppsV1().Deployments(namespace).Patch(ctx, name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}); err != nil {
+ return fmt.Errorf("failed to scale instance deployment %s/%s: %w", namespace, name, err)
+ }
+ return nil
+}
+
+func (s *InstanceDeploymentService) RestartDeployment(ctx context.Context, userID, instanceID int) error {
+ if s.client == nil {
+ return fmt.Errorf("k8s client not initialized")
+ }
+ namespace := s.client.GetNamespace(userID)
+ name := InstanceDeploymentName(instanceID)
+ patch, err := json.Marshal(map[string]any{
+ "spec": map[string]any{
+ "template": map[string]any{
+ "metadata": map[string]any{
+ "annotations": map[string]string{
+ instanceDeploymentRestartedAtAnnotation: time.Now().UTC().Format(time.RFC3339Nano),
+ },
+ },
+ },
+ },
+ })
+ if err != nil {
+ return fmt.Errorf("failed to build instance deployment restart patch: %w", err)
+ }
+ if _, err := s.client.Clientset.AppsV1().Deployments(namespace).Patch(ctx, name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}); err != nil {
+ return fmt.Errorf("failed to restart instance deployment %s/%s: %w", namespace, name, err)
+ }
+ return nil
+}
+
+func (s *InstanceDeploymentService) DeleteDeployment(ctx context.Context, userID, instanceID int) error {
+ if s.client == nil {
+ return fmt.Errorf("k8s client not initialized")
+ }
+ namespace := s.client.GetNamespace(userID)
+ name := InstanceDeploymentName(instanceID)
+ err := s.client.Clientset.AppsV1().Deployments(namespace).Delete(ctx, name, metav1.DeleteOptions{})
+ if err != nil && !errors.IsNotFound(err) {
+ return fmt.Errorf("failed to delete instance deployment %s/%s: %w", namespace, name, err)
+ }
+ return nil
+}
+
+func (s *InstanceDeploymentService) GetDeployment(ctx context.Context, userID, instanceID int) (*appsv1.Deployment, error) {
+ if s.client == nil {
+ return nil, fmt.Errorf("k8s client not initialized")
+ }
+ namespace := s.client.GetNamespace(userID)
+ name := InstanceDeploymentName(instanceID)
+ deployment, err := s.client.Clientset.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{})
+ if err != nil {
+ return nil, fmt.Errorf("failed to get instance deployment %s/%s: %w", namespace, name, err)
+ }
+ return deployment, nil
+}
+
+func (s *InstanceDeploymentService) GetActivePod(ctx context.Context, userID, instanceID int) (*corev1.Pod, error) {
+ if s.client == nil {
+ return nil, fmt.Errorf("k8s client not initialized")
+ }
+ namespace := s.client.GetNamespace(userID)
+ selector := fmt.Sprintf("instance-id=%d,app=clawreef", instanceID)
+ pods, err := s.client.Clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: selector})
+ if err != nil {
+ return nil, fmt.Errorf("failed to list instance deployment pods: %w", err)
+ }
+ if len(pods.Items) == 0 {
+ return nil, fmt.Errorf("pod not found for instance %d", instanceID)
+ }
+ for _, pod := range pods.Items {
+ if pod.DeletionTimestamp == nil && pod.Status.Phase == corev1.PodRunning && podReady(&pod) {
+ selected := pod
+ return &selected, nil
+ }
+ }
+ selected := pods.Items[0]
+ return &selected, nil
+}
+
+func BuildInstanceDeployment(client *Client, config PodConfig, replicas int32) *appsv1.Deployment {
+ namespace := client.GetNamespace(config.UserID)
+ name := InstanceDeploymentName(config.InstanceID)
+ runtimeType := normalizePodRuntimeType(config.RuntimeType)
+ labels := map[string]string{
+ "app": "clawreef",
+ "instance-id": fmt.Sprintf("%d", config.InstanceID),
+ "instance-name": config.InstanceName,
+ "user-id": fmt.Sprintf("%d", config.UserID),
+ "instance-type": config.Type,
+ "runtime-type": runtimeType,
+ "managed-by": "clawreef",
+ }
+
+ return &appsv1.Deployment{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Namespace: namespace,
+ Labels: copyStringMap(labels),
+ },
+ Spec: appsv1.DeploymentSpec{
+ Replicas: &replicas,
+ Selector: &metav1.LabelSelector{
+ MatchLabels: map[string]string{
+ "app": "clawreef",
+ "instance-id": fmt.Sprintf("%d", config.InstanceID),
+ },
+ },
+ Template: corev1.PodTemplateSpec{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: instanceDeploymentAnnotations(config),
+ Labels: labels,
+ },
+ Spec: buildInstanceDeploymentPodSpec(client, config, runtimeType),
+ },
+ },
+ }
+}
+
+func instanceDeploymentAnnotations(config PodConfig) map[string]string {
+ if config.SecurityMode == PodSecurityChromiumCompat {
+ return map[string]string{"container.apparmor.security.beta.kubernetes.io/desktop": "unconfined"}
+ }
+ return nil
+}
+
+func buildInstanceDeploymentPodSpec(client *Client, config PodConfig, runtimeType string) corev1.PodSpec {
+ pvcName := client.GetPVCName(config.InstanceID)
+ if config.ContainerPort == 0 {
+ config.ContainerPort = 3001
+ }
+ pullPolicy := config.ImagePullPolicy
+ if pullPolicy == "" {
+ pullPolicy = corev1.PullIfNotPresent
+ }
+
+ resources := corev1.ResourceRequirements{
+ Requests: corev1.ResourceList{
+ corev1.ResourceCPU: resource.MustParse(fmt.Sprintf("%g", config.CPUCores)),
+ corev1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", config.MemoryGB)),
+ },
+ Limits: corev1.ResourceList{
+ corev1.ResourceCPU: resource.MustParse(fmt.Sprintf("%g", config.CPUCores)),
+ corev1.ResourceMemory: resource.MustParse(fmt.Sprintf("%dGi", config.MemoryGB)),
+ },
+ }
+ if config.GPUEnabled && config.GPUCount > 0 {
+ resources.Limits["nvidia.com/gpu"] = resource.MustParse(fmt.Sprintf("%d", config.GPUCount))
+ resources.Requests["nvidia.com/gpu"] = resource.MustParse(fmt.Sprintf("%d", config.GPUCount))
+ }
+
+ container := corev1.Container{
+ Name: "desktop",
+ Image: config.Image,
+ ImagePullPolicy: pullPolicy,
+ Ports: []corev1.ContainerPort{{
+ ContainerPort: config.ContainerPort,
+ Name: "http",
+ }},
+ StartupProbe: &corev1.Probe{
+ ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstrFromInt32(config.ContainerPort)}},
+ FailureThreshold: 30,
+ PeriodSeconds: 5,
+ TimeoutSeconds: 2,
+ },
+ ReadinessProbe: &corev1.Probe{
+ ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstrFromInt32(config.ContainerPort)}},
+ InitialDelaySeconds: 3,
+ PeriodSeconds: 5,
+ TimeoutSeconds: 2,
+ FailureThreshold: 6,
+ },
+ LivenessProbe: &corev1.Probe{
+ ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstrFromInt32(config.ContainerPort)}},
+ InitialDelaySeconds: 15,
+ PeriodSeconds: 10,
+ TimeoutSeconds: 2,
+ FailureThreshold: 3,
+ },
+ SecurityContext: buildContainerSecurityContext(config.SecurityMode),
+ Resources: resources,
+ VolumeMounts: []corev1.VolumeMount{{
+ Name: "data",
+ MountPath: config.MountPath,
+ }},
+ Env: []corev1.EnvVar{
+ {Name: "INSTANCE_ID", Value: fmt.Sprintf("%d", config.InstanceID)},
+ {Name: "USER_ID", Value: fmt.Sprintf("%d", config.UserID)},
+ },
+ }
+ if runtimeType == "shell" {
+ container.Ports = nil
+ container.StartupProbe = nil
+ container.ReadinessProbe = nil
+ container.LivenessProbe = nil
+ }
+ for key, value := range config.ExtraEnv {
+ container.Env = append(container.Env, corev1.EnvVar{Name: key, Value: value})
+ }
+ for _, secretName := range config.EnvFromSecretNames {
+ if secretName == "" {
+ continue
+ }
+ container.EnvFrom = append(container.EnvFrom, corev1.EnvFromSource{
+ SecretRef: &corev1.SecretEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: secretName}},
+ })
+ }
+
+ spec := corev1.PodSpec{
+ RestartPolicy: corev1.RestartPolicyAlways,
+ SecurityContext: buildPodSecurityContext(config.FSGroup),
+ Containers: []corev1.Container{container},
+ Volumes: []corev1.Volume{{
+ Name: "data",
+ VolumeSource: corev1.VolumeSource{
+ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvcName},
+ },
+ }},
+ }
+
+ for _, mount := range config.ExtraPVCMounts {
+ if mount.Name == "" || mount.ClaimName == "" || mount.MountPath == "" {
+ continue
+ }
+ spec.Volumes = append(spec.Volumes, corev1.Volume{
+ Name: mount.Name,
+ VolumeSource: corev1.VolumeSource{
+ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: mount.ClaimName, ReadOnly: mount.ReadOnly},
+ },
+ })
+ spec.Containers[0].VolumeMounts = append(spec.Containers[0].VolumeMounts, corev1.VolumeMount{
+ Name: mount.Name,
+ MountPath: mount.MountPath,
+ ReadOnly: mount.ReadOnly,
+ })
+ }
+
+ for index, fix := range config.VolumeOwnershipFixes {
+ if fix.Name == "" || fix.MountPath == "" || fix.UID < 0 || fix.GID < 0 {
+ continue
+ }
+ spec.InitContainers = append(spec.InitContainers, buildVolumeOwnershipInitContainer(index, config.Image, pullPolicy, fix))
+ }
+
+ for _, mount := range config.ConfigMapFileMounts {
+ if mount.Name == "" || mount.ConfigMapName == "" || mount.Key == "" || mount.MountPath == "" {
+ continue
+ }
+ spec.Volumes = append(spec.Volumes, corev1.Volume{
+ Name: mount.Name,
+ VolumeSource: corev1.VolumeSource{
+ ConfigMap: &corev1.ConfigMapVolumeSource{
+ LocalObjectReference: corev1.LocalObjectReference{Name: mount.ConfigMapName},
+ Items: []corev1.KeyToPath{{Key: mount.Key, Path: mount.Key}},
+ },
+ },
+ })
+ volumeMount := corev1.VolumeMount{Name: mount.Name, MountPath: mount.MountPath, ReadOnly: true}
+ if !mount.AsDirectory {
+ volumeMount.SubPath = mount.Key
+ }
+ spec.Containers[0].VolumeMounts = append(spec.Containers[0].VolumeMounts, volumeMount)
+ }
+
+ if config.SHMSizeGB > 0 {
+ shmLimit := resource.MustParse(fmt.Sprintf("%dGi", config.SHMSizeGB))
+ spec.Volumes = append(spec.Volumes, corev1.Volume{
+ Name: "shm",
+ VolumeSource: corev1.VolumeSource{
+ EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory, SizeLimit: &shmLimit},
+ },
+ })
+ spec.Containers[0].VolumeMounts = append(spec.Containers[0].VolumeMounts, corev1.VolumeMount{Name: "shm", MountPath: "/dev/shm"})
+ }
+
+ return spec
+}
+
+func podReady(pod *corev1.Pod) bool {
+ for _, condition := range pod.Status.Conditions {
+ if condition.Type == corev1.PodReady {
+ return condition.Status == corev1.ConditionTrue
+ }
+ }
+ return false
+}
diff --git a/backend/internal/services/k8s/instance_deployment_service_test.go b/backend/internal/services/k8s/instance_deployment_service_test.go
new file mode 100644
index 0000000..743b3d7
--- /dev/null
+++ b/backend/internal/services/k8s/instance_deployment_service_test.go
@@ -0,0 +1,99 @@
+package k8s
+
+import (
+ "context"
+ "testing"
+
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/kubernetes/fake"
+)
+
+func TestBuildInstanceDeploymentUsesStableIdentityAndPVC(t *testing.T) {
+ client := &Client{Clientset: fake.NewSimpleClientset(), Namespace: "clawreef"}
+ deployment := BuildInstanceDeployment(client, PodConfig{
+ InstanceID: 42,
+ InstanceName: "Pro Desktop",
+ UserID: 7,
+ Type: "openclaw",
+ RuntimeType: "desktop",
+ CPUCores: 2,
+ MemoryGB: 4,
+ Image: "registry/openclaw:pro",
+ MountPath: "/config",
+ ContainerPort: 3001,
+ ImagePullPolicy: corev1.PullIfNotPresent,
+ }, 1)
+
+ if deployment.Name != "clawreef-42-deployment" {
+ t.Fatalf("deployment name = %q, want stable instance deployment name", deployment.Name)
+ }
+ if deployment.Namespace != "clawreef-user-7" {
+ t.Fatalf("namespace = %q, want user namespace", deployment.Namespace)
+ }
+ if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != 1 {
+ t.Fatalf("replicas = %#v, want 1", deployment.Spec.Replicas)
+ }
+ if got := deployment.Spec.Selector.MatchLabels["instance-id"]; got != "42" {
+ t.Fatalf("selector instance-id = %q, want 42", got)
+ }
+ template := deployment.Spec.Template
+ if got := template.Labels["runtime-type"]; got != "desktop" {
+ t.Fatalf("template runtime-type = %q, want desktop", got)
+ }
+ container := template.Spec.Containers[0]
+ if container.Name != "desktop" || container.Image != "registry/openclaw:pro" {
+ t.Fatalf("unexpected container: %#v", container)
+ }
+ if template.Spec.RestartPolicy != corev1.RestartPolicyAlways {
+ t.Fatalf("restart policy = %q, want Always", template.Spec.RestartPolicy)
+ }
+ if len(template.Spec.Volumes) == 0 || template.Spec.Volumes[0].PersistentVolumeClaim == nil {
+ t.Fatalf("expected PVC data volume, got %#v", template.Spec.Volumes)
+ }
+ if got := template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName; got != "clawreef-42-pvc" {
+ t.Fatalf("PVC name = %q, want clawreef-42-pvc", got)
+ }
+}
+
+func TestInstanceDeploymentServiceEnsureAndScale(t *testing.T) {
+ client := &Client{Clientset: fake.NewSimpleClientset(), Namespace: "clawreef"}
+ service := &InstanceDeploymentService{
+ client: client,
+ namespaceService: &NamespaceService{client: client},
+ }
+ config := PodConfig{
+ InstanceID: 43,
+ InstanceName: "Pro Desktop",
+ UserID: 8,
+ Type: "openclaw",
+ RuntimeType: "desktop",
+ CPUCores: 2,
+ MemoryGB: 4,
+ Image: "registry/openclaw:v1",
+ MountPath: "/config",
+ ContainerPort: 3001,
+ }
+
+ if _, err := service.EnsureDeployment(context.Background(), config, 1); err != nil {
+ t.Fatalf("EnsureDeployment create returned error: %v", err)
+ }
+ deployment, err := client.Clientset.AppsV1().Deployments("clawreef-user-8").Get(context.Background(), "clawreef-43-deployment", metav1.GetOptions{})
+ if err != nil {
+ t.Fatalf("get deployment: %v", err)
+ }
+ if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != 1 {
+ t.Fatalf("created replicas = %#v, want 1", deployment.Spec.Replicas)
+ }
+
+ if err := service.ScaleDeployment(context.Background(), 8, 43, 0); err != nil {
+ t.Fatalf("ScaleDeployment returned error: %v", err)
+ }
+ scaled, err := client.Clientset.AppsV1().Deployments("clawreef-user-8").Get(context.Background(), "clawreef-43-deployment", metav1.GetOptions{})
+ if err != nil {
+ t.Fatalf("get scaled deployment: %v", err)
+ }
+ if scaled.Spec.Replicas == nil || *scaled.Spec.Replicas != 0 {
+ t.Fatalf("scaled replicas = %#v, want 0", scaled.Spec.Replicas)
+ }
+}
diff --git a/backend/internal/services/k8s/pvc_service.go b/backend/internal/services/k8s/pvc_service.go
index 9f8ec0e..ac115b7 100644
--- a/backend/internal/services/k8s/pvc_service.go
+++ b/backend/internal/services/k8s/pvc_service.go
@@ -3,6 +3,10 @@ package k8s
import (
"context"
"fmt"
+ "net"
+ "os"
+ "path"
+ "path/filepath"
"sort"
"strings"
"time"
@@ -136,6 +140,12 @@ func (s *PVCService) CreateTeamSharedPVC(ctx context.Context, userID, teamID, st
storageClass = s.client.StorageClass
}
storageSize := resource.MustParse(fmt.Sprintf("%dGi", storageSizeGB))
+ useWorkspaceNFS := strings.TrimSpace(s.client.WorkspaceNFSServer) != ""
+ if useWorkspaceNFS {
+ if err := s.ensureTeamSharedWorkspaceDirectory(userID, teamID); err != nil {
+ return nil, err
+ }
+ }
pvc := &corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
@@ -165,6 +175,11 @@ func (s *PVCService) CreateTeamSharedPVC(ctx context.Context, userID, teamID, st
if errors.IsAlreadyExists(err) {
existingPVC, getErr := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{})
if getErr == nil && existingPVC != nil && existingPVC.Labels["team-id"] == fmt.Sprintf("%d", teamID) {
+ if useWorkspaceNFS {
+ if err := s.ensureWorkspaceNFSPVForTeamSharedPVC(ctx, namespace, pvcName, existingPVC, userID, teamID, storageSizeGB, storageClass, fmt.Sprintf("clawreef-pv-user-%d-team-%d-shared", userID, teamID)); err != nil {
+ return nil, err
+ }
+ }
if existingPVC.Status.Phase != corev1.ClaimBound {
go s.monitorTeamSharedPVCBinding(context.Background(), namespace, pvcName, userID, teamID, storageSizeGB, storageClass, 15*time.Second)
}
@@ -174,6 +189,11 @@ func (s *PVCService) CreateTeamSharedPVC(ctx context.Context, userID, teamID, st
return nil, fmt.Errorf("failed to create Team shared PVC %s: %w", pvcName, err)
}
+ if useWorkspaceNFS {
+ if err := s.ensureWorkspaceNFSPVForTeamSharedPVC(ctx, namespace, pvcName, createdPVC, userID, teamID, storageSizeGB, storageClass, fmt.Sprintf("clawreef-pv-user-%d-team-%d-shared", userID, teamID)); err != nil {
+ return nil, err
+ }
+ }
go s.monitorTeamSharedPVCBinding(context.Background(), namespace, pvcName, userID, teamID, storageSizeGB, storageClass, 15*time.Second)
return createdPVC, nil
}
@@ -218,11 +238,6 @@ func (s *PVCService) waitForTeamSharedPVCBinding(ctx context.Context, namespace,
func (s *PVCService) createPVForTeamSharedPVC(ctx context.Context, namespace, pvcName string, userID, teamID, storageSizeGB int, storageClass string) (*corev1.PersistentVolumeClaim, error) {
pvName := fmt.Sprintf("clawreef-pv-user-%d-team-%d-shared", userID, teamID)
- hostPathPrefix := "/data/clawreef"
- if s.client != nil && s.client.HostPathPrefix != "" {
- hostPathPrefix = s.client.HostPathPrefix
- }
- hostPath := fmt.Sprintf("%s/user-%d/team-%d-shared", hostPathPrefix, userID, teamID)
existingPV, err := s.client.Clientset.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{})
if err == nil && existingPV != nil {
@@ -245,6 +260,16 @@ func (s *PVCService) createPVForTeamSharedPVC(ctx context.Context, namespace, pv
if err != nil {
return nil, fmt.Errorf("failed to get Team shared PVC %s for UID: %w", pvcName, err)
}
+
+ if strings.TrimSpace(s.client.WorkspaceNFSServer) != "" {
+ return s.createWorkspaceNFSPVForTeamSharedPVC(ctx, namespace, pvcName, pvc, userID, teamID, storageSizeGB, storageClass, pvName)
+ }
+
+ hostPathPrefix := "/data/clawreef"
+ if s.client != nil && s.client.HostPathPrefix != "" {
+ hostPathPrefix = s.client.HostPathPrefix
+ }
+ hostPath := fmt.Sprintf("%s/user-%d/team-%d-shared", hostPathPrefix, userID, teamID)
storageSize := resource.MustParse(fmt.Sprintf("%dGi", storageSizeGB))
nodeAffinity, err := s.hostPathPVNodeAffinity(ctx)
if err != nil {
@@ -305,6 +330,183 @@ func (s *PVCService) createPVForTeamSharedPVC(ctx context.Context, namespace, pv
return pvc, nil
}
+func (s *PVCService) createWorkspaceNFSPVForTeamSharedPVC(ctx context.Context, namespace, pvcName string, pvc *corev1.PersistentVolumeClaim, userID, teamID, storageSizeGB int, storageClass, pvName string) (*corev1.PersistentVolumeClaim, error) {
+ if err := s.ensureWorkspaceNFSPVForTeamSharedPVC(ctx, namespace, pvcName, pvc, userID, teamID, storageSizeGB, storageClass, pvName); err != nil {
+ return nil, err
+ }
+
+ time.Sleep(3 * time.Second)
+ pvc, err := s.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{})
+ if err != nil {
+ return nil, fmt.Errorf("failed to get Team shared PVC after NFS PV creation: %w", err)
+ }
+ if pvc.Status.Phase != corev1.ClaimBound {
+ return nil, fmt.Errorf("Team shared PVC %s is still not bound after NFS PV creation, status: %s", pvcName, pvc.Status.Phase)
+ }
+ fmt.Printf("Team shared PVC %s successfully bound to NFS PV %s (%s:%s)\n", pvcName, pvName, s.client.WorkspaceNFSServer, teamSharedWorkspaceNFSPath(s.client.WorkspaceNFSPath, userID, teamID))
+ return pvc, nil
+}
+
+func (s *PVCService) ensureWorkspaceNFSPVForTeamSharedPVC(ctx context.Context, namespace, pvcName string, pvc *corev1.PersistentVolumeClaim, userID, teamID, storageSizeGB int, storageClass, pvName string) error {
+ if err := s.ensureTeamSharedWorkspaceDirectory(userID, teamID); err != nil {
+ return err
+ }
+
+ storageSize := resource.MustParse(fmt.Sprintf("%dGi", storageSizeGB))
+ nfsPath := teamSharedWorkspaceNFSPath(s.client.WorkspaceNFSPath, userID, teamID)
+ nfsServer, err := s.workspaceNFSServerForPV(ctx)
+ if err != nil {
+ return err
+ }
+ pv := &corev1.PersistentVolume{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: pvName,
+ Labels: map[string]string{
+ "app": "clawreef",
+ "user-id": fmt.Sprintf("%d", userID),
+ "team-id": fmt.Sprintf("%d", teamID),
+ "managed-by": "clawreef",
+ },
+ },
+ Spec: corev1.PersistentVolumeSpec{
+ Capacity: corev1.ResourceList{
+ corev1.ResourceStorage: storageSize,
+ },
+ AccessModes: []corev1.PersistentVolumeAccessMode{
+ corev1.ReadWriteMany,
+ },
+ PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain,
+ StorageClassName: storageClass,
+ PersistentVolumeSource: corev1.PersistentVolumeSource{
+ NFS: &corev1.NFSVolumeSource{
+ Server: nfsServer,
+ Path: nfsPath,
+ },
+ },
+ ClaimRef: &corev1.ObjectReference{
+ Kind: "PersistentVolumeClaim",
+ APIVersion: "v1",
+ Namespace: namespace,
+ Name: pvcName,
+ UID: pvc.UID,
+ ResourceVersion: pvc.ResourceVersion,
+ },
+ },
+ }
+ if _, err := s.client.Clientset.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}); err != nil {
+ if errors.IsAlreadyExists(err) {
+ existing, getErr := s.client.Clientset.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{})
+ if getErr != nil {
+ return fmt.Errorf("failed to get existing Team shared NFS PV %s: %w", pvName, getErr)
+ }
+ if existing.Spec.ClaimRef != nil && existing.Spec.ClaimRef.Namespace == namespace && existing.Spec.ClaimRef.Name == pvcName {
+ if existing.Spec.NFS != nil &&
+ strings.TrimSpace(existing.Spec.NFS.Server) == nfsServer &&
+ strings.TrimSpace(existing.Spec.NFS.Path) == nfsPath {
+ return nil
+ }
+ return fmt.Errorf("Team shared PV %s already exists for %s/%s but is not workspace NFS-backed", pvName, namespace, pvcName)
+ }
+ }
+ return fmt.Errorf("failed to create Team shared NFS PV %s: %w", pvName, err)
+ }
+ return nil
+}
+
+func (s *PVCService) workspaceNFSServerForPV(ctx context.Context) (string, error) {
+ server := strings.TrimSpace(s.client.WorkspaceNFSServer)
+ if server == "" || s.client.Clientset == nil {
+ return server, nil
+ }
+ serviceName, namespace, ok := workspaceNFSServiceRef(server, s.client.Namespace)
+ if !ok {
+ return server, nil
+ }
+ svc, err := s.client.Clientset.CoreV1().Services(namespace).Get(ctx, serviceName, metav1.GetOptions{})
+ if err != nil {
+ if errors.IsNotFound(err) {
+ return server, nil
+ }
+ return "", fmt.Errorf("failed to resolve workspace NFS service %s/%s: %w", namespace, serviceName, err)
+ }
+ clusterIP := strings.TrimSpace(svc.Spec.ClusterIP)
+ if clusterIP == "" || strings.EqualFold(clusterIP, corev1.ClusterIPNone) {
+ return server, nil
+ }
+ return clusterIP, nil
+}
+
+func workspaceNFSServiceRef(server, defaultNamespace string) (string, string, bool) {
+ host := strings.TrimSuffix(strings.TrimSpace(server), ".")
+ if host == "" || net.ParseIP(host) != nil {
+ return "", "", false
+ }
+ parts := strings.Split(strings.ToLower(host), ".")
+ if len(parts) == 1 {
+ namespace := strings.TrimSpace(defaultNamespace)
+ if namespace == "" {
+ namespace = "default"
+ }
+ return parts[0], namespace, true
+ }
+ if len(parts) >= 3 && parts[2] == "svc" && parts[0] != "" && parts[1] != "" {
+ return parts[0], parts[1], true
+ }
+ return "", "", false
+}
+
+func (s *PVCService) ensureTeamSharedWorkspaceDirectory(userID, teamID int) error {
+ root := strings.TrimSpace(s.client.WorkspaceRoot)
+ if root == "" {
+ return nil
+ }
+ dir := filepath.Join(root, filepath.FromSlash(TeamSharedWorkspaceRelativePath(userID, teamID)))
+ dirs := append([]string{dir}, teamSharedRuntimeSubdirectories(dir)...)
+ for _, target := range dirs {
+ if err := os.MkdirAll(target, 0o2775); err != nil {
+ return fmt.Errorf("failed to create Team shared runtime workspace directory %s: %w", target, err)
+ }
+ _ = os.Chown(target, 1000, 1000)
+ if err := os.Chmod(target, 0o2775); err != nil {
+ return fmt.Errorf("failed to chmod Team shared runtime workspace directory %s: %w", target, err)
+ }
+ }
+ return nil
+}
+
+func teamSharedRuntimeSubdirectories(root string) []string {
+ return []string{
+ filepath.Join(root, "status"),
+ filepath.Join(root, "inbox"),
+ filepath.Join(root, "results"),
+ filepath.Join(root, "tasks"),
+ }
+}
+
+func TeamSharedWorkspaceRelativePath(userID, teamID int) string {
+ return fmt.Sprintf("teams/user-%d/team-%d-shared", userID, teamID)
+}
+
+func TeamSharedWorkspacePath(workspaceRoot string, userID, teamID int) string {
+ root := strings.TrimRight(strings.TrimSpace(workspaceRoot), "/")
+ if root == "" {
+ root = "/workspaces"
+ }
+ return root + "/" + TeamSharedWorkspaceRelativePath(userID, teamID)
+}
+
+func teamSharedWorkspaceNFSPath(basePath string, userID, teamID int) string {
+ basePath = strings.TrimSpace(basePath)
+ if basePath == "" {
+ basePath = "/"
+ }
+ relativePath := TeamSharedWorkspaceRelativePath(userID, teamID)
+ if basePath == "/" {
+ return "/" + relativePath
+ }
+ return path.Join(basePath, relativePath)
+}
+
func (s *PVCService) hostPathPVNodeAffinity(ctx context.Context) (*corev1.VolumeNodeAffinity, error) {
if s == nil || s.client == nil {
return nil, fmt.Errorf("k8s client not initialized")
diff --git a/backend/internal/services/k8s/pvc_service_test.go b/backend/internal/services/k8s/pvc_service_test.go
index 0965961..764dcb0 100644
--- a/backend/internal/services/k8s/pvc_service_test.go
+++ b/backend/internal/services/k8s/pvc_service_test.go
@@ -2,10 +2,14 @@ package k8s
import (
"context"
+ "os"
+ "path/filepath"
+ "strings"
"testing"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes/fake"
)
@@ -105,6 +109,222 @@ func TestHostPathPVNodeAffinitySkipsUnschedulableAndNotReadyNodes(t *testing.T)
requireHostnameAffinity(t, affinity, "node-c-host")
}
+func TestCreatePVForTeamSharedPVCUsesWorkspaceNFSWhenConfigured(t *testing.T) {
+ ctx := context.Background()
+ workspaceRoot := t.TempDir()
+ namespace := "clawmanager-user-1"
+ pvcName := "clawreef-team-28-shared"
+ pvc := &corev1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: pvcName,
+ Namespace: namespace,
+ UID: types.UID("pvc-uid"),
+ ResourceVersion: "1",
+ },
+ Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound},
+ }
+ clientset := fake.NewSimpleClientset(pvc)
+ service := &PVCService{
+ client: &Client{
+ Clientset: clientset,
+ WorkspaceRoot: workspaceRoot,
+ WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local",
+ WorkspaceNFSPath: "/",
+ },
+ }
+
+ if _, err := service.createPVForTeamSharedPVC(ctx, namespace, pvcName, 1, 28, 10, "manual"); err != nil {
+ t.Fatalf("createPVForTeamSharedPVC returned error: %v", err)
+ }
+
+ pv, err := clientset.CoreV1().PersistentVolumes().Get(ctx, "clawreef-pv-user-1-team-28-shared", metav1.GetOptions{})
+ if err != nil {
+ t.Fatalf("expected Team shared PV: %v", err)
+ }
+ if pv.Spec.PersistentVolumeSource.NFS == nil {
+ t.Fatalf("expected NFS PV source, got %#v", pv.Spec.PersistentVolumeSource)
+ }
+ if pv.Spec.PersistentVolumeSource.NFS.Server != "workspace-store.clawmanager-system.svc.cluster.local" {
+ t.Fatalf("unexpected NFS server: %#v", pv.Spec.PersistentVolumeSource.NFS)
+ }
+ if pv.Spec.PersistentVolumeSource.NFS.Path != "/teams/user-1/team-28-shared" {
+ t.Fatalf("unexpected NFS path: %#v", pv.Spec.PersistentVolumeSource.NFS)
+ }
+ if pv.Spec.NodeAffinity != nil {
+ t.Fatalf("NFS-backed Team PV must not pin runtime sharing to one node: %#v", pv.Spec.NodeAffinity)
+ }
+ if _, err := os.Stat(filepath.Join(workspaceRoot, "teams", "user-1", "team-28-shared")); err != nil {
+ t.Fatalf("expected runtime workspace team directory to be created: %v", err)
+ }
+}
+
+func TestCreatePVForTeamSharedPVCResolvesWorkspaceServiceDNS(t *testing.T) {
+ ctx := context.Background()
+ namespace := "clawmanager-user-1"
+ pvcName := "clawreef-team-28-shared"
+ pvc := &corev1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: pvcName,
+ Namespace: namespace,
+ UID: types.UID("pvc-uid"),
+ ResourceVersion: "1",
+ },
+ Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound},
+ }
+ service := &corev1.Service{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "workspace-store",
+ Namespace: "clawmanager-system",
+ },
+ Spec: corev1.ServiceSpec{ClusterIP: "10.96.0.44"},
+ }
+ clientset := fake.NewSimpleClientset(pvc, service)
+ pvcService := &PVCService{
+ client: &Client{
+ Clientset: clientset,
+ WorkspaceRoot: t.TempDir(),
+ WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local",
+ WorkspaceNFSPath: "/",
+ },
+ }
+
+ if _, err := pvcService.createPVForTeamSharedPVC(ctx, namespace, pvcName, 1, 28, 10, "manual"); err != nil {
+ t.Fatalf("createPVForTeamSharedPVC returned error: %v", err)
+ }
+
+ pv, err := clientset.CoreV1().PersistentVolumes().Get(ctx, "clawreef-pv-user-1-team-28-shared", metav1.GetOptions{})
+ if err != nil {
+ t.Fatalf("expected Team shared PV: %v", err)
+ }
+ if pv.Spec.NFS == nil || pv.Spec.NFS.Server != "10.96.0.44" {
+ t.Fatalf("expected resolved ClusterIP NFS server, got %#v", pv.Spec.PersistentVolumeSource)
+ }
+}
+
+func TestCreateTeamSharedPVCCreatesWorkspaceDirectoryBeforeReturning(t *testing.T) {
+ ctx := context.Background()
+ workspaceRoot := t.TempDir()
+ client := &Client{
+ Clientset: fake.NewSimpleClientset(),
+ Namespace: "clawmanager",
+ StorageClass: "manual",
+ WorkspaceRoot: workspaceRoot,
+ WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local",
+ WorkspaceNFSPath: "/",
+ }
+ service := &PVCService{
+ client: client,
+ namespaceService: &NamespaceService{client: client},
+ }
+
+ if _, err := service.CreateTeamSharedPVC(ctx, 1, 28, 10, "manual"); err != nil {
+ t.Fatalf("CreateTeamSharedPVC returned error: %v", err)
+ }
+
+ if _, err := os.Stat(filepath.Join(workspaceRoot, "teams", "user-1", "team-28-shared")); err != nil {
+ t.Fatalf("expected Team shared runtime workspace directory before Lite gateways start: %v", err)
+ }
+}
+
+func TestCreateTeamSharedPVCCreatesWritableRuntimeSubdirectories(t *testing.T) {
+ ctx := context.Background()
+ workspaceRoot := t.TempDir()
+ client := &Client{
+ Clientset: fake.NewSimpleClientset(),
+ Namespace: "clawmanager",
+ StorageClass: "manual",
+ WorkspaceRoot: workspaceRoot,
+ WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local",
+ WorkspaceNFSPath: "/",
+ }
+ service := &PVCService{
+ client: client,
+ namespaceService: &NamespaceService{client: client},
+ }
+
+ if _, err := service.CreateTeamSharedPVC(ctx, 1, 28, 10, "manual"); err != nil {
+ t.Fatalf("CreateTeamSharedPVC returned error: %v", err)
+ }
+
+ for _, name := range []string{"status", "inbox", "results", "tasks"} {
+ dir := filepath.Join(workspaceRoot, "teams", "user-1", "team-28-shared", name)
+ info, err := os.Stat(dir)
+ if err != nil {
+ t.Fatalf("expected Team shared runtime subdirectory %s before gateways start: %v", name, err)
+ }
+ if !info.IsDir() {
+ t.Fatalf("expected %s to be a directory", dir)
+ }
+ }
+}
+
+func TestCreateTeamSharedPVCCreatesWorkspaceNFSPVBeforeReturning(t *testing.T) {
+ ctx := context.Background()
+ client := &Client{
+ Clientset: fake.NewSimpleClientset(),
+ Namespace: "clawmanager",
+ StorageClass: "manual",
+ WorkspaceRoot: t.TempDir(),
+ WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local",
+ WorkspaceNFSPath: "/",
+ }
+ service := &PVCService{
+ client: client,
+ namespaceService: &NamespaceService{client: client},
+ }
+
+ if _, err := service.CreateTeamSharedPVC(ctx, 1, 28, 10, "manual"); err != nil {
+ t.Fatalf("CreateTeamSharedPVC returned error: %v", err)
+ }
+
+ pv, err := client.Clientset.CoreV1().PersistentVolumes().Get(ctx, "clawreef-pv-user-1-team-28-shared", metav1.GetOptions{})
+ if err != nil {
+ t.Fatalf("expected Team shared NFS PV before Lite gateways start: %v", err)
+ }
+ if pv.Spec.NFS == nil || pv.Spec.NFS.Path != "/teams/user-1/team-28-shared" {
+ t.Fatalf("unexpected Team shared PV source: %#v", pv.Spec.PersistentVolumeSource)
+ }
+}
+
+func TestCreateTeamSharedPVCRejectsExistingNonWorkspaceNFSPV(t *testing.T) {
+ ctx := context.Background()
+ namespace := "clawmanager-user-1"
+ pvcName := "clawreef-team-28-shared"
+ pvc := &corev1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: pvcName,
+ Namespace: namespace,
+ Labels: map[string]string{"team-id": "28"},
+ },
+ }
+ pv := &corev1.PersistentVolume{
+ ObjectMeta: metav1.ObjectMeta{Name: "clawreef-pv-user-1-team-28-shared"},
+ Spec: corev1.PersistentVolumeSpec{
+ ClaimRef: &corev1.ObjectReference{Namespace: namespace, Name: pvcName},
+ PersistentVolumeSource: corev1.PersistentVolumeSource{
+ HostPath: &corev1.HostPathVolumeSource{Path: "/data/clawreef/user-1/team-28-shared"},
+ },
+ },
+ }
+ client := &Client{
+ Clientset: fake.NewSimpleClientset(pvc, pv),
+ Namespace: "clawmanager",
+ StorageClass: "manual",
+ WorkspaceRoot: t.TempDir(),
+ WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local",
+ WorkspaceNFSPath: "/",
+ }
+ service := &PVCService{
+ client: client,
+ namespaceService: &NamespaceService{client: client},
+ }
+
+ _, err := service.CreateTeamSharedPVC(ctx, 1, 28, 10, "manual")
+ if err == nil || !strings.Contains(err.Error(), "workspace NFS") {
+ t.Fatalf("expected existing hostPath Team PV to be rejected, got %v", err)
+ }
+}
+
func TestHostPathPVNodeAffinitySkipsHardTaintedNodes(t *testing.T) {
service := &PVCService{
client: &Client{
diff --git a/backend/internal/services/k8s/runtime_deployment_service.go b/backend/internal/services/k8s/runtime_deployment_service.go
new file mode 100644
index 0000000..0e98c94
--- /dev/null
+++ b/backend/internal/services/k8s/runtime_deployment_service.go
@@ -0,0 +1,454 @@
+package k8s
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "reflect"
+ "strconv"
+ "strings"
+
+ appsv1 "k8s.io/api/apps/v1"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/errors"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/labels"
+ "k8s.io/apimachinery/pkg/types"
+ "k8s.io/apimachinery/pkg/util/intstr"
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/client-go/util/retry"
+)
+
+const (
+ runtimeAgentPort int32 = 19090
+ runtimeWorkspaceVolume = "workspaces"
+ defaultWorkspaceMount = "/workspaces"
+ defaultGatewayPortStart = 20000
+ defaultGatewayPortEnd = 20099
+ hermesTUIDir = "/usr/local/lib/hermes-agent/ui-tui"
+)
+
+type RuntimeDeploymentSpec struct {
+ Name string
+ Namespace string
+ RuntimeType string
+ Image string
+ Replicas int32
+ WorkspaceNFSServer string
+ WorkspaceNFSPath string
+ WorkspaceMountPath string
+ GatewayPortStart int
+ GatewayPortEnd int
+ AgentControlToken string
+ AgentReportToken string
+ BackendURL string
+ TrustedProxyCIDRs string
+}
+
+type RuntimeDeploymentPod struct {
+ RuntimeType string
+ Namespace string
+ DeploymentName string
+ PodName string
+ PodIP *string
+ NodeName *string
+ ImageRef string
+ State string
+}
+
+type RuntimeDeploymentService interface {
+ Ensure(ctx context.Context, spec RuntimeDeploymentSpec) error
+ Scale(ctx context.Context, namespace, name string, replicas int32) error
+ RolloutImage(ctx context.Context, namespace, name, image string, maxUnavailable, maxSurge int) error
+ ListPods(ctx context.Context, namespace, runtimeType string) ([]RuntimeDeploymentPod, error)
+}
+
+type runtimeDeploymentService struct {
+ client kubernetes.Interface
+}
+
+func NewRuntimeDeploymentService(client kubernetes.Interface) RuntimeDeploymentService {
+ return &runtimeDeploymentService{client: client}
+}
+
+func BuildRuntimeDeployment(spec RuntimeDeploymentSpec) *appsv1.Deployment {
+ labels := map[string]string{
+ "app": spec.Name,
+ "clawmanager.io/runtime-type": spec.RuntimeType,
+ }
+ replicas := spec.Replicas
+ workspaceMountPath := runtimeWorkspaceMountPath(spec.WorkspaceMountPath)
+ gatewayPortStart := runtimeGatewayPortValue(spec.GatewayPortStart, defaultGatewayPortStart)
+ gatewayPortEnd := runtimeGatewayPortValue(spec.GatewayPortEnd, defaultGatewayPortEnd)
+ env := buildRuntimeAgentEnv(spec, workspaceMountPath, gatewayPortStart, gatewayPortEnd)
+
+ return &appsv1.Deployment{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: spec.Name,
+ Namespace: spec.Namespace,
+ Labels: copyStringMap(labels),
+ },
+ Spec: appsv1.DeploymentSpec{
+ Replicas: &replicas,
+ Selector: &metav1.LabelSelector{
+ MatchLabels: copyStringMap(labels),
+ },
+ Template: corev1.PodTemplateSpec{
+ ObjectMeta: metav1.ObjectMeta{
+ Labels: copyStringMap(labels),
+ },
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{
+ {
+ Name: "runtime",
+ Image: spec.Image,
+ Ports: []corev1.ContainerPort{
+ {
+ Name: "agent",
+ ContainerPort: runtimeAgentPort,
+ },
+ },
+ Env: env,
+ Resources: corev1.ResourceRequirements{
+ Requests: corev1.ResourceList{
+ corev1.ResourceCPU: resource.MustParse("500m"),
+ corev1.ResourceMemory: resource.MustParse("1Gi"),
+ },
+ },
+ VolumeMounts: []corev1.VolumeMount{
+ {
+ Name: runtimeWorkspaceVolume,
+ MountPath: workspaceMountPath,
+ },
+ },
+ },
+ },
+ Volumes: []corev1.Volume{
+ {
+ Name: runtimeWorkspaceVolume,
+ VolumeSource: corev1.VolumeSource{
+ NFS: &corev1.NFSVolumeSource{
+ Server: spec.WorkspaceNFSServer,
+ Path: spec.WorkspaceNFSPath,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ }
+}
+
+func buildRuntimeAgentEnv(spec RuntimeDeploymentSpec, workspaceRoot string, gatewayPortStart, gatewayPortEnd int) []corev1.EnvVar {
+ env := []corev1.EnvVar{
+ {Name: "CLAWMANAGER_RUNTIME_TYPE", Value: spec.RuntimeType},
+ {Name: "CLAWMANAGER_BACKEND_URL", Value: runtimeBackendURL(spec)},
+ {Name: "CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME", Value: spec.Name},
+ {Name: "CLAWMANAGER_RUNTIME_IMAGE_REF", Value: spec.Image},
+ {Name: "CLAWMANAGER_AGENT_PORT", Value: "19090"},
+ {Name: "CLAWMANAGER_GATEWAY_PORT_START", Value: strconv.Itoa(gatewayPortStart)},
+ {Name: "CLAWMANAGER_GATEWAY_PORT_END", Value: strconv.Itoa(gatewayPortEnd)},
+ {Name: "CLAWMANAGER_AGENT_CONTROL_TOKEN", Value: spec.AgentControlToken},
+ {Name: "CLAWMANAGER_AGENT_REPORT_TOKEN", Value: spec.AgentReportToken},
+ {Name: "RUNTIME_WORKSPACE_ROOT", Value: workspaceRoot},
+ {Name: "RUNTIME_AGENT_LISTEN_ADDR", Value: "0.0.0.0:19090"},
+ {Name: "RUNTIME_AGENT_PUBLIC_PORT", Value: "19090"},
+ {Name: "RUNTIME_AGENT_CONTROL_TOKEN", Value: spec.AgentControlToken},
+ {Name: "RUNTIME_AGENT_REPORT_TOKEN", Value: spec.AgentReportToken},
+ {Name: "RUNTIME_GATEWAY_PORT_START", Value: strconv.Itoa(gatewayPortStart)},
+ {Name: "RUNTIME_GATEWAY_PORT_END", Value: strconv.Itoa(gatewayPortEnd)},
+ }
+ if strings.EqualFold(spec.RuntimeType, "hermes") {
+ env = append(env, corev1.EnvVar{Name: "HERMES_TUI_DIR", Value: hermesTUIDir})
+ }
+ env = append(env,
+ fieldRefEnv("POD_NAME", "metadata.name"),
+ fieldRefEnv("POD_NAMESPACE", "metadata.namespace"),
+ fieldRefEnv("POD_IP", "status.podIP"),
+ fieldRefEnv("NODE_NAME", "spec.nodeName"),
+ )
+ if spec.TrustedProxyCIDRs != "" {
+ env = append(env, corev1.EnvVar{Name: "CLAWMANAGER_TRUSTED_PROXY_CIDRS", Value: spec.TrustedProxyCIDRs})
+ }
+ return env
+}
+
+func runtimeBackendURL(spec RuntimeDeploymentSpec) string {
+ if spec.BackendURL != "" {
+ return spec.BackendURL
+ }
+ namespace := spec.Namespace
+ if namespace == "" {
+ namespace = "clawmanager-system"
+ }
+ return fmt.Sprintf("http://clawmanager-gateway.%s.svc.cluster.local:9001", namespace)
+}
+
+func fieldRefEnv(name, fieldPath string) corev1.EnvVar {
+ return corev1.EnvVar{
+ Name: name,
+ ValueFrom: &corev1.EnvVarSource{
+ FieldRef: &corev1.ObjectFieldSelector{FieldPath: fieldPath},
+ },
+ }
+}
+
+func runtimeWorkspaceMountPath(value string) string {
+ if value == "" {
+ return defaultWorkspaceMount
+ }
+ return value
+}
+
+func runtimeGatewayPortValue(value, defaultValue int) int {
+ if value == 0 {
+ return defaultValue
+ }
+ return value
+}
+
+func (s *runtimeDeploymentService) Ensure(ctx context.Context, spec RuntimeDeploymentSpec) error {
+ if s == nil || s.client == nil {
+ return fmt.Errorf("k8s client not initialized")
+ }
+
+ desired := BuildRuntimeDeployment(spec)
+ deployments := s.client.AppsV1().Deployments(spec.Namespace)
+ existing, err := deployments.Get(ctx, spec.Name, metav1.GetOptions{})
+ if errors.IsNotFound(err) {
+ _, createErr := deployments.Create(ctx, desired, metav1.CreateOptions{})
+ if createErr != nil {
+ return fmt.Errorf("failed to create runtime deployment %s/%s: %w", spec.Namespace, spec.Name, createErr)
+ }
+ return nil
+ }
+ if err != nil {
+ return fmt.Errorf("failed to get runtime deployment %s/%s: %w", spec.Namespace, spec.Name, err)
+ }
+
+ if !reflect.DeepEqual(existing.Spec.Selector, desired.Spec.Selector) {
+ return fmt.Errorf("runtime deployment %s/%s selector mismatch; delete and recreate the deployment to change immutable selector", spec.Namespace, spec.Name)
+ }
+
+ updated := existing.DeepCopy()
+ if updated.Labels == nil {
+ updated.Labels = map[string]string{}
+ }
+ for key, value := range desired.Labels {
+ updated.Labels[key] = value
+ }
+ updated.Spec.Replicas = desired.Spec.Replicas
+ if updated.Spec.Template.Labels == nil {
+ updated.Spec.Template.Labels = map[string]string{}
+ }
+ for key, value := range desired.Spec.Template.Labels {
+ updated.Spec.Template.Labels[key] = value
+ }
+ updated.Spec.Template.Spec = desired.Spec.Template.Spec
+
+ _, err = deployments.Update(ctx, updated, metav1.UpdateOptions{})
+ if err != nil {
+ return fmt.Errorf("failed to update runtime deployment %s/%s: %w", spec.Namespace, spec.Name, err)
+ }
+ return nil
+}
+
+func (s *runtimeDeploymentService) Scale(ctx context.Context, namespace, name string, replicas int32) error {
+ if s == nil || s.client == nil {
+ return fmt.Errorf("k8s client not initialized")
+ }
+
+ deployments := s.client.AppsV1().Deployments(namespace)
+ patch, err := json.Marshal(map[string]interface{}{
+ "spec": map[string]interface{}{
+ "replicas": replicas,
+ },
+ })
+ if err != nil {
+ return fmt.Errorf("failed to build runtime deployment scale patch %s/%s: %w", namespace, name, err)
+ }
+ if _, err := deployments.Patch(ctx, name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}); err != nil {
+ return fmt.Errorf("failed to scale runtime deployment %s/%s: %w", namespace, name, err)
+ }
+ return nil
+}
+
+func (s *runtimeDeploymentService) RolloutImage(ctx context.Context, namespace, name, image string, maxUnavailable, maxSurge int) error {
+ if s == nil || s.client == nil {
+ return fmt.Errorf("k8s client not initialized")
+ }
+ image = strings.TrimSpace(image)
+ if image == "" {
+ return fmt.Errorf("runtime rollout image is required")
+ }
+
+ deployments := s.client.AppsV1().Deployments(namespace)
+ err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+ existing, err := deployments.Get(ctx, name, metav1.GetOptions{})
+ if err != nil {
+ return err
+ }
+
+ updated := existing.DeepCopy()
+ containerIndex := -1
+ for index, container := range updated.Spec.Template.Spec.Containers {
+ if container.Name == "runtime" {
+ containerIndex = index
+ break
+ }
+ }
+ if containerIndex < 0 {
+ return fmt.Errorf("runtime deployment %s/%s has no runtime container", namespace, name)
+ }
+
+ container := &updated.Spec.Template.Spec.Containers[containerIndex]
+ container.Image = image
+ upsertEnvVar(container, "CLAWMANAGER_RUNTIME_IMAGE_REF", image)
+
+ updated.Spec.Strategy.Type = appsv1.RollingUpdateDeploymentStrategyType
+ updated.Spec.Strategy.RollingUpdate = &appsv1.RollingUpdateDeployment{
+ MaxUnavailable: intOrStringPtr(positiveRolloutInt(maxUnavailable)),
+ MaxSurge: intOrStringPtr(positiveRolloutInt(maxSurge)),
+ }
+
+ _, err = deployments.Update(ctx, updated, metav1.UpdateOptions{})
+ return err
+ })
+ if err != nil {
+ return fmt.Errorf("failed to update runtime deployment %s/%s image: %w", namespace, name, err)
+ }
+ return nil
+}
+
+func (s *runtimeDeploymentService) ListPods(ctx context.Context, namespace, runtimeType string) ([]RuntimeDeploymentPod, error) {
+ if s == nil || s.client == nil {
+ return nil, fmt.Errorf("k8s client not initialized")
+ }
+ namespace = strings.TrimSpace(namespace)
+ if namespace == "" {
+ return nil, fmt.Errorf("runtime namespace is required")
+ }
+ runtimeType = strings.ToLower(strings.TrimSpace(runtimeType))
+
+ listOptions := metav1.ListOptions{}
+ if runtimeType != "" {
+ listOptions.LabelSelector = labels.Set{"clawmanager.io/runtime-type": runtimeType}.String()
+ }
+ deploymentList, err := s.client.AppsV1().Deployments(namespace).List(ctx, listOptions)
+ if err != nil {
+ return nil, fmt.Errorf("failed to list runtime deployments in %s: %w", namespace, err)
+ }
+
+ var pods []RuntimeDeploymentPod
+ for _, deployment := range deploymentList.Items {
+ deploymentRuntimeType := strings.ToLower(strings.TrimSpace(deployment.Labels["clawmanager.io/runtime-type"]))
+ if deploymentRuntimeType == "" {
+ continue
+ }
+ if runtimeType != "" && deploymentRuntimeType != runtimeType {
+ continue
+ }
+ if deployment.Spec.Selector == nil {
+ return nil, fmt.Errorf("runtime deployment %s/%s has no selector", deployment.Namespace, deployment.Name)
+ }
+ selector, err := metav1.LabelSelectorAsSelector(deployment.Spec.Selector)
+ if err != nil {
+ return nil, fmt.Errorf("runtime deployment %s/%s has invalid selector: %w", deployment.Namespace, deployment.Name, err)
+ }
+ podList, err := s.client.CoreV1().Pods(deployment.Namespace).List(ctx, metav1.ListOptions{LabelSelector: selector.String()})
+ if err != nil {
+ return nil, fmt.Errorf("failed to list runtime deployment pods %s/%s: %w", deployment.Namespace, deployment.Name, err)
+ }
+ deploymentImage := runtimeContainerImage(deployment.Spec.Template.Spec.Containers)
+ for _, pod := range podList.Items {
+ image := runtimeContainerImage(pod.Spec.Containers)
+ if image == "" {
+ image = deploymentImage
+ }
+ pods = append(pods, RuntimeDeploymentPod{
+ RuntimeType: deploymentRuntimeType,
+ Namespace: pod.Namespace,
+ DeploymentName: deployment.Name,
+ PodName: pod.Name,
+ PodIP: stringPtrIfNotEmpty(pod.Status.PodIP),
+ NodeName: stringPtrIfNotEmpty(pod.Spec.NodeName),
+ ImageRef: image,
+ State: runtimeK8sPodState(pod),
+ })
+ }
+ }
+ return pods, nil
+}
+
+func runtimeContainerImage(containers []corev1.Container) string {
+ for _, container := range containers {
+ if container.Name == "runtime" {
+ return strings.TrimSpace(container.Image)
+ }
+ }
+ if len(containers) == 1 {
+ return strings.TrimSpace(containers[0].Image)
+ }
+ return ""
+}
+
+func runtimeK8sPodState(pod corev1.Pod) string {
+ if pod.DeletionTimestamp != nil {
+ return "deleted"
+ }
+ switch pod.Status.Phase {
+ case corev1.PodPending:
+ return "pending"
+ case corev1.PodRunning:
+ if runtimeK8sPodReady(pod) {
+ return "ready"
+ }
+ return "pending"
+ case corev1.PodSucceeded, corev1.PodFailed:
+ return "unhealthy"
+ default:
+ return "pending"
+ }
+}
+
+func runtimeK8sPodReady(pod corev1.Pod) bool {
+ for _, condition := range pod.Status.Conditions {
+ if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue {
+ return true
+ }
+ }
+ return false
+}
+
+func stringPtrIfNotEmpty(value string) *string {
+ value = strings.TrimSpace(value)
+ if value == "" {
+ return nil
+ }
+ return &value
+}
+
+func upsertEnvVar(container *corev1.Container, name, value string) {
+ for index := range container.Env {
+ if container.Env[index].Name == name {
+ container.Env[index].Value = value
+ container.Env[index].ValueFrom = nil
+ return
+ }
+ }
+ container.Env = append(container.Env, corev1.EnvVar{Name: name, Value: value})
+}
+
+func positiveRolloutInt(value int) int {
+ if value <= 0 {
+ return 1
+ }
+ return value
+}
+
+func intOrStringPtr(value int) *intstr.IntOrString {
+ result := intstr.FromInt(value)
+ return &result
+}
diff --git a/backend/internal/services/k8s/runtime_deployment_service_test.go b/backend/internal/services/k8s/runtime_deployment_service_test.go
new file mode 100644
index 0000000..0cced48
--- /dev/null
+++ b/backend/internal/services/k8s/runtime_deployment_service_test.go
@@ -0,0 +1,481 @@
+package k8s
+
+import (
+ "context"
+ "fmt"
+ "testing"
+
+ appsv1 "k8s.io/api/apps/v1"
+ corev1 "k8s.io/api/core/v1"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/runtime/schema"
+ "k8s.io/client-go/kubernetes/fake"
+ k8stesting "k8s.io/client-go/testing"
+)
+
+func TestBuildRuntimeDeployment(t *testing.T) {
+ deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{
+ Name: "runtime-openclaw",
+ Namespace: "runtime-system",
+ RuntimeType: "openclaw",
+ Image: "registry/openclaw:latest",
+ Replicas: 2,
+ WorkspaceNFSServer: "10.0.0.9",
+ WorkspaceNFSPath: "/exports/workspaces",
+ AgentControlToken: "control-secret",
+ AgentReportToken: "report-secret",
+ })
+
+ if deployment.Name != "runtime-openclaw" || deployment.Namespace != "runtime-system" {
+ t.Fatalf("unexpected deployment identity: %s/%s", deployment.Namespace, deployment.Name)
+ }
+ if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != 2 {
+ t.Fatalf("expected replicas 2, got %#v", deployment.Spec.Replicas)
+ }
+ if got := deployment.Labels["clawmanager.io/runtime-type"]; got != "openclaw" {
+ t.Fatalf("expected runtime-type label openclaw, got %q", got)
+ }
+
+ if len(deployment.Spec.Template.Spec.Containers) != 1 {
+ t.Fatalf("expected one container, got %d", len(deployment.Spec.Template.Spec.Containers))
+ }
+ container := deployment.Spec.Template.Spec.Containers[0]
+ if container.Name != "runtime" || container.Image != "registry/openclaw:latest" {
+ t.Fatalf("unexpected runtime container: %#v", container)
+ }
+ requireContainerPort(t, container, "agent", 19090)
+ requireVolumeMount(t, container, "workspaces", "/workspaces")
+ requireEnv(t, container, "CLAWMANAGER_RUNTIME_TYPE", "openclaw")
+ requireEnv(t, container, "CLAWMANAGER_AGENT_PORT", "19090")
+ requireEnv(t, container, "CLAWMANAGER_GATEWAY_PORT_START", "20000")
+ requireEnv(t, container, "CLAWMANAGER_GATEWAY_PORT_END", "20099")
+ requireEnv(t, container, "CLAWMANAGER_AGENT_CONTROL_TOKEN", "control-secret")
+ requireEnv(t, container, "CLAWMANAGER_AGENT_REPORT_TOKEN", "report-secret")
+
+ if got := container.Resources.Requests[corev1.ResourceCPU]; got.Cmp(resource.MustParse("500m")) != 0 {
+ t.Fatalf("expected CPU request 500m, got %s", got.String())
+ }
+ if got := container.Resources.Requests[corev1.ResourceMemory]; got.Cmp(resource.MustParse("1Gi")) != 0 {
+ t.Fatalf("expected memory request 1Gi, got %s", got.String())
+ }
+
+ if len(deployment.Spec.Template.Spec.Volumes) != 1 {
+ t.Fatalf("expected one volume, got %#v", deployment.Spec.Template.Spec.Volumes)
+ }
+ volume := deployment.Spec.Template.Spec.Volumes[0]
+ if volume.Name != "workspaces" || volume.NFS == nil {
+ t.Fatalf("expected workspaces NFS volume, got %#v", volume)
+ }
+ if volume.NFS.Server != "10.0.0.9" || volume.NFS.Path != "/exports/workspaces" {
+ t.Fatalf("unexpected NFS source: %#v", volume.NFS)
+ }
+}
+
+func TestBuildRuntimeDeploymentKeepsCapacityLimitOutOfRuntimeEnv(t *testing.T) {
+ deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{
+ Name: "runtime-openclaw",
+ Namespace: "runtime-system",
+ RuntimeType: "openclaw",
+ Image: "registry/openclaw:latest",
+ Replicas: 2,
+ WorkspaceNFSServer: "10.0.0.9",
+ WorkspaceNFSPath: "/exports/workspaces",
+ WorkspaceMountPath: "/runtime-workspaces",
+ GatewayPortStart: 21000,
+ GatewayPortEnd: 21049,
+ })
+
+ container := deployment.Spec.Template.Spec.Containers[0]
+ requireVolumeMount(t, container, "workspaces", "/runtime-workspaces")
+ requireEnv(t, container, "CLAWMANAGER_GATEWAY_PORT_START", "21000")
+ requireEnv(t, container, "CLAWMANAGER_GATEWAY_PORT_END", "21049")
+ requireEnvAbsent(t, container, "CLAWMANAGER_MAX_GATEWAYS_PER_POD")
+ requireEnvAbsent(t, container, "RUNTIME_MAX_GATEWAYS_PER_POD")
+}
+
+func TestBuildRuntimeDeploymentInjectsAgentV2Environment(t *testing.T) {
+ deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{
+ Name: "runtime-hermes",
+ Namespace: "runtime-system",
+ RuntimeType: "hermes",
+ Image: "registry/hermes:v2",
+ Replicas: 1,
+ WorkspaceNFSServer: "10.0.0.9",
+ WorkspaceNFSPath: "/exports/workspaces",
+ WorkspaceMountPath: "/workspaces",
+ GatewayPortStart: 21000,
+ GatewayPortEnd: 21099,
+ AgentControlToken: "control-secret",
+ AgentReportToken: "report-secret",
+ BackendURL: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001",
+ TrustedProxyCIDRs: "10.42.0.0/16,10.43.0.0/16",
+ })
+
+ container := deployment.Spec.Template.Spec.Containers[0]
+ requireEnv(t, container, "CLAWMANAGER_RUNTIME_TYPE", "hermes")
+ requireEnv(t, container, "CLAWMANAGER_BACKEND_URL", "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001")
+ requireEnv(t, container, "CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME", "runtime-hermes")
+ requireEnv(t, container, "CLAWMANAGER_RUNTIME_IMAGE_REF", "registry/hermes:v2")
+ requireEnv(t, container, "RUNTIME_WORKSPACE_ROOT", "/workspaces")
+ requireEnv(t, container, "RUNTIME_AGENT_LISTEN_ADDR", "0.0.0.0:19090")
+ requireEnv(t, container, "RUNTIME_AGENT_PUBLIC_PORT", "19090")
+ requireEnv(t, container, "RUNTIME_AGENT_CONTROL_TOKEN", "control-secret")
+ requireEnv(t, container, "RUNTIME_AGENT_REPORT_TOKEN", "report-secret")
+ requireEnv(t, container, "RUNTIME_GATEWAY_PORT_START", "21000")
+ requireEnv(t, container, "RUNTIME_GATEWAY_PORT_END", "21099")
+ requireEnv(t, container, "HERMES_TUI_DIR", "/usr/local/lib/hermes-agent/ui-tui")
+ requireEnv(t, container, "CLAWMANAGER_TRUSTED_PROXY_CIDRS", "10.42.0.0/16,10.43.0.0/16")
+ requireEnvFieldRef(t, container, "POD_NAME", "metadata.name")
+ requireEnvFieldRef(t, container, "POD_NAMESPACE", "metadata.namespace")
+ requireEnvFieldRef(t, container, "POD_IP", "status.podIP")
+ requireEnvFieldRef(t, container, "NODE_NAME", "spec.nodeName")
+}
+
+func TestRuntimeDeploymentServiceEnsureCreatesAndUpdates(t *testing.T) {
+ client := fake.NewSimpleClientset()
+ service := NewRuntimeDeploymentService(client)
+ spec := RuntimeDeploymentSpec{
+ Name: "runtime-hermes",
+ Namespace: "runtime-system",
+ RuntimeType: "hermes",
+ Image: "registry/hermes:v1",
+ Replicas: 1,
+ WorkspaceNFSServer: "nfs.local",
+ WorkspaceNFSPath: "/exports",
+ }
+
+ if err := service.Ensure(context.Background(), spec); err != nil {
+ t.Fatalf("Ensure create returned error: %v", err)
+ }
+ created, err := client.AppsV1().Deployments(spec.Namespace).Get(context.Background(), spec.Name, metav1.GetOptions{})
+ if err != nil {
+ t.Fatalf("failed to get created deployment: %v", err)
+ }
+ if created.Spec.Replicas == nil || *created.Spec.Replicas != 1 {
+ t.Fatalf("expected created replicas 1, got %#v", created.Spec.Replicas)
+ }
+
+ spec.Image = "registry/hermes:v2"
+ spec.Replicas = 3
+ if err := service.Ensure(context.Background(), spec); err != nil {
+ t.Fatalf("Ensure update returned error: %v", err)
+ }
+ updated, err := client.AppsV1().Deployments(spec.Namespace).Get(context.Background(), spec.Name, metav1.GetOptions{})
+ if err != nil {
+ t.Fatalf("failed to get updated deployment: %v", err)
+ }
+ if updated.Spec.Template.Spec.Containers[0].Image != "registry/hermes:v2" {
+ t.Fatalf("expected updated image, got %q", updated.Spec.Template.Spec.Containers[0].Image)
+ }
+ if updated.Spec.Replicas == nil || *updated.Spec.Replicas != 3 {
+ t.Fatalf("expected updated replicas 3, got %#v", updated.Spec.Replicas)
+ }
+}
+
+func TestRuntimeDeploymentServiceEnsurePreservesUnownedMetadata(t *testing.T) {
+ existing := BuildRuntimeDeployment(RuntimeDeploymentSpec{
+ Name: "runtime-openclaw",
+ Namespace: "runtime-system",
+ RuntimeType: "openclaw",
+ Image: "registry/openclaw:v1",
+ Replicas: 1,
+ WorkspaceNFSServer: "nfs.local",
+ WorkspaceNFSPath: "/exports",
+ })
+ existing.Labels["admission.example.com/injected"] = "keep"
+ existing.Annotations = map[string]string{"admission.example.com/sidecar": "enabled"}
+ existing.Finalizers = []string{"protect.example.com/runtime"}
+ existing.Spec.Template.Labels["admission.example.com/template-label"] = "keep"
+ existing.Spec.Template.Annotations = map[string]string{"admission.example.com/template-annotation": "enabled"}
+ client := fake.NewSimpleClientset(existing)
+ service := NewRuntimeDeploymentService(client)
+
+ if err := service.Ensure(context.Background(), RuntimeDeploymentSpec{
+ Name: "runtime-openclaw",
+ Namespace: "runtime-system",
+ RuntimeType: "openclaw",
+ Image: "registry/openclaw:v2",
+ Replicas: 3,
+ WorkspaceNFSServer: "nfs.local",
+ WorkspaceNFSPath: "/exports",
+ }); err != nil {
+ t.Fatalf("Ensure update returned error: %v", err)
+ }
+
+ updated, err := client.AppsV1().Deployments("runtime-system").Get(context.Background(), "runtime-openclaw", metav1.GetOptions{})
+ if err != nil {
+ t.Fatalf("failed to get updated deployment: %v", err)
+ }
+ if updated.Labels["admission.example.com/injected"] != "keep" {
+ t.Fatalf("expected unrelated label to survive, got %#v", updated.Labels)
+ }
+ if updated.Annotations["admission.example.com/sidecar"] != "enabled" {
+ t.Fatalf("expected annotation to survive, got %#v", updated.Annotations)
+ }
+ if len(updated.Finalizers) != 1 || updated.Finalizers[0] != "protect.example.com/runtime" {
+ t.Fatalf("expected finalizer to survive, got %#v", updated.Finalizers)
+ }
+ if updated.Labels["clawmanager.io/runtime-type"] != "openclaw" {
+ t.Fatalf("expected owned runtime label to be present, got %#v", updated.Labels)
+ }
+ if updated.Spec.Template.Labels["admission.example.com/template-label"] != "keep" {
+ t.Fatalf("expected unrelated template label to survive, got %#v", updated.Spec.Template.Labels)
+ }
+ if updated.Spec.Template.Annotations["admission.example.com/template-annotation"] != "enabled" {
+ t.Fatalf("expected template annotation to survive, got %#v", updated.Spec.Template.Annotations)
+ }
+ if updated.Spec.Template.Labels["clawmanager.io/runtime-type"] != "openclaw" {
+ t.Fatalf("expected owned template runtime label to be present, got %#v", updated.Spec.Template.Labels)
+ }
+ if updated.Spec.Template.Spec.Containers[0].Image != "registry/openclaw:v2" {
+ t.Fatalf("expected image update while preserving template metadata, got %q", updated.Spec.Template.Spec.Containers[0].Image)
+ }
+}
+
+func TestRuntimeDeploymentServiceEnsureRejectsSelectorChange(t *testing.T) {
+ existing := &appsv1.Deployment{
+ ObjectMeta: metav1.ObjectMeta{Name: "runtime-openclaw", Namespace: "runtime-system"},
+ Spec: appsv1.DeploymentSpec{
+ Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "different"}},
+ },
+ }
+ client := fake.NewSimpleClientset(existing)
+ service := NewRuntimeDeploymentService(client)
+
+ err := service.Ensure(context.Background(), RuntimeDeploymentSpec{
+ Name: "runtime-openclaw",
+ Namespace: "runtime-system",
+ RuntimeType: "openclaw",
+ Image: "registry/openclaw:latest",
+ Replicas: 1,
+ })
+ if err == nil {
+ t.Fatalf("expected selector mismatch error")
+ }
+}
+
+func TestRuntimeDeploymentServiceScale(t *testing.T) {
+ deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{
+ Name: "runtime-openclaw",
+ Namespace: "runtime-system",
+ RuntimeType: "openclaw",
+ Image: "registry/openclaw:latest",
+ Replicas: 1,
+ WorkspaceNFSServer: "nfs.local",
+ WorkspaceNFSPath: "/exports",
+ })
+ deployment.Spec.Template.Annotations = map[string]string{"admission.example.com/template": "keep"}
+ client := fake.NewSimpleClientset(deployment)
+ service := NewRuntimeDeploymentService(client)
+
+ if err := service.Scale(context.Background(), "runtime-system", "runtime-openclaw", 4); err != nil {
+ t.Fatalf("Scale returned error: %v", err)
+ }
+ scaled, err := client.AppsV1().Deployments("runtime-system").Get(context.Background(), "runtime-openclaw", metav1.GetOptions{})
+ if err != nil {
+ t.Fatalf("failed to get scaled deployment: %v", err)
+ }
+ if scaled.Spec.Replicas == nil || *scaled.Spec.Replicas != 4 {
+ t.Fatalf("expected replicas 4, got %#v", scaled.Spec.Replicas)
+ }
+ if scaled.Spec.Template.Spec.Containers[0].Image != "registry/openclaw:latest" {
+ t.Fatalf("expected Scale to preserve image, got %q", scaled.Spec.Template.Spec.Containers[0].Image)
+ }
+ if scaled.Spec.Template.Annotations["admission.example.com/template"] != "keep" {
+ t.Fatalf("expected Scale to preserve template metadata, got %#v", scaled.Spec.Template.Annotations)
+ }
+}
+
+func TestRuntimeDeploymentServiceRolloutImage(t *testing.T) {
+ deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{
+ Name: "runtime-hermes",
+ Namespace: "runtime-system",
+ RuntimeType: "hermes",
+ Image: "registry/hermes:v1",
+ Replicas: 3,
+ WorkspaceNFSServer: "nfs.local",
+ WorkspaceNFSPath: "/exports",
+ })
+ client := fake.NewSimpleClientset(deployment)
+ service := NewRuntimeDeploymentService(client)
+
+ if err := service.RolloutImage(context.Background(), "runtime-system", "runtime-hermes", "registry/hermes:v2", 1, 2); err != nil {
+ t.Fatalf("RolloutImage returned error: %v", err)
+ }
+
+ updated, err := client.AppsV1().Deployments("runtime-system").Get(context.Background(), "runtime-hermes", metav1.GetOptions{})
+ if err != nil {
+ t.Fatalf("failed to get updated deployment: %v", err)
+ }
+ container := updated.Spec.Template.Spec.Containers[0]
+ if container.Image != "registry/hermes:v2" {
+ t.Fatalf("expected updated image, got %q", container.Image)
+ }
+ requireEnv(t, container, "CLAWMANAGER_RUNTIME_IMAGE_REF", "registry/hermes:v2")
+ if updated.Spec.Strategy.Type != appsv1.RollingUpdateDeploymentStrategyType {
+ t.Fatalf("expected RollingUpdate strategy, got %q", updated.Spec.Strategy.Type)
+ }
+ if updated.Spec.Strategy.RollingUpdate == nil {
+ t.Fatal("expected rolling update strategy")
+ }
+ if got := updated.Spec.Strategy.RollingUpdate.MaxUnavailable.IntValue(); got != 1 {
+ t.Fatalf("maxUnavailable = %d, want 1", got)
+ }
+ if got := updated.Spec.Strategy.RollingUpdate.MaxSurge.IntValue(); got != 2 {
+ t.Fatalf("maxSurge = %d, want 2", got)
+ }
+}
+
+func TestRuntimeDeploymentServiceRolloutImageRetriesConflict(t *testing.T) {
+ deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{
+ Name: "runtime-hermes",
+ Namespace: "runtime-system",
+ RuntimeType: "hermes",
+ Image: "registry/hermes:v1",
+ Replicas: 1,
+ WorkspaceNFSServer: "nfs.local",
+ WorkspaceNFSPath: "/exports",
+ })
+ client := fake.NewSimpleClientset(deployment)
+ updateAttempts := 0
+ client.Fake.PrependReactor("update", "deployments", func(action k8stesting.Action) (bool, runtime.Object, error) {
+ updateAttempts++
+ if updateAttempts == 1 {
+ return true, nil, apierrors.NewConflict(schema.GroupResource{Group: "apps", Resource: "deployments"}, "runtime-hermes", fmt.Errorf("stale resource version"))
+ }
+ return false, nil, nil
+ })
+ service := NewRuntimeDeploymentService(client)
+
+ if err := service.RolloutImage(context.Background(), "runtime-system", "runtime-hermes", "registry/hermes:v2", 1, 1); err != nil {
+ t.Fatalf("RolloutImage returned error: %v", err)
+ }
+ if updateAttempts < 2 {
+ t.Fatalf("update attempts = %d, want retry after conflict", updateAttempts)
+ }
+
+ updated, err := client.AppsV1().Deployments("runtime-system").Get(context.Background(), "runtime-hermes", metav1.GetOptions{})
+ if err != nil {
+ t.Fatalf("failed to get updated deployment: %v", err)
+ }
+ if got := updated.Spec.Template.Spec.Containers[0].Image; got != "registry/hermes:v2" {
+ t.Fatalf("image = %q, want registry/hermes:v2", got)
+ }
+}
+
+func TestRuntimeDeploymentServiceListPodsReturnsRuntimeDeploymentPods(t *testing.T) {
+ deployment := BuildRuntimeDeployment(RuntimeDeploymentSpec{
+ Name: "openclaw-runtime",
+ Namespace: "runtime-system",
+ RuntimeType: "openclaw",
+ Image: "registry/openclaw-lite:final2",
+ Replicas: 1,
+ WorkspaceNFSServer: "nfs.local",
+ WorkspaceNFSPath: "/exports",
+ })
+ podIP := "10.42.0.12"
+ nodeName := "node-a"
+ pod := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "openclaw-runtime-abc",
+ Namespace: "runtime-system",
+ Labels: map[string]string{
+ "app": "openclaw-runtime",
+ "clawmanager.io/runtime-type": "openclaw",
+ },
+ },
+ Spec: corev1.PodSpec{
+ NodeName: nodeName,
+ Containers: []corev1.Container{{
+ Name: "runtime",
+ Image: "registry/openclaw-lite:final2",
+ }},
+ },
+ Status: corev1.PodStatus{
+ Phase: corev1.PodRunning,
+ PodIP: podIP,
+ Conditions: []corev1.PodCondition{{
+ Type: corev1.PodReady,
+ Status: corev1.ConditionTrue,
+ }},
+ },
+ }
+ client := fake.NewSimpleClientset(deployment, pod)
+ service := NewRuntimeDeploymentService(client)
+
+ pods, err := service.ListPods(context.Background(), "runtime-system", "openclaw")
+ if err != nil {
+ t.Fatalf("ListPods returned error: %v", err)
+ }
+ if len(pods) != 1 {
+ t.Fatalf("pods length = %d, want 1: %#v", len(pods), pods)
+ }
+ got := pods[0]
+ if got.RuntimeType != "openclaw" || got.Namespace != "runtime-system" || got.DeploymentName != "openclaw-runtime" || got.PodName != "openclaw-runtime-abc" {
+ t.Fatalf("runtime pod identity = %+v, want openclaw runtime pod", got)
+ }
+ if got.ImageRef != "registry/openclaw-lite:final2" {
+ t.Fatalf("image = %q, want registry/openclaw-lite:final2", got.ImageRef)
+ }
+ if got.State != "ready" {
+ t.Fatalf("state = %q, want ready", got.State)
+ }
+ if got.PodIP == nil || *got.PodIP != podIP || got.NodeName == nil || *got.NodeName != nodeName {
+ t.Fatalf("pod network fields = ip:%v node:%v, want %s/%s", got.PodIP, got.NodeName, podIP, nodeName)
+ }
+}
+
+func requireContainerPort(t *testing.T, container corev1.Container, name string, port int32) {
+ t.Helper()
+ for _, got := range container.Ports {
+ if got.Name == name && got.ContainerPort == port {
+ return
+ }
+ }
+ t.Fatalf("expected container port %s=%d, got %#v", name, port, container.Ports)
+}
+
+func requireVolumeMount(t *testing.T, container corev1.Container, name, mountPath string) {
+ t.Helper()
+ for _, got := range container.VolumeMounts {
+ if got.Name == name && got.MountPath == mountPath {
+ return
+ }
+ }
+ t.Fatalf("expected volume mount %s at %s, got %#v", name, mountPath, container.VolumeMounts)
+}
+
+func requireEnv(t *testing.T, container corev1.Container, name, value string) {
+ t.Helper()
+ for _, got := range container.Env {
+ if got.Name == name && got.Value == value {
+ return
+ }
+ }
+ t.Fatalf("expected env %s=%q, got %#v", name, value, container.Env)
+}
+
+func requireEnvAbsent(t *testing.T, container corev1.Container, name string) {
+ t.Helper()
+ for _, got := range container.Env {
+ if got.Name == name {
+ t.Fatalf("expected env %s to be absent, got %#v", name, container.Env)
+ }
+ }
+}
+
+func requireEnvFieldRef(t *testing.T, container corev1.Container, name, fieldPath string) {
+ t.Helper()
+ for _, got := range container.Env {
+ if got.Name != name || got.ValueFrom == nil || got.ValueFrom.FieldRef == nil {
+ continue
+ }
+ if got.ValueFrom.FieldRef.FieldPath == fieldPath {
+ return
+ }
+ }
+ t.Fatalf("expected env %s from fieldRef %q, got %#v", name, fieldPath, container.Env)
+}
diff --git a/backend/internal/services/openclaw_transfer_service.go b/backend/internal/services/openclaw_transfer_service.go
index 41af7c4..22b8814 100644
--- a/backend/internal/services/openclaw_transfer_service.go
+++ b/backend/internal/services/openclaw_transfer_service.go
@@ -7,7 +7,10 @@ import (
"fmt"
"io"
"strings"
+ "sync"
+ "clawreef/internal/models"
+ "clawreef/internal/repository"
"clawreef/internal/services/k8s"
corev1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes/scheme"
@@ -39,9 +42,29 @@ type OpenClawTransferService interface {
}
type openClawTransferService struct {
- podService *k8s.PodService
+ podService *k8s.PodService
+ instanceRepo repository.InstanceRepository
+ bindingRepo repository.InstanceRuntimeBindingRepository
+ runtimePodRepo repository.RuntimePodRepository
}
+type openClawTransferRuntimeRepositories struct {
+ instanceRepo repository.InstanceRepository
+ bindingRepo repository.InstanceRuntimeBindingRepository
+ runtimePodRepo repository.RuntimePodRepository
+}
+
+type transferExecTarget struct {
+ namespace string
+ podName string
+ container string
+}
+
+var (
+ openClawTransferReposMu sync.RWMutex
+ openClawTransferRepos openClawTransferRuntimeRepositories
+)
+
type workspaceTransferSpec struct {
dirName string
baseDirExpr string
@@ -51,11 +74,27 @@ type workspaceTransferSpec struct {
}
func NewOpenClawTransferService() OpenClawTransferService {
+ openClawTransferReposMu.RLock()
+ repos := openClawTransferRepos
+ openClawTransferReposMu.RUnlock()
return &openClawTransferService{
- podService: k8s.NewPodService(),
+ podService: k8s.NewPodService(),
+ instanceRepo: repos.instanceRepo,
+ bindingRepo: repos.bindingRepo,
+ runtimePodRepo: repos.runtimePodRepo,
}
}
+func SetOpenClawTransferRuntimeRepositories(instanceRepo repository.InstanceRepository, bindingRepo repository.InstanceRuntimeBindingRepository, runtimePodRepo repository.RuntimePodRepository) {
+ openClawTransferReposMu.Lock()
+ openClawTransferRepos = openClawTransferRuntimeRepositories{
+ instanceRepo: instanceRepo,
+ bindingRepo: bindingRepo,
+ runtimePodRepo: runtimePodRepo,
+ }
+ openClawTransferReposMu.Unlock()
+}
+
// buildBaseDirExpr returns a POSIX shell expression that resolves the
// OpenClaw persistent directory inside the desktop container. It honors the
// CLAWMANAGER_AGENT_PERSISTENT_DIR env var (injected by ClawManager at pod
@@ -150,21 +189,26 @@ func buildWorkspaceImportCommand(spec workspaceTransferSpec) []string {
}
func (s *openClawTransferService) Export(ctx context.Context, userID, instanceID int) ([]byte, error) {
- return s.exportWorkspace(ctx, userID, instanceID, openClawWorkspaceSpec(), buildExportCommand())
+ return s.exportWorkspace(ctx, userID, instanceID, openClawWorkspaceSpec())
}
func (s *openClawTransferService) ExportHermes(ctx context.Context, userID, instanceID int) ([]byte, error) {
- return s.exportWorkspace(ctx, userID, instanceID, hermesWorkspaceSpec(), buildHermesExportCommand())
+ return s.exportWorkspace(ctx, userID, instanceID, hermesWorkspaceSpec())
}
-func (s *openClawTransferService) exportWorkspace(ctx context.Context, userID, instanceID int, spec workspaceTransferSpec, command []string) ([]byte, error) {
+func (s *openClawTransferService) exportWorkspace(ctx context.Context, userID, instanceID int, spec workspaceTransferSpec) ([]byte, error) {
+ resolvedSpec, err := s.workspaceSpecForInstance(ctx, userID, instanceID, spec)
+ if err != nil {
+ return nil, err
+ }
+ command := buildWorkspaceExportCommand(resolvedSpec)
var stdout bytes.Buffer
var stderr bytes.Buffer
if err := s.exec(ctx, userID, instanceID, command, nil, &stdout, &stderr); err != nil {
if isExportEmptyWorkspaceError(err) {
- return nil, spec.missingErr
+ return nil, resolvedSpec.missingErr
}
- return nil, formatExecError("export "+spec.actionLabel, err, stderr.String())
+ return nil, formatExecError("export "+resolvedSpec.actionLabel, err, stderr.String())
}
return stdout.Bytes(), nil
@@ -187,40 +231,76 @@ func isExportEmptyWorkspaceError(err error) bool {
}
func (s *openClawTransferService) Import(ctx context.Context, userID, instanceID int, archive io.Reader) error {
- return s.importWorkspace(ctx, userID, instanceID, archive, openClawWorkspaceSpec(), buildImportCommand())
+ return s.importWorkspace(ctx, userID, instanceID, archive, openClawWorkspaceSpec())
}
func (s *openClawTransferService) ImportHermes(ctx context.Context, userID, instanceID int, archive io.Reader) error {
- return s.importWorkspace(ctx, userID, instanceID, archive, hermesWorkspaceSpec(), buildHermesImportCommand())
+ return s.importWorkspace(ctx, userID, instanceID, archive, hermesWorkspaceSpec())
}
-func (s *openClawTransferService) importWorkspace(ctx context.Context, userID, instanceID int, archive io.Reader, spec workspaceTransferSpec, command []string) error {
+func (s *openClawTransferService) importWorkspace(ctx context.Context, userID, instanceID int, archive io.Reader, spec workspaceTransferSpec) error {
+ resolvedSpec, err := s.workspaceSpecForInstance(ctx, userID, instanceID, spec)
+ if err != nil {
+ return err
+ }
+ command := buildWorkspaceImportCommand(resolvedSpec)
var stderr bytes.Buffer
if err := s.exec(ctx, userID, instanceID, command, archive, nil, &stderr); err != nil {
- return formatExecError("import "+spec.actionLabel, err, stderr.String())
+ return formatExecError("import "+resolvedSpec.actionLabel, err, stderr.String())
}
return nil
}
+func (s *openClawTransferService) workspaceSpecForInstance(ctx context.Context, userID, instanceID int, spec workspaceTransferSpec) (workspaceTransferSpec, error) {
+ instance, err := s.runtimeManagedInstance(ctx, userID, instanceID)
+ if err != nil || instance == nil {
+ return spec, err
+ }
+ baseDir := ""
+ if instance.WorkspacePath != nil {
+ baseDir = strings.TrimSpace(*instance.WorkspacePath)
+ }
+ if baseDir == "" {
+ runtimeType, ok := NormalizeV2RuntimeType(instance.Type)
+ if !ok {
+ return spec, nil
+ }
+ baseDir = RuntimeWorkspacePath(runtimeType, instance.UserID, instance.ID)
+ }
+ spec.baseDirExpr = baseDir
+ return spec, nil
+}
+
func (s *openClawTransferService) exec(ctx context.Context, userID, instanceID int, command []string, stdin io.Reader, stdout, stderr io.Writer) error {
if s.podService == nil || s.podService.GetClient() == nil || s.podService.GetClient().Clientset == nil {
return fmt.Errorf("k8s client not initialized")
}
- pod, err := s.podService.GetPod(ctx, userID, instanceID)
+ target, err := s.runtimeExecTarget(ctx, userID, instanceID)
if err != nil {
- return fmt.Errorf("failed to get pod: %w", err)
+ return err
+ }
+ if target == nil {
+ pod, podErr := s.podService.GetPod(ctx, userID, instanceID)
+ if podErr != nil {
+ return fmt.Errorf("failed to get pod: %w", podErr)
+ }
+ target = &transferExecTarget{
+ namespace: pod.Namespace,
+ podName: pod.Name,
+ container: "desktop",
+ }
}
req := s.podService.GetClient().Clientset.CoreV1().RESTClient().Post().
Resource("pods").
- Name(pod.Name).
- Namespace(pod.Namespace).
+ Name(target.podName).
+ Namespace(target.namespace).
SubResource("exec")
req.VersionedParams(&corev1.PodExecOptions{
- Container: "desktop",
+ Container: target.container,
Command: command,
Stdin: stdin != nil,
Stdout: stdout != nil,
@@ -241,6 +321,68 @@ func (s *openClawTransferService) exec(ctx context.Context, userID, instanceID i
})
}
+func (s *openClawTransferService) runtimeManagedInstance(ctx context.Context, userID, instanceID int) (*models.Instance, error) {
+ if s == nil || s.instanceRepo == nil {
+ return nil, nil
+ }
+ instance, err := s.instanceRepo.GetByID(instanceID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get instance: %w", err)
+ }
+ if instance == nil {
+ return nil, nil
+ }
+ if instance.UserID != userID {
+ return nil, fmt.Errorf("instance does not belong to user")
+ }
+ if !isLiteRuntimeInstance(instance) {
+ return nil, nil
+ }
+ return instance, nil
+}
+
+func (s *openClawTransferService) runtimeExecTarget(ctx context.Context, userID, instanceID int) (*transferExecTarget, error) {
+ instance, err := s.runtimeManagedInstance(ctx, userID, instanceID)
+ if err != nil || instance == nil {
+ return nil, err
+ }
+ if s.bindingRepo == nil || s.runtimePodRepo == nil {
+ return nil, fmt.Errorf("runtime repositories are not configured for lite workspace transfer")
+ }
+ binding, err := s.bindingRepo.GetRunningByInstanceID(ctx, instanceID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get runtime binding: %w", err)
+ }
+ if binding == nil {
+ return nil, fmt.Errorf("runtime binding not found for instance %d", instanceID)
+ }
+ if binding.Generation != instance.RuntimeGeneration {
+ return nil, fmt.Errorf("runtime binding generation does not match instance")
+ }
+ runtimePod, err := s.runtimePodRepo.GetByID(ctx, binding.RuntimePodID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get runtime pod: %w", err)
+ }
+ if runtimePod == nil {
+ return nil, fmt.Errorf("runtime pod not found for instance %d", instanceID)
+ }
+ return &transferExecTarget{
+ namespace: runtimePod.Namespace,
+ podName: runtimePod.PodName,
+ container: "runtime",
+ }, nil
+}
+
+func isLiteRuntimeInstance(instance *models.Instance) bool {
+ if instance == nil {
+ return false
+ }
+ if strings.EqualFold(strings.TrimSpace(instance.InstanceMode), InstanceModeLite) {
+ return true
+ }
+ return strings.EqualFold(strings.TrimSpace(instance.RuntimeType), RuntimeBackendGateway)
+}
+
func shellQuote(value string) string {
return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'"
}
diff --git a/backend/internal/services/openclaw_transfer_service_test.go b/backend/internal/services/openclaw_transfer_service_test.go
index b2445b0..8afe957 100644
--- a/backend/internal/services/openclaw_transfer_service_test.go
+++ b/backend/internal/services/openclaw_transfer_service_test.go
@@ -1,8 +1,11 @@
package services
import (
+ "context"
"strings"
"testing"
+
+ "clawreef/internal/models"
)
func TestBuildBaseDirExpr_UsesEnvVarWithFallback(t *testing.T) {
@@ -110,6 +113,47 @@ func TestBuildHermesImportCommand_PreservesMountedHermesDirectory(t *testing.T)
}
}
+func TestWorkspaceSpecForLiteUsesRuntimeWorkspacePath(t *testing.T) {
+ instanceRepo := newFakeRuntimeInstanceRepo()
+ instanceRepo.byID[123] = &models.Instance{
+ ID: 123,
+ UserID: 45,
+ Type: RuntimeTypeOpenClaw,
+ RuntimeType: RuntimeBackendGateway,
+ InstanceMode: InstanceModeLite,
+ RuntimeGeneration: 2,
+ }
+ service := &openClawTransferService{instanceRepo: instanceRepo}
+
+ spec, err := service.workspaceSpecForInstance(context.Background(), 45, 123, openClawWorkspaceSpec())
+ if err != nil {
+ t.Fatalf("workspaceSpecForInstance returned error: %v", err)
+ }
+ if got, want := spec.baseDirExpr, RuntimeWorkspacePath(RuntimeTypeOpenClaw, 45, 123); got != want {
+ t.Fatalf("base dir = %q, want %q", got, want)
+ }
+}
+
+func TestWorkspaceSpecForProKeepsDesktopConfigPath(t *testing.T) {
+ instanceRepo := newFakeRuntimeInstanceRepo()
+ instanceRepo.byID[124] = &models.Instance{
+ ID: 124,
+ UserID: 45,
+ Type: RuntimeTypeOpenClaw,
+ RuntimeType: RuntimeBackendDesktop,
+ InstanceMode: InstanceModePro,
+ }
+ service := &openClawTransferService{instanceRepo: instanceRepo}
+
+ spec, err := service.workspaceSpecForInstance(context.Background(), 45, 124, openClawWorkspaceSpec())
+ if err != nil {
+ t.Fatalf("workspaceSpecForInstance returned error: %v", err)
+ }
+ if got, want := spec.baseDirExpr, buildBaseDirExpr(); got != want {
+ t.Fatalf("base dir = %q, want %q", got, want)
+ }
+}
+
func TestShellQuote_HandlesSingleQuotes(t *testing.T) {
cases := []struct {
in, want string
diff --git a/backend/internal/services/redis_client.go b/backend/internal/services/redis_client.go
new file mode 100644
index 0000000..b10baae
--- /dev/null
+++ b/backend/internal/services/redis_client.go
@@ -0,0 +1,17 @@
+package services
+
+import (
+ "context"
+ "time"
+)
+
+type PlatformRedisClient interface {
+ SetNX(ctx context.Context, key, value string, ttl time.Duration) (bool, error)
+ Del(ctx context.Context, key string) error
+ XAdd(ctx context.Context, key string, fields map[string]string) (string, error)
+ XRead(ctx context.Context, key, lastID string, block time.Duration) ([]redisStreamMessage, error)
+}
+
+func NewPlatformRedisClient(rawURL string) (PlatformRedisClient, error) {
+ return newRedisBus(rawURL)
+}
diff --git a/backend/internal/services/risk_hit_service.go b/backend/internal/services/risk_hit_service.go
index d198933..28df2d7 100644
--- a/backend/internal/services/risk_hit_service.go
+++ b/backend/internal/services/risk_hit_service.go
@@ -10,10 +10,17 @@ import (
// RiskHitService defines operations for persisted risk hits.
type RiskHitService interface {
- RecordHits(traceID string, sessionID, requestID *string, userID, instanceID, invocationID *int, action string, hits []RiskMatch) error
+ RecordHits(traceID string, sessionID, requestID *string, userID, instanceID, invocationID *int, attribution *RiskHitAttribution, action string, hits []RiskMatch) error
ListHitsByTraceID(traceID string) ([]models.RiskHit, error)
}
+type RiskHitAttribution struct {
+ InstanceMode *string
+ RuntimeType *string
+ GatewayID *string
+ RuntimePodID *int64
+}
+
type riskHitService struct {
repo repository.RiskHitRepository
}
@@ -23,7 +30,7 @@ func NewRiskHitService(repo repository.RiskHitRepository) RiskHitService {
return &riskHitService{repo: repo}
}
-func (s *riskHitService) RecordHits(traceID string, sessionID, requestID *string, userID, instanceID, invocationID *int, action string, hits []RiskMatch) error {
+func (s *riskHitService) RecordHits(traceID string, sessionID, requestID *string, userID, instanceID, invocationID *int, attribution *RiskHitAttribution, action string, hits []RiskMatch) error {
traceID = strings.TrimSpace(traceID)
if traceID == "" || len(hits) == 0 {
return nil
@@ -39,6 +46,10 @@ func (s *riskHitService) RecordHits(traceID string, sessionID, requestID *string
RequestID: requestID,
UserID: userID,
InstanceID: instanceID,
+ InstanceMode: nil,
+ RuntimeType: nil,
+ GatewayID: nil,
+ RuntimePodID: nil,
InvocationID: invocationID,
RuleID: strings.TrimSpace(hit.RuleID),
RuleName: strings.TrimSpace(hit.RuleName),
@@ -46,6 +57,12 @@ func (s *riskHitService) RecordHits(traceID string, sessionID, requestID *string
Action: action,
MatchSummary: strings.TrimSpace(hit.MatchSummary),
}
+ if attribution != nil {
+ record.InstanceMode = attribution.InstanceMode
+ record.RuntimeType = attribution.RuntimeType
+ record.GatewayID = attribution.GatewayID
+ record.RuntimePodID = attribution.RuntimePodID
+ }
if record.RuleID == "" || record.RuleName == "" || record.Severity == "" || record.MatchSummary == "" {
return fmt.Errorf("risk hit record is incomplete")
}
diff --git a/backend/internal/services/runtime_agent_client.go b/backend/internal/services/runtime_agent_client.go
new file mode 100644
index 0000000..fcf5b0d
--- /dev/null
+++ b/backend/internal/services/runtime_agent_client.go
@@ -0,0 +1,130 @@
+package services
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "strings"
+ "time"
+)
+
+type RuntimeAgentClient interface {
+ Health(ctx context.Context, endpoint string) error
+ CreateGateway(ctx context.Context, endpoint string, req RuntimeAgentCreateGatewayRequest) (*RuntimeAgentCreateGatewayResponse, error)
+ DeleteGateway(ctx context.Context, endpoint, gatewayID string) error
+ Drain(ctx context.Context, endpoint string) error
+}
+
+type RuntimeAgentPortRange struct {
+ Start int `json:"start"`
+ End int `json:"end"`
+}
+
+type RuntimeAgentCreateGatewayRequest struct {
+ InstanceID int `json:"instance_id"`
+ UserID int `json:"user_id"`
+ AgentType string `json:"agent_type"`
+ WorkspacePath string `json:"workspace_path"`
+ PortRange RuntimeAgentPortRange `json:"port_range"`
+ UID int `json:"uid"`
+ GID int `json:"gid"`
+ CPUCores float64 `json:"cpu_cores"`
+ MemoryMB int `json:"memory_mb"`
+ DiskQuotaMB int `json:"disk_quota_mb"`
+ Generation int `json:"generation"`
+ Environment map[string]string `json:"environment,omitempty"`
+}
+
+type RuntimeAgentCreateGatewayResponse struct {
+ GatewayID string `json:"gateway_id"`
+ Port int `json:"port"`
+ PID *int `json:"pid,omitempty"`
+ Status string `json:"status"`
+}
+
+type runtimeAgentHTTPClient struct {
+ controlToken string
+ httpClient *http.Client
+}
+
+func NewRuntimeAgentClient(controlToken string) RuntimeAgentClient {
+ return NewRuntimeAgentClientWithHTTPClient(controlToken, nil)
+}
+
+func NewRuntimeAgentClientWithHTTPClient(controlToken string, httpClient *http.Client) RuntimeAgentClient {
+ if httpClient == nil {
+ httpClient = &http.Client{
+ Timeout: 30 * time.Second,
+ }
+ }
+ clientCopy := *httpClient
+ clientCopy.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+ return http.ErrUseLastResponse
+ }
+ return &runtimeAgentHTTPClient{
+ controlToken: controlToken,
+ httpClient: &clientCopy,
+ }
+}
+
+func (c *runtimeAgentHTTPClient) Health(ctx context.Context, endpoint string) error {
+ return c.do(ctx, http.MethodGet, endpoint, "/v1/health", nil, nil)
+}
+
+func (c *runtimeAgentHTTPClient) CreateGateway(ctx context.Context, endpoint string, req RuntimeAgentCreateGatewayRequest) (*RuntimeAgentCreateGatewayResponse, error) {
+ var resp RuntimeAgentCreateGatewayResponse
+ if err := c.do(ctx, http.MethodPost, endpoint, "/v1/gateways", req, &resp); err != nil {
+ return nil, err
+ }
+ return &resp, nil
+}
+
+func (c *runtimeAgentHTTPClient) DeleteGateway(ctx context.Context, endpoint, gatewayID string) error {
+ return c.do(ctx, http.MethodDelete, endpoint, "/v1/gateways/"+url.PathEscape(gatewayID), nil, nil)
+}
+
+func (c *runtimeAgentHTTPClient) Drain(ctx context.Context, endpoint string) error {
+ return c.do(ctx, http.MethodPost, endpoint, "/v1/drain", map[string]bool{"draining": true}, nil)
+}
+
+func (c *runtimeAgentHTTPClient) do(ctx context.Context, method, endpoint, path string, body any, out any) error {
+ endpoint = strings.TrimRight(endpoint, "/")
+ var reader io.Reader
+ if body != nil {
+ payload, err := json.Marshal(body)
+ if err != nil {
+ return err
+ }
+ reader = bytes.NewReader(payload)
+ }
+
+ req, err := http.NewRequestWithContext(ctx, method, endpoint+path, reader)
+ if err != nil {
+ return err
+ }
+ req.Header.Set("X-ClawManager-Control-Token", c.controlToken)
+ if body != nil {
+ req.Header.Set("Content-Type", "application/json")
+ }
+
+ resp, err := c.httpClient.Do(req)
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+ msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+ if resp.StatusCode == http.StatusConflict {
+ return fmt.Errorf("runtime agent conflict: %s", string(msg))
+ }
+ return fmt.Errorf("runtime agent status %d: %s", resp.StatusCode, string(msg))
+ }
+ if out == nil {
+ return nil
+ }
+ return json.NewDecoder(resp.Body).Decode(out)
+}
diff --git a/backend/internal/services/runtime_agent_client_test.go b/backend/internal/services/runtime_agent_client_test.go
new file mode 100644
index 0000000..fc461bf
--- /dev/null
+++ b/backend/internal/services/runtime_agent_client_test.go
@@ -0,0 +1,223 @@
+package services
+
+import (
+ "context"
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+)
+
+func TestRuntimeAgentClientCreateGateway(t *testing.T) {
+ var gotToken string
+ var gotReq RuntimeAgentCreateGatewayRequest
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/v1/gateways" {
+ t.Fatalf("unexpected path %s", r.URL.Path)
+ }
+ gotToken = r.Header.Get("X-ClawManager-Control-Token")
+ if err := json.NewDecoder(r.Body).Decode(&gotReq); err != nil {
+ t.Fatalf("decode request: %v", err)
+ }
+ _ = json.NewEncoder(w).Encode(RuntimeAgentCreateGatewayResponse{
+ GatewayID: "gw-7-3",
+ Port: 20017,
+ PID: runtimeAgentIntPtr(8842),
+ Status: "running",
+ })
+ }))
+ defer server.Close()
+
+ client := NewRuntimeAgentClient("secret")
+ resp, err := client.CreateGateway(context.Background(), server.URL, RuntimeAgentCreateGatewayRequest{
+ InstanceID: 7,
+ UserID: 8,
+ AgentType: "openclaw",
+ WorkspacePath: "/workspaces/openclaw/user-8/instance-7",
+ PortRange: RuntimeAgentPortRange{Start: 20000, End: 20099},
+ UID: 200007,
+ GID: 200007,
+ CPUCores: 2,
+ MemoryMB: 4096,
+ DiskQuotaMB: 20480,
+ Generation: 3,
+ })
+ if err != nil {
+ t.Fatalf("CreateGateway returned error: %v", err)
+ }
+ if gotToken != "secret" {
+ t.Fatalf("unexpected token %q", gotToken)
+ }
+ if gotReq.InstanceID != 7 || gotReq.PortRange.Start != 20000 {
+ t.Fatalf("unexpected request %#v", gotReq)
+ }
+ if resp.GatewayID != "gw-7-3" || resp.Port != 20017 {
+ t.Fatalf("unexpected response %#v", resp)
+ }
+}
+
+func TestRuntimeAgentClientCreateGatewayTrimsEndpointSlash(t *testing.T) {
+ var calls int
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ calls++
+ if r.URL.Path != "/v1/gateways" {
+ t.Fatalf("unexpected path %s", r.URL.Path)
+ }
+ _ = json.NewEncoder(w).Encode(RuntimeAgentCreateGatewayResponse{
+ GatewayID: "gw-7-3",
+ Port: 20017,
+ Status: "running",
+ })
+ }))
+ defer server.Close()
+
+ client := NewRuntimeAgentClient("secret")
+ if _, err := client.CreateGateway(context.Background(), server.URL+"/", RuntimeAgentCreateGatewayRequest{}); err != nil {
+ t.Fatalf("CreateGateway returned error: %v", err)
+ }
+ if calls != 1 {
+ t.Fatalf("expected one gateway create call, got %d", calls)
+ }
+}
+
+func TestRuntimeAgentClientDefaultTimeoutAllowsGatewayStartup(t *testing.T) {
+ client := NewRuntimeAgentClient("secret")
+ agentClient, ok := client.(*runtimeAgentHTTPClient)
+ if !ok {
+ t.Fatalf("unexpected client type %T", client)
+ }
+ if agentClient.httpClient.Timeout != 30*time.Second {
+ t.Fatalf("default timeout = %v, want 30s", agentClient.httpClient.Timeout)
+ }
+}
+
+func TestRuntimeAgentClientDeleteGatewayEscapesGatewayID(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodDelete {
+ t.Fatalf("unexpected method %s", r.Method)
+ }
+ if r.URL.EscapedPath() != "/v1/gateways/gw-7%2F3" {
+ t.Fatalf("unexpected escaped path %s", r.URL.EscapedPath())
+ }
+ w.WriteHeader(http.StatusNoContent)
+ }))
+ defer server.Close()
+
+ client := NewRuntimeAgentClient("secret")
+ if err := client.DeleteGateway(context.Background(), server.URL, "gw-7/3"); err != nil {
+ t.Fatalf("DeleteGateway returned error: %v", err)
+ }
+}
+
+func TestRuntimeAgentClientHealthHandlesSuccessAndRedirect(t *testing.T) {
+ successServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/v1/health" {
+ t.Fatalf("unexpected success path %s", r.URL.Path)
+ }
+ w.WriteHeader(http.StatusNoContent)
+ }))
+ defer successServer.Close()
+ redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/v1/health":
+ w.Header().Set("Location", "/ok")
+ w.WriteHeader(http.StatusFound)
+ _, _ = w.Write([]byte("redirect not allowed"))
+ case "/ok":
+ w.WriteHeader(http.StatusOK)
+ default:
+ t.Fatalf("unexpected redirect path %s", r.URL.Path)
+ }
+ }))
+ defer redirectServer.Close()
+
+ client := NewRuntimeAgentClient("secret")
+ if err := client.Health(context.Background(), successServer.URL); err != nil {
+ t.Fatalf("Health returned error for 204: %v", err)
+ }
+ err := client.Health(context.Background(), redirectServer.URL)
+ if err == nil || !strings.Contains(err.Error(), "runtime agent status 302") || !strings.Contains(err.Error(), "redirect not allowed") {
+ t.Fatalf("unexpected redirect error %v", err)
+ }
+}
+
+func TestRuntimeAgentClientWithHTTPClientDoesNotMutateRedirectPolicy(t *testing.T) {
+ supplied := &http.Client{Timeout: 10 * time.Second}
+ client := NewRuntimeAgentClientWithHTTPClient("secret", supplied)
+
+ if supplied.CheckRedirect != nil {
+ t.Fatalf("constructor mutated supplied CheckRedirect")
+ }
+
+ agentClient, ok := client.(*runtimeAgentHTTPClient)
+ if !ok {
+ t.Fatalf("unexpected client type %T", client)
+ }
+ if agentClient.httpClient == supplied {
+ t.Fatalf("constructor reused supplied client pointer")
+ }
+ if agentClient.httpClient.Timeout != supplied.Timeout {
+ t.Fatalf("timeout was not preserved: got %v want %v", agentClient.httpClient.Timeout, supplied.Timeout)
+ }
+ if agentClient.httpClient.CheckRedirect == nil {
+ t.Fatalf("copied client has no redirect policy")
+ }
+}
+
+func TestRuntimeAgentClientDrainSendsJSONBody(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/v1/drain" {
+ t.Fatalf("unexpected path %s", r.URL.Path)
+ }
+ if r.Header.Get("Content-Type") != "application/json" {
+ t.Fatalf("unexpected content type %q", r.Header.Get("Content-Type"))
+ }
+ body, err := io.ReadAll(r.Body)
+ if err != nil {
+ t.Fatalf("read body: %v", err)
+ }
+ if strings.TrimSpace(string(body)) != `{"draining":true}` {
+ t.Fatalf("unexpected body %s", body)
+ }
+ w.WriteHeader(http.StatusAccepted)
+ }))
+ defer server.Close()
+
+ client := NewRuntimeAgentClient("secret")
+ if err := client.Drain(context.Background(), server.URL); err != nil {
+ t.Fatalf("Drain returned error: %v", err)
+ }
+}
+
+func TestRuntimeAgentClientConflict(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Error(w, "no free port", http.StatusConflict)
+ }))
+ defer server.Close()
+
+ client := NewRuntimeAgentClient("secret")
+ _, err := client.CreateGateway(context.Background(), server.URL, RuntimeAgentCreateGatewayRequest{})
+ if err == nil || !strings.Contains(err.Error(), "runtime agent conflict") || !strings.Contains(err.Error(), "no free port") {
+ t.Fatalf("unexpected error %v", err)
+ }
+}
+
+func TestRuntimeAgentClientNonConflictErrorIncludesStatusAndBody(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Error(w, "agent exploded", http.StatusInternalServerError)
+ }))
+ defer server.Close()
+
+ client := NewRuntimeAgentClient("secret")
+ _, err := client.CreateGateway(context.Background(), server.URL, RuntimeAgentCreateGatewayRequest{})
+ if err == nil || !strings.Contains(err.Error(), "runtime agent status 500") || !strings.Contains(err.Error(), "agent exploded") {
+ t.Fatalf("unexpected error %v", err)
+ }
+}
+
+func runtimeAgentIntPtr(v int) *int {
+ return &v
+}
diff --git a/backend/internal/services/runtime_capacity.go b/backend/internal/services/runtime_capacity.go
new file mode 100644
index 0000000..e5a59b7
--- /dev/null
+++ b/backend/internal/services/runtime_capacity.go
@@ -0,0 +1,80 @@
+package services
+
+import (
+ "fmt"
+ "path"
+ "strings"
+)
+
+const (
+ RuntimeTypeOpenClaw = "openclaw"
+ RuntimeTypeHermes = "hermes"
+
+ InstanceModeLite = "lite"
+ InstanceModePro = "pro"
+
+ RuntimeBackendGateway = "gateway"
+ RuntimeBackendDesktop = "desktop"
+ RuntimeBackendShell = "shell"
+
+ RuntimeGatewayPortStart = 20000
+ RuntimeGatewayPortEnd = 20099
+ RuntimePodCapacity = 100
+ RuntimeLinuxIDBase = 200000
+)
+
+func NormalizeV2RuntimeType(instanceType string) (string, bool) {
+ switch strings.ToLower(strings.TrimSpace(instanceType)) {
+ case RuntimeTypeOpenClaw:
+ return RuntimeTypeOpenClaw, true
+ case RuntimeTypeHermes:
+ return RuntimeTypeHermes, true
+ default:
+ return "", false
+ }
+}
+
+func NormalizeInstanceMode(mode string) (string, bool) {
+ switch strings.ToLower(strings.TrimSpace(mode)) {
+ case InstanceModeLite:
+ return InstanceModeLite, true
+ case InstanceModePro:
+ return InstanceModePro, true
+ default:
+ return "", false
+ }
+}
+
+func RuntimeTypeForInstanceMode(mode string) (string, bool) {
+ normalized, ok := NormalizeInstanceMode(mode)
+ if !ok {
+ return "", false
+ }
+ if normalized == InstanceModeLite {
+ return RuntimeBackendGateway, true
+ }
+ return RuntimeBackendDesktop, true
+}
+
+func InstanceModeForRuntimeType(runtimeType string) string {
+ if strings.EqualFold(strings.TrimSpace(runtimeType), RuntimeBackendGateway) {
+ return InstanceModeLite
+ }
+ return InstanceModePro
+}
+
+func RuntimeWorkspacePath(runtimeType string, userID int, instanceID int) string {
+ return RuntimeWorkspacePathWithRoot("/workspaces", runtimeType, userID, instanceID)
+}
+
+func RuntimeWorkspacePathWithRoot(root, runtimeType string, userID int, instanceID int) string {
+ root = strings.TrimSpace(root)
+ if root == "" {
+ root = "/workspaces"
+ }
+ return path.Join(root, fmt.Sprintf("%s/user-%d/instance-%d", runtimeType, userID, instanceID))
+}
+
+func RuntimeLinuxID(instanceID int) int {
+ return RuntimeLinuxIDBase + instanceID
+}
diff --git a/backend/internal/services/runtime_capacity_test.go b/backend/internal/services/runtime_capacity_test.go
new file mode 100644
index 0000000..3435615
--- /dev/null
+++ b/backend/internal/services/runtime_capacity_test.go
@@ -0,0 +1,52 @@
+package services
+
+import "testing"
+
+func TestNormalizeV2RuntimeTypeAcceptsManagedRuntimeTypes(t *testing.T) {
+ for _, input := range []string{"openclaw", " OpenClaw ", "hermes", "Hermes"} {
+ got, ok := NormalizeV2RuntimeType(input)
+ if !ok {
+ t.Fatalf("expected %q to be accepted", input)
+ }
+ if got != RuntimeTypeOpenClaw && got != RuntimeTypeHermes {
+ t.Fatalf("expected normalized managed runtime type, got %q", got)
+ }
+ }
+}
+
+func TestNormalizeV2RuntimeTypeRejectsLegacyRuntimeTypes(t *testing.T) {
+ for _, input := range []string{"webtop", "ubuntu", "", "desktop", "shell"} {
+ if got, ok := NormalizeV2RuntimeType(input); ok {
+ t.Fatalf("expected %q to be rejected, got %q", input, got)
+ }
+ }
+}
+
+func TestRuntimeWorkspacePath(t *testing.T) {
+ got := RuntimeWorkspacePath("openclaw", 45, 123)
+ want := "/workspaces/openclaw/user-45/instance-123"
+ if got != want {
+ t.Fatalf("expected workspace path %q, got %q", want, got)
+ }
+}
+
+func TestRuntimeLinuxID(t *testing.T) {
+ if got, want := RuntimeLinuxID(123), 200123; got != want {
+ t.Fatalf("expected linux id %d, got %d", want, got)
+ }
+}
+
+func TestInstanceModeRuntimeTypeMapping(t *testing.T) {
+ if got, ok := RuntimeTypeForInstanceMode(" lite "); !ok || got != RuntimeBackendGateway {
+ t.Fatalf("lite runtime type = %q/%v, want gateway/true", got, ok)
+ }
+ if got, ok := RuntimeTypeForInstanceMode("Pro"); !ok || got != RuntimeBackendDesktop {
+ t.Fatalf("pro runtime type = %q/%v, want desktop/true", got, ok)
+ }
+ if got := InstanceModeForRuntimeType(RuntimeBackendGateway); got != InstanceModeLite {
+ t.Fatalf("gateway mode = %q, want lite", got)
+ }
+ if got := InstanceModeForRuntimeType(RuntimeBackendDesktop); got != InstanceModePro {
+ t.Fatalf("desktop mode = %q, want pro", got)
+ }
+}
diff --git a/backend/internal/services/runtime_events.go b/backend/internal/services/runtime_events.go
new file mode 100644
index 0000000..d0f1f5b
--- /dev/null
+++ b/backend/internal/services/runtime_events.go
@@ -0,0 +1,68 @@
+package services
+
+import (
+ "context"
+ "encoding/json"
+ "time"
+)
+
+const runtimeEventStreamKey = "clawmanager:runtime-events"
+
+type RuntimeEvent struct {
+ Type string `json:"type"`
+ Payload json.RawMessage `json:"payload"`
+ CreatedAt time.Time `json:"created_at"`
+}
+
+type RuntimeEventService interface {
+ Publish(ctx context.Context, eventType string, payload any) error
+ Read(ctx context.Context, lastID string, block time.Duration) ([]redisStreamMessage, error)
+}
+
+func NewRuntimeEventService(redis PlatformRedisClient) RuntimeEventService {
+ if redis == nil {
+ return noopRuntimeEventService{}
+ }
+ return &redisRuntimeEventService{redis: redis}
+}
+
+type redisRuntimeEventService struct {
+ redis PlatformRedisClient
+}
+
+func (s *redisRuntimeEventService) Publish(ctx context.Context, eventType string, payload any) error {
+ payloadJSON, err := json.Marshal(payload)
+ if err != nil {
+ return err
+ }
+ event := RuntimeEvent{
+ Type: eventType,
+ Payload: payloadJSON,
+ CreatedAt: time.Now().UTC(),
+ }
+ eventJSON, err := json.Marshal(event)
+ if err != nil {
+ return err
+ }
+ _, err = s.redis.XAdd(ctx, runtimeEventStreamKey, map[string]string{
+ "type": event.Type,
+ "payload": string(event.Payload),
+ "created_at": event.CreatedAt.Format(time.RFC3339Nano),
+ "event": string(eventJSON),
+ })
+ return err
+}
+
+func (s *redisRuntimeEventService) Read(ctx context.Context, lastID string, block time.Duration) ([]redisStreamMessage, error) {
+ return s.redis.XRead(ctx, runtimeEventStreamKey, lastID, block)
+}
+
+type noopRuntimeEventService struct{}
+
+func (noopRuntimeEventService) Publish(ctx context.Context, eventType string, payload any) error {
+ return nil
+}
+
+func (noopRuntimeEventService) Read(ctx context.Context, lastID string, block time.Duration) ([]redisStreamMessage, error) {
+ return nil, nil
+}
diff --git a/backend/internal/services/runtime_events_test.go b/backend/internal/services/runtime_events_test.go
new file mode 100644
index 0000000..51b1dd4
--- /dev/null
+++ b/backend/internal/services/runtime_events_test.go
@@ -0,0 +1,75 @@
+package services
+
+import (
+ "context"
+ "encoding/json"
+ "testing"
+ "time"
+)
+
+func TestRuntimeEventServiceNoopWhenRedisNil(t *testing.T) {
+ service := NewRuntimeEventService(nil)
+
+ if err := service.Publish(context.Background(), "runtime.test", map[string]string{"ok": "true"}); err != nil {
+ t.Fatalf("Publish returned error: %v", err)
+ }
+ messages, err := service.Read(context.Background(), "0", time.Millisecond)
+ if err != nil {
+ t.Fatalf("Read returned error: %v", err)
+ }
+ if messages != nil {
+ t.Fatalf("messages = %#v, want nil", messages)
+ }
+}
+
+func TestRuntimeEventServicePublishesJSONFields(t *testing.T) {
+ redis := &fakePlatformRedisClient{}
+ service := NewRuntimeEventService(redis)
+
+ if err := service.Publish(context.Background(), "runtime.instance.running", map[string]int{"instance_id": 17}); err != nil {
+ t.Fatalf("Publish returned error: %v", err)
+ }
+ if redis.xaddKey != runtimeEventStreamKey {
+ t.Fatalf("XAdd key = %q", redis.xaddKey)
+ }
+ if redis.xaddFields["type"] != "runtime.instance.running" {
+ t.Fatalf("type field = %q", redis.xaddFields["type"])
+ }
+ var payload map[string]int
+ if err := json.Unmarshal([]byte(redis.xaddFields["payload"]), &payload); err != nil {
+ t.Fatalf("payload is not json: %v", err)
+ }
+ if payload["instance_id"] != 17 {
+ t.Fatalf("payload instance_id = %d", payload["instance_id"])
+ }
+ var event RuntimeEvent
+ if err := json.Unmarshal([]byte(redis.xaddFields["event"]), &event); err != nil {
+ t.Fatalf("event is not json: %v", err)
+ }
+ if event.Type != "runtime.instance.running" {
+ t.Fatalf("event type = %q", event.Type)
+ }
+}
+
+type fakePlatformRedisClient struct {
+ xaddKey string
+ xaddFields map[string]string
+}
+
+func (c *fakePlatformRedisClient) SetNX(ctx context.Context, key, value string, ttl time.Duration) (bool, error) {
+ return true, nil
+}
+
+func (c *fakePlatformRedisClient) Del(ctx context.Context, key string) error {
+ return nil
+}
+
+func (c *fakePlatformRedisClient) XAdd(ctx context.Context, key string, fields map[string]string) (string, error) {
+ c.xaddKey = key
+ c.xaddFields = fields
+ return "1-0", nil
+}
+
+func (c *fakePlatformRedisClient) XRead(ctx context.Context, key, lastID string, block time.Duration) ([]redisStreamMessage, error) {
+ return []redisStreamMessage{{ID: "1-0", Fields: map[string]string{"type": "runtime.test"}}}, nil
+}
diff --git a/backend/internal/services/runtime_leader.go b/backend/internal/services/runtime_leader.go
new file mode 100644
index 0000000..26d994e
--- /dev/null
+++ b/backend/internal/services/runtime_leader.go
@@ -0,0 +1,99 @@
+package services
+
+import (
+ "context"
+ "time"
+
+ coordinationv1 "k8s.io/api/coordination/v1"
+ "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/kubernetes"
+)
+
+const (
+ runtimeLeaderLeaseName = "clawmanager-runtime-scheduler"
+ runtimeLeaderLeaseTTL = 15 * time.Second
+)
+
+type RuntimeLeaderService interface {
+ IsLeader(ctx context.Context) bool
+}
+
+type runtimeLeaderService struct {
+ client kubernetes.Interface
+ namespace string
+ holder string
+}
+
+func NewRuntimeLeaderService(client kubernetes.Interface, namespace, holder string) RuntimeLeaderService {
+ return &runtimeLeaderService{
+ client: client,
+ namespace: namespace,
+ holder: holder,
+ }
+}
+
+func (s *runtimeLeaderService) IsLeader(ctx context.Context) bool {
+ if s == nil || s.client == nil || s.namespace == "" || s.holder == "" {
+ return false
+ }
+
+ leases := s.client.CoordinationV1().Leases(s.namespace)
+ now := metav1.NewMicroTime(time.Now())
+ lease, err := leases.Get(ctx, runtimeLeaderLeaseName, metav1.GetOptions{})
+ if errors.IsNotFound(err) {
+ _, createErr := leases.Create(ctx, s.newLease(now), metav1.CreateOptions{})
+ if createErr == nil {
+ return true
+ }
+ if !errors.IsAlreadyExists(createErr) {
+ return false
+ }
+ lease, err = leases.Get(ctx, runtimeLeaderLeaseName, metav1.GetOptions{})
+ }
+ if err != nil {
+ return false
+ }
+
+ if !s.canAcquire(lease, now.Time) {
+ return false
+ }
+
+ lease.Spec.HolderIdentity = &s.holder
+ lease.Spec.RenewTime = &now
+ ttlSeconds := int32(runtimeLeaderLeaseTTL / time.Second)
+ lease.Spec.LeaseDurationSeconds = &ttlSeconds
+ if _, err := leases.Update(ctx, lease, metav1.UpdateOptions{}); err != nil {
+ return false
+ }
+ return true
+}
+
+func (s *runtimeLeaderService) newLease(now metav1.MicroTime) *coordinationv1.Lease {
+ ttlSeconds := int32(runtimeLeaderLeaseTTL / time.Second)
+ return &coordinationv1.Lease{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: runtimeLeaderLeaseName,
+ Namespace: s.namespace,
+ },
+ Spec: coordinationv1.LeaseSpec{
+ HolderIdentity: &s.holder,
+ AcquireTime: &now,
+ RenewTime: &now,
+ LeaseDurationSeconds: &ttlSeconds,
+ },
+ }
+}
+
+func (s *runtimeLeaderService) canAcquire(lease *coordinationv1.Lease, now time.Time) bool {
+ if lease == nil {
+ return true
+ }
+ if lease.Spec.HolderIdentity != nil && *lease.Spec.HolderIdentity == s.holder {
+ return true
+ }
+ if lease.Spec.RenewTime == nil {
+ return true
+ }
+ return lease.Spec.RenewTime.Add(runtimeLeaderLeaseTTL).Before(now)
+}
diff --git a/backend/internal/services/runtime_leader_test.go b/backend/internal/services/runtime_leader_test.go
new file mode 100644
index 0000000..fe8e1e4
--- /dev/null
+++ b/backend/internal/services/runtime_leader_test.go
@@ -0,0 +1,67 @@
+package services
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ coordinationv1 "k8s.io/api/coordination/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/kubernetes/fake"
+)
+
+func TestRuntimeLeaderServiceFirstCallerBecomesLeader(t *testing.T) {
+ service := NewRuntimeLeaderService(fake.NewSimpleClientset(), "runtime-system", "backend-a")
+
+ if !service.IsLeader(context.Background()) {
+ t.Fatalf("expected first caller to become leader")
+ }
+}
+
+func TestRuntimeLeaderServiceDifferentCallerCannotStealUnexpiredLease(t *testing.T) {
+ now := metav1.NewMicroTime(time.Now())
+ holder := "backend-a"
+ client := fake.NewSimpleClientset(&coordinationv1.Lease{
+ ObjectMeta: metav1.ObjectMeta{Name: "clawmanager-runtime-scheduler", Namespace: "runtime-system"},
+ Spec: coordinationv1.LeaseSpec{
+ HolderIdentity: &holder,
+ RenewTime: &now,
+ LeaseDurationSeconds: int32Ptr(15),
+ },
+ })
+ service := NewRuntimeLeaderService(client, "runtime-system", "backend-b")
+
+ if service.IsLeader(context.Background()) {
+ t.Fatalf("expected different caller not to steal unexpired lease")
+ }
+}
+
+func TestRuntimeLeaderServiceDifferentCallerCanAcquireExpiredLease(t *testing.T) {
+ renewTime := metav1.NewMicroTime(time.Now().Add(-30 * time.Second))
+ holder := "backend-a"
+ client := fake.NewSimpleClientset(&coordinationv1.Lease{
+ ObjectMeta: metav1.ObjectMeta{Name: "clawmanager-runtime-scheduler", Namespace: "runtime-system"},
+ Spec: coordinationv1.LeaseSpec{
+ HolderIdentity: &holder,
+ RenewTime: &renewTime,
+ LeaseDurationSeconds: int32Ptr(15),
+ },
+ })
+ service := NewRuntimeLeaderService(client, "runtime-system", "backend-b")
+
+ if !service.IsLeader(context.Background()) {
+ t.Fatalf("expected different caller to acquire expired lease")
+ }
+
+ lease, err := client.CoordinationV1().Leases("runtime-system").Get(context.Background(), "clawmanager-runtime-scheduler", metav1.GetOptions{})
+ if err != nil {
+ t.Fatalf("failed to load lease: %v", err)
+ }
+ if lease.Spec.HolderIdentity == nil || *lease.Spec.HolderIdentity != "backend-b" {
+ t.Fatalf("expected lease holder backend-b, got %#v", lease.Spec.HolderIdentity)
+ }
+}
+
+func int32Ptr(value int32) *int32 {
+ return &value
+}
diff --git a/backend/internal/services/runtime_scheduler.go b/backend/internal/services/runtime_scheduler.go
new file mode 100644
index 0000000..a4dbf6c
--- /dev/null
+++ b/backend/internal/services/runtime_scheduler.go
@@ -0,0 +1,880 @@
+package services
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log"
+ "strconv"
+ "strings"
+ "time"
+
+ "clawreef/internal/models"
+ "clawreef/internal/repository"
+ "clawreef/internal/services/k8s"
+)
+
+type RuntimeScheduler struct {
+ instanceRepo repository.InstanceRepository
+ podRepo repository.RuntimePodRepository
+ bindingRepo repository.InstanceRuntimeBindingRepository
+ rolloutRepo repository.RuntimeRolloutRepository
+ agentClient RuntimeAgentClient
+ events RuntimeEventService
+ leader RuntimeLeaderService
+ deployments k8s.RuntimeDeploymentService
+ envBuilder RuntimeGatewayEnvBuilder
+ tick time.Duration
+
+ workspaceRoot string
+ runtimeNamespace string
+ gatewayPortStart int
+ gatewayPortEnd int
+ heartbeatTimeout time.Duration
+ maxGatewaysPerPod int
+}
+
+var errRuntimeScaleOutPending = errors.New("runtime scale-out pending")
+
+const runtimeRolloutStaleWindowMultiplier = 3
+
+type RuntimeGatewayEnvBuilder func(*models.Instance) (map[string]string, error)
+
+type RuntimeSchedulerOption func(*RuntimeScheduler)
+
+func WithRuntimeSchedulerWorkspaceRoot(root string) RuntimeSchedulerOption {
+ return func(s *RuntimeScheduler) {
+ if strings.TrimSpace(root) != "" {
+ s.workspaceRoot = strings.TrimSpace(root)
+ }
+ }
+}
+
+func WithRuntimeSchedulerGatewayPortRange(start, end int) RuntimeSchedulerOption {
+ return func(s *RuntimeScheduler) {
+ if start > 0 {
+ s.gatewayPortStart = start
+ }
+ if end > 0 {
+ s.gatewayPortEnd = end
+ }
+ if s.gatewayPortEnd < s.gatewayPortStart {
+ s.gatewayPortEnd = s.gatewayPortStart
+ }
+ }
+}
+
+func WithRuntimeSchedulerHeartbeatTimeout(timeout time.Duration) RuntimeSchedulerOption {
+ return func(s *RuntimeScheduler) {
+ s.heartbeatTimeout = timeout
+ }
+}
+
+func WithRuntimeSchedulerNamespace(namespace string) RuntimeSchedulerOption {
+ return func(s *RuntimeScheduler) {
+ if strings.TrimSpace(namespace) != "" {
+ s.runtimeNamespace = strings.TrimSpace(namespace)
+ }
+ }
+}
+
+func WithRuntimeSchedulerMaxGatewaysPerPod(capacity int) RuntimeSchedulerOption {
+ return func(s *RuntimeScheduler) {
+ if capacity > 0 {
+ s.maxGatewaysPerPod = capacity
+ }
+ }
+}
+
+func WithRuntimeSchedulerGatewayEnvBuilder(builder RuntimeGatewayEnvBuilder) RuntimeSchedulerOption {
+ return func(s *RuntimeScheduler) {
+ s.envBuilder = builder
+ }
+}
+
+func NewRuntimeScheduler(
+ instanceRepo repository.InstanceRepository,
+ podRepo repository.RuntimePodRepository,
+ bindingRepo repository.InstanceRuntimeBindingRepository,
+ rolloutRepo repository.RuntimeRolloutRepository,
+ agentClient RuntimeAgentClient,
+ events RuntimeEventService,
+ leader RuntimeLeaderService,
+ deployments k8s.RuntimeDeploymentService,
+ tick time.Duration,
+ opts ...RuntimeSchedulerOption,
+) *RuntimeScheduler {
+ if tick <= 0 {
+ tick = 2 * time.Second
+ }
+ s := &RuntimeScheduler{
+ instanceRepo: instanceRepo,
+ podRepo: podRepo,
+ bindingRepo: bindingRepo,
+ rolloutRepo: rolloutRepo,
+ agentClient: agentClient,
+ events: events,
+ leader: leader,
+ deployments: deployments,
+ tick: tick,
+ workspaceRoot: "/workspaces",
+ runtimeNamespace: "clawmanager-system",
+ gatewayPortStart: RuntimeGatewayPortStart,
+ gatewayPortEnd: RuntimeGatewayPortEnd,
+ heartbeatTimeout: 10 * time.Second,
+ maxGatewaysPerPod: RuntimePodCapacity,
+ }
+ for _, opt := range opts {
+ opt(s)
+ }
+ return s
+}
+
+func (s *RuntimeScheduler) Start(ctx context.Context) {
+ if s == nil {
+ return
+ }
+ go func() {
+ ticker := time.NewTicker(s.tick)
+ defer ticker.Stop()
+
+ s.reconcileIfLeader(ctx)
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ s.reconcileIfLeader(ctx)
+ }
+ }
+ }()
+}
+
+func (s *RuntimeScheduler) HeartbeatTimeout() time.Duration {
+ if s == nil {
+ return 0
+ }
+ return s.heartbeatTimeout
+}
+
+func (s *RuntimeScheduler) DrainPod(ctx context.Context, podID int64) error {
+ if s == nil || s.podRepo == nil {
+ return fmt.Errorf("runtime scheduler pod repository is not configured")
+ }
+ pod, err := s.podRepo.GetByID(ctx, podID)
+ if err != nil {
+ return err
+ }
+ if pod == nil {
+ return fmt.Errorf("runtime pod %d not found", podID)
+ }
+ if pod.AgentEndpoint != nil && strings.TrimSpace(*pod.AgentEndpoint) != "" && s.agentClient != nil {
+ if err := s.agentClient.Drain(ctx, strings.TrimSpace(*pod.AgentEndpoint)); err != nil {
+ return err
+ }
+ }
+ return s.podRepo.MarkState(ctx, podID, "draining", true)
+}
+
+func (s *RuntimeScheduler) FailoverPod(ctx context.Context, podID int64, reason string) error {
+ if s == nil || s.podRepo == nil || s.bindingRepo == nil || s.instanceRepo == nil {
+ return fmt.Errorf("runtime scheduler dependencies are not configured")
+ }
+ var errs []error
+ if err := s.podRepo.MarkState(ctx, podID, "unhealthy", false); err != nil {
+ errs = append(errs, err)
+ }
+
+ bindings, err := s.bindingRepo.ListByRuntimePodID(ctx, podID)
+ if err != nil {
+ errs = append(errs, err)
+ return errors.Join(errs...)
+ }
+ for _, binding := range bindings {
+ nextGeneration := binding.Generation + 1
+ instance, err := s.instanceRepo.GetByID(binding.InstanceID)
+ if err != nil {
+ errs = append(errs, err)
+ continue
+ }
+ if instance != nil {
+ nextGeneration = instance.RuntimeGeneration + 1
+ }
+ message := reason
+ if err := s.instanceRepo.UpdateRuntimeState(ctx, binding.InstanceID, "creating", nextGeneration, &message); err != nil {
+ errs = append(errs, err)
+ continue
+ }
+ if err := s.bindingRepo.DeleteByInstanceID(ctx, binding.InstanceID); err != nil {
+ errs = append(errs, err)
+ continue
+ }
+ if err := s.podRepo.ReleaseSlot(ctx, podID); err != nil {
+ errs = append(errs, err)
+ }
+ }
+ return errors.Join(errs...)
+}
+
+func (s *RuntimeScheduler) StartRollout(ctx context.Context, rolloutID int64) error {
+ if s == nil || s.rolloutRepo == nil || s.podRepo == nil {
+ return fmt.Errorf("runtime scheduler rollout dependencies are not configured")
+ }
+ rollout, err := s.rolloutRepo.GetByID(ctx, rolloutID)
+ if err != nil {
+ return err
+ }
+ if rollout == nil {
+ return fmt.Errorf("runtime rollout %d not found", rolloutID)
+ }
+
+ startedAt := time.Now().UTC()
+ if err := s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "running", &startedAt, nil, nil); err != nil {
+ return err
+ }
+
+ allPods, err := s.podRepo.List(ctx, rollout.RuntimeType)
+ if err != nil {
+ message := err.Error()
+ _ = s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "error", &startedAt, nil, &message)
+ return err
+ }
+ currentPods := s.currentRuntimePods(allPods, time.Now().UTC())
+ if runtimePodsAlreadyAtImage(currentPods, rollout.RuntimeType, rollout.TargetImageRef) {
+ finishedAt := time.Now().UTC()
+ return s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "finished", &startedAt, &finishedAt, nil)
+ }
+ batchSize := rollout.BatchSize
+ if batchSize <= 0 {
+ batchSize = 1
+ }
+ maxUnavailable := rollout.MaxUnavailable
+ if maxUnavailable <= 0 {
+ maxUnavailable = 1
+ }
+ if err := s.rolloutRuntimeDeployments(ctx, rollout, allPods, maxUnavailable, batchSize); err != nil {
+ message := err.Error()
+ _ = s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "error", &startedAt, nil, &message)
+ return err
+ }
+ unavailable := 0
+ readyCandidates := 0
+ for _, pod := range currentPods {
+ if pod.Draining || pod.State != "ready" {
+ unavailable++
+ continue
+ }
+ readyCandidates++
+ }
+ remaining := maxUnavailable - unavailable
+ if remaining <= 0 {
+ return nil
+ }
+ drainLimit := minInt(batchSize, remaining)
+ var errs []error
+ drained := 0
+ for _, pod := range currentPods {
+ if drained >= drainLimit {
+ break
+ }
+ if pod.State != "ready" || pod.Draining {
+ continue
+ }
+ if err := s.DrainPod(ctx, pod.ID); err != nil {
+ errs = append(errs, err)
+ continue
+ }
+ drained++
+ }
+ if len(errs) > 0 {
+ joined := errors.Join(errs...)
+ message := joined.Error()
+ _ = s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "error", &startedAt, nil, &message)
+ return joined
+ }
+ if drained == 0 && readyCandidates == 0 && unavailable == 0 && len(currentPods) > 0 {
+ finishedAt := time.Now().UTC()
+ return s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "finished", &startedAt, &finishedAt, nil)
+ }
+ return nil
+}
+
+func (s *RuntimeScheduler) reconcileRollouts(ctx context.Context) error {
+ if s == nil || s.rolloutRepo == nil {
+ return nil
+ }
+ rollouts, err := s.rolloutRepo.ListActive(ctx, "")
+ if err != nil {
+ return fmt.Errorf("list active runtime rollouts: %w", err)
+ }
+ var errs []error
+ for _, rollout := range rollouts {
+ switch rollout.Status {
+ case "pending":
+ if err := s.StartRollout(ctx, rollout.ID); err != nil {
+ errs = append(errs, fmt.Errorf("start runtime rollout %d: %w", rollout.ID, err))
+ }
+ case "running":
+ if err := s.finishRolloutIfReady(ctx, rollout); err != nil {
+ errs = append(errs, fmt.Errorf("finish runtime rollout %d: %w", rollout.ID, err))
+ }
+ }
+ }
+ return errors.Join(errs...)
+}
+
+func (s *RuntimeScheduler) rolloutRuntimeDeployments(ctx context.Context, rollout *models.RuntimeRollout, pods []models.RuntimePod, maxUnavailable, maxSurge int) error {
+ if s == nil || s.deployments == nil || rollout == nil {
+ return nil
+ }
+ targetImage := strings.TrimSpace(rollout.TargetImageRef)
+ if targetImage == "" {
+ return nil
+ }
+ type deploymentRef struct {
+ namespace string
+ name string
+ }
+ refs := map[deploymentRef]struct{}{}
+ for _, pod := range pods {
+ if pod.RuntimeType != rollout.RuntimeType {
+ continue
+ }
+ namespace := strings.TrimSpace(pod.Namespace)
+ name := strings.TrimSpace(pod.DeploymentName)
+ if namespace == "" || name == "" {
+ continue
+ }
+ refs[deploymentRef{namespace: namespace, name: name}] = struct{}{}
+ }
+ if len(refs) == 0 {
+ name := defaultRuntimeDeploymentName(rollout.RuntimeType)
+ if name == "" {
+ return fmt.Errorf("no runtime deployment found for %s rollout", rollout.RuntimeType)
+ }
+ refs[deploymentRef{namespace: s.runtimeNamespace, name: name}] = struct{}{}
+ }
+ var errs []error
+ for ref := range refs {
+ if err := s.deployments.RolloutImage(ctx, ref.namespace, ref.name, targetImage, maxUnavailable, maxSurge); err != nil {
+ errs = append(errs, err)
+ }
+ }
+ return errors.Join(errs...)
+}
+
+func (s *RuntimeScheduler) RuntimeDeploymentPods(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) {
+ if s == nil || s.deployments == nil {
+ return nil, nil
+ }
+ runtimeType = strings.TrimSpace(runtimeType)
+ pods, err := s.deployments.ListPods(ctx, s.runtimeNamespace, runtimeType)
+ if err != nil {
+ return nil, err
+ }
+ result := make([]models.RuntimePod, 0, len(pods))
+ for index, pod := range pods {
+ result = append(result, models.RuntimePod{
+ ID: -int64(index + 1),
+ RuntimeType: pod.RuntimeType,
+ Namespace: pod.Namespace,
+ PodName: pod.PodName,
+ PodIP: pod.PodIP,
+ NodeName: pod.NodeName,
+ DeploymentName: pod.DeploymentName,
+ ImageRef: pod.ImageRef,
+ State: runtimeDeploymentFallbackState(pod.State),
+ Capacity: s.maxGatewaysPerPod,
+ UsedSlots: 0,
+ Draining: false,
+ })
+ }
+ return result, nil
+}
+
+func defaultRuntimeDeploymentName(runtimeType string) string {
+ switch strings.ToLower(strings.TrimSpace(runtimeType)) {
+ case RuntimeTypeOpenClaw:
+ return "openclaw-runtime"
+ case RuntimeTypeHermes:
+ return "hermes-runtime"
+ default:
+ return ""
+ }
+}
+
+func runtimeDeploymentFallbackState(k8sState string) string {
+ switch strings.TrimSpace(k8sState) {
+ case "pending", "deleted":
+ return strings.TrimSpace(k8sState)
+ default:
+ return "unhealthy"
+ }
+}
+
+func (s *RuntimeScheduler) finishRolloutIfReady(ctx context.Context, rollout models.RuntimeRollout) error {
+ if s == nil || s.podRepo == nil || s.rolloutRepo == nil {
+ return nil
+ }
+ targetImage := strings.TrimSpace(rollout.TargetImageRef)
+ if targetImage == "" {
+ return nil
+ }
+ pods, err := s.podRepo.List(ctx, rollout.RuntimeType)
+ if err != nil {
+ return err
+ }
+ pods = s.currentRuntimePods(pods, time.Now().UTC())
+ if len(pods) == 0 {
+ return nil
+ }
+ for _, pod := range pods {
+ if pod.RuntimeType != rollout.RuntimeType {
+ continue
+ }
+ if pod.State != "ready" || pod.Draining || strings.TrimSpace(pod.ImageRef) != targetImage {
+ return nil
+ }
+ }
+ finishedAt := time.Now().UTC()
+ return s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "finished", rollout.StartedAt, &finishedAt, nil)
+}
+
+func (s *RuntimeScheduler) currentRuntimePods(pods []models.RuntimePod, now time.Time) []models.RuntimePod {
+ if s == nil || s.heartbeatTimeout <= 0 {
+ return pods
+ }
+ cutoff := now.UTC().Add(-runtimeRolloutStaleWindowMultiplier * s.heartbeatTimeout)
+ current := pods[:0]
+ for _, pod := range pods {
+ if pod.LastSeenAt != nil && pod.LastSeenAt.UTC().Before(cutoff) {
+ continue
+ }
+ current = append(current, pod)
+ }
+ return current
+}
+
+func runtimePodsAlreadyAtImage(pods []models.RuntimePod, runtimeType, targetImage string) bool {
+ targetImage = strings.TrimSpace(targetImage)
+ if targetImage == "" {
+ return false
+ }
+ matched := 0
+ for _, pod := range pods {
+ if pod.RuntimeType != runtimeType {
+ continue
+ }
+ matched++
+ if pod.State != "ready" || pod.Draining || strings.TrimSpace(pod.ImageRef) != targetImage {
+ return false
+ }
+ }
+ return matched > 0
+}
+
+func (s *RuntimeScheduler) reconcileIfLeader(ctx context.Context) {
+ if s.leader != nil && !s.leader.IsLeader(ctx) {
+ return
+ }
+ if err := s.reconcile(ctx); err != nil {
+ log.Printf("runtime scheduler reconcile failed: %v", err)
+ }
+}
+
+func (s *RuntimeScheduler) reconcile(ctx context.Context) error {
+ if s == nil || s.instanceRepo == nil || s.bindingRepo == nil || s.podRepo == nil {
+ return fmt.Errorf("runtime scheduler dependencies are not configured")
+ }
+ var errs []error
+ if err := s.failoverStalePods(ctx); err != nil {
+ errs = append(errs, err)
+ }
+ if err := s.reconcileRollouts(ctx); err != nil {
+ errs = append(errs, err)
+ }
+ creating, err := s.instanceRepo.GetV2Creating(ctx, 100)
+ if err != nil {
+ errs = append(errs, err)
+ } else {
+ for _, instance := range creating {
+ if !isSchedulerManagedV2Instance(instance) {
+ continue
+ }
+ binding, err := s.bindingRepo.GetByInstanceID(ctx, instance.ID)
+ if err != nil {
+ errs = append(errs, fmt.Errorf("get binding for creating instance %d: %w", instance.ID, err))
+ continue
+ }
+ if binding != nil {
+ continue
+ }
+ if err := s.assignInstance(ctx, instance); err != nil {
+ if errors.Is(err, errRuntimeScaleOutPending) {
+ continue
+ }
+ errs = append(errs, fmt.Errorf("assign creating instance %d: %w", instance.ID, err))
+ s.markInstanceError(ctx, instance, err, &errs)
+ }
+ }
+ }
+
+ desired, err := s.instanceRepo.GetV2DesiredRunning(ctx, 100)
+ if err != nil {
+ errs = append(errs, err)
+ } else {
+ for _, instance := range desired {
+ if !isSchedulerManagedV2Instance(instance) {
+ continue
+ }
+ if isRuntimeErrorInstance(instance) && !isRecoverableRuntimeSchedulingError(instance) {
+ continue
+ }
+ binding, err := s.bindingRepo.GetRunningByInstanceID(ctx, instance.ID)
+ if err != nil {
+ errs = append(errs, fmt.Errorf("get running binding for instance %d: %w", instance.ID, err))
+ continue
+ }
+ if binding != nil {
+ continue
+ }
+ binding, err = s.bindingRepo.GetByInstanceID(ctx, instance.ID)
+ if err != nil {
+ errs = append(errs, fmt.Errorf("get binding for desired instance %d: %w", instance.ID, err))
+ continue
+ }
+ if binding != nil {
+ continue
+ }
+ if assignErr := s.assignInstance(ctx, instance); assignErr != nil {
+ if errors.Is(assignErr, errRuntimeScaleOutPending) {
+ continue
+ }
+ errs = append(errs, fmt.Errorf("assign desired instance %d: %w", instance.ID, assignErr))
+ s.markInstanceError(ctx, instance, assignErr, &errs)
+ }
+ }
+ }
+ return errors.Join(errs...)
+}
+
+func (s *RuntimeScheduler) failoverStalePods(ctx context.Context) error {
+ if s.heartbeatTimeout <= 0 {
+ return nil
+ }
+ pods, err := s.podRepo.List(ctx, "")
+ if err != nil {
+ return fmt.Errorf("list runtime pods for heartbeat failover: %w", err)
+ }
+ cutoff := time.Now().UTC().Add(-s.heartbeatTimeout)
+ var errs []error
+ for _, pod := range pods {
+ if pod.State == "unhealthy" || pod.State == "pending" {
+ continue
+ }
+ if pod.LastSeenAt == nil || !pod.LastSeenAt.Before(cutoff) {
+ continue
+ }
+ reason := fmt.Sprintf("runtime pod heartbeat lost since %s", pod.LastSeenAt.UTC().Format(time.RFC3339))
+ if err := s.FailoverPod(ctx, pod.ID, reason); err != nil {
+ errs = append(errs, fmt.Errorf("failover stale runtime pod %d: %w", pod.ID, err))
+ }
+ }
+ return errors.Join(errs...)
+}
+
+func (s *RuntimeScheduler) assignInstance(ctx context.Context, instance models.Instance) error {
+ if s == nil || s.podRepo == nil {
+ return fmt.Errorf("runtime scheduler pod repository is not configured")
+ }
+ if !isSchedulerManagedV2Instance(instance) {
+ return fmt.Errorf("instance %d is not scheduler-managed v2", instance.ID)
+ }
+ if s.bindingRepo != nil {
+ binding, err := s.bindingRepo.GetByInstanceID(ctx, instance.ID)
+ if err != nil {
+ return fmt.Errorf("failed to check existing binding: %w", err)
+ }
+ if binding != nil {
+ return nil
+ }
+ }
+ runtimeType, ok := schedulerRuntimeType(instance)
+ if !ok {
+ return fmt.Errorf("unsupported runtime type %q", instance.Type)
+ }
+ pods, err := s.podRepo.ListSchedulable(ctx, runtimeType)
+ if err != nil {
+ return err
+ }
+ if len(pods) == 0 {
+ scaled, err := s.scaleOutIfAtCapacity(ctx, runtimeType)
+ if err != nil {
+ return fmt.Errorf("scale out %s runtime deployment: %w", runtimeType, err)
+ }
+ if scaled {
+ return errRuntimeScaleOutPending
+ }
+ }
+ var lastErr error
+ for _, pod := range pods {
+ if pod.AgentEndpoint == nil || strings.TrimSpace(*pod.AgentEndpoint) == "" {
+ continue
+ }
+ claimed, err := s.podRepo.TryClaimSlot(ctx, pod.ID)
+ if err != nil {
+ lastErr = err
+ continue
+ }
+ if !claimed {
+ continue
+ }
+ if err := s.createGatewayOnPod(ctx, instance, runtimeType, pod); err != nil {
+ if releaseErr := s.podRepo.ReleaseSlot(ctx, pod.ID); releaseErr != nil {
+ return errors.Join(err, releaseErr)
+ }
+ lastErr = err
+ continue
+ }
+ return nil
+ }
+ if lastErr != nil {
+ return fmt.Errorf("no schedulable %s runtime pod: %w", runtimeType, lastErr)
+ }
+ return fmt.Errorf("no schedulable %s runtime pod", runtimeType)
+}
+
+type runtimeDeploymentCapacity struct {
+ namespace string
+ name string
+ active int
+ full int
+}
+
+func (s *RuntimeScheduler) scaleOutIfAtCapacity(ctx context.Context, runtimeType string) (bool, error) {
+ if s == nil || s.deployments == nil || s.podRepo == nil {
+ return false, nil
+ }
+ pods, err := s.podRepo.List(ctx, runtimeType)
+ if err != nil {
+ return false, fmt.Errorf("list %s runtime pods for scale-out: %w", runtimeType, err)
+ }
+ groups := map[string]*runtimeDeploymentCapacity{}
+ for _, pod := range pods {
+ if pod.RuntimeType != runtimeType || pod.State != "ready" || pod.Draining || pod.Capacity <= 0 {
+ continue
+ }
+ namespace := strings.TrimSpace(pod.Namespace)
+ name := strings.TrimSpace(pod.DeploymentName)
+ if namespace == "" || name == "" {
+ continue
+ }
+ key := namespace + "/" + name
+ group := groups[key]
+ if group == nil {
+ group = &runtimeDeploymentCapacity{namespace: namespace, name: name}
+ groups[key] = group
+ }
+ group.active++
+ if pod.UsedSlots >= pod.Capacity {
+ group.full++
+ }
+ }
+ var target *runtimeDeploymentCapacity
+ for _, group := range groups {
+ if group.active == 0 || group.active != group.full {
+ continue
+ }
+ if target == nil || group.active > target.active {
+ target = group
+ }
+ }
+ if target == nil {
+ return false, nil
+ }
+ replicas := int32(target.active + 1)
+ if err := s.deployments.Scale(ctx, target.namespace, target.name, replicas); err != nil {
+ return false, err
+ }
+ if s.events != nil {
+ if err := s.events.Publish(ctx, "runtime.pool.scaleout", map[string]any{
+ "runtime_type": runtimeType,
+ "namespace": target.namespace,
+ "deployment": target.name,
+ "replicas": replicas,
+ }); err != nil {
+ log.Printf("runtime scheduler publish scale-out event failed: %v", err)
+ }
+ }
+ return true, nil
+}
+
+func (s *RuntimeScheduler) createGatewayOnPod(ctx context.Context, instance models.Instance, runtimeType string, pod models.RuntimePod) error {
+ if s.agentClient == nil || s.bindingRepo == nil || s.instanceRepo == nil {
+ return fmt.Errorf("runtime scheduler gateway dependencies are not configured")
+ }
+ endpoint := strings.TrimSpace(*pod.AgentEndpoint)
+ workspacePath := RuntimeWorkspacePathWithRoot(s.workspaceRoot, runtimeType, instance.UserID, instance.ID)
+ environment, err := s.gatewayEnvironment(&instance)
+ if err != nil {
+ return fmt.Errorf("build runtime gateway environment: %w", err)
+ }
+ uid, gid := runtimeGatewayLinuxIDs(instance.ID, environment)
+ resp, err := s.agentClient.CreateGateway(ctx, endpoint, RuntimeAgentCreateGatewayRequest{
+ InstanceID: instance.ID,
+ UserID: instance.UserID,
+ AgentType: runtimeType,
+ WorkspacePath: workspacePath,
+ PortRange: RuntimeAgentPortRange{
+ Start: s.gatewayPortStart,
+ End: s.gatewayPortEnd,
+ },
+ UID: uid,
+ GID: gid,
+ CPUCores: instance.CPUCores,
+ MemoryMB: instance.MemoryGB * 1024,
+ DiskQuotaMB: instance.DiskGB * 1024,
+ Generation: instance.RuntimeGeneration,
+ Environment: environment,
+ })
+ if err != nil {
+ return err
+ }
+ if resp == nil {
+ return fmt.Errorf("runtime agent returned empty gateway response")
+ }
+
+ now := time.Now().UTC()
+ binding := &models.InstanceRuntimeBinding{
+ InstanceID: instance.ID,
+ RuntimePodID: pod.ID,
+ RuntimeType: runtimeType,
+ GatewayID: resp.GatewayID,
+ GatewayPort: resp.Port,
+ GatewayPID: resp.PID,
+ WorkspacePath: workspacePath,
+ State: "running",
+ Generation: instance.RuntimeGeneration,
+ LastHealthAt: &now,
+ }
+ if err := s.bindingRepo.Create(ctx, binding); err != nil {
+ return s.cleanupGatewayAfterAssignFailure(ctx, endpoint, instance.ID, resp.GatewayID, false, err)
+ }
+ if err := s.instanceRepo.SetWorkspacePath(ctx, instance.ID, workspacePath); err != nil {
+ return s.cleanupGatewayAfterAssignFailure(ctx, endpoint, instance.ID, resp.GatewayID, true, err)
+ }
+ if err := s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "running", instance.RuntimeGeneration, nil); err != nil {
+ return s.cleanupGatewayAfterAssignFailure(ctx, endpoint, instance.ID, resp.GatewayID, true, err)
+ }
+ if s.events != nil {
+ if err := s.events.Publish(ctx, "runtime.instance.running", map[string]any{
+ "instance_id": instance.ID,
+ "runtime_type": runtimeType,
+ "runtime_pod_id": pod.ID,
+ "gateway_id": resp.GatewayID,
+ "gateway_port": resp.Port,
+ "workspace_path": workspacePath,
+ "generation": instance.RuntimeGeneration,
+ }); err != nil {
+ log.Printf("runtime scheduler publish event failed: %v", err)
+ }
+ }
+ return nil
+}
+
+func runtimeGatewayLinuxIDs(instanceID int, environment map[string]string) (int, int) {
+ linuxID := RuntimeLinuxID(instanceID)
+ if !strings.EqualFold(strings.TrimSpace(environment["CLAWMANAGER_TEAM_ENABLED"]), "true") {
+ return linuxID, linuxID
+ }
+ sharedGID, err := strconv.Atoi(strings.TrimSpace(environment["CLAWMANAGER_TEAM_SHARED_GID"]))
+ if err != nil || sharedGID <= 0 {
+ return linuxID, linuxID
+ }
+ return linuxID, sharedGID
+}
+
+func (s *RuntimeScheduler) gatewayEnvironment(instance *models.Instance) (map[string]string, error) {
+ if s == nil || s.envBuilder == nil {
+ return nil, nil
+ }
+ env, err := s.envBuilder(instance)
+ if err != nil {
+ return nil, err
+ }
+ if len(env) == 0 {
+ return nil, nil
+ }
+ copied := make(map[string]string, len(env))
+ for key, value := range env {
+ if strings.TrimSpace(key) != "" {
+ copied[key] = value
+ }
+ }
+ if len(copied) == 0 {
+ return nil, nil
+ }
+ return copied, nil
+}
+
+func (s *RuntimeScheduler) cleanupGatewayAfterAssignFailure(ctx context.Context, endpoint string, instanceID int, gatewayID string, bindingCreated bool, cause error) error {
+ errs := []error{cause}
+ if gatewayID != "" && s.agentClient != nil {
+ if err := s.agentClient.DeleteGateway(ctx, endpoint, gatewayID); err != nil {
+ errs = append(errs, fmt.Errorf("delete gateway %s: %w", gatewayID, err))
+ }
+ }
+ if bindingCreated && s.bindingRepo != nil {
+ if err := s.bindingRepo.DeleteByInstanceID(ctx, instanceID); err != nil {
+ errs = append(errs, fmt.Errorf("delete binding for instance %d: %w", instanceID, err))
+ }
+ }
+ return errors.Join(errs...)
+}
+
+func (s *RuntimeScheduler) markInstanceError(ctx context.Context, instance models.Instance, cause error, errs *[]error) {
+ message := cause.Error()
+ if err := s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "error", instance.RuntimeGeneration, &message); err != nil {
+ *errs = append(*errs, fmt.Errorf("mark instance %d error: %w", instance.ID, err))
+ }
+}
+
+func schedulerRuntimeType(instance models.Instance) (string, bool) {
+ return NormalizeV2RuntimeType(instance.Type)
+}
+
+func isSchedulerManagedV2Instance(instance models.Instance) bool {
+ if _, ok := schedulerRuntimeType(instance); !ok {
+ return false
+ }
+ if normalizeInstanceRuntimeType(instance.RuntimeType) != RuntimeBackendGateway {
+ return false
+ }
+ if mode, ok := NormalizeInstanceMode(instance.InstanceMode); ok && mode != InstanceModeLite {
+ return false
+ }
+ return instance.WorkspacePath != nil && strings.TrimSpace(*instance.WorkspacePath) != ""
+}
+
+func isRuntimeErrorInstance(instance models.Instance) bool {
+ return strings.EqualFold(strings.TrimSpace(instance.Status), "error")
+}
+
+func isRecoverableRuntimeSchedulingError(instance models.Instance) bool {
+ if !isRuntimeErrorInstance(instance) || instance.RuntimeErrorMessage == nil {
+ return false
+ }
+ runtimeType, ok := schedulerRuntimeType(instance)
+ if !ok {
+ return false
+ }
+ message := strings.TrimSpace(*instance.RuntimeErrorMessage)
+ return message == fmt.Sprintf("no schedulable %s runtime pod", runtimeType)
+}
+
+func minInt(a, b int) int {
+ if a < b {
+ return a
+ }
+ return b
+}
diff --git a/backend/internal/services/runtime_scheduler_test.go b/backend/internal/services/runtime_scheduler_test.go
new file mode 100644
index 0000000..89c02ff
--- /dev/null
+++ b/backend/internal/services/runtime_scheduler_test.go
@@ -0,0 +1,1840 @@
+package services
+
+import (
+ "context"
+ "errors"
+ "strings"
+ "testing"
+ "time"
+
+ "clawreef/internal/models"
+ "clawreef/internal/repository"
+ "clawreef/internal/services/k8s"
+)
+
+func TestRuntimeSchedulerAssignsCreatingInstanceToReadyPod(t *testing.T) {
+ ctx := context.Background()
+ endpoint := "http://agent.runtime"
+ workspacePath := "/workspaces/openclaw/user-45/instance-17"
+ instanceRepo := newFakeRuntimeInstanceRepo()
+ instanceRepo.creating = []models.Instance{{
+ ID: 17,
+ UserID: 45,
+ Type: RuntimeTypeOpenClaw,
+ RuntimeType: RuntimeBackendGateway,
+ InstanceMode: InstanceModeLite,
+ Status: "creating",
+ CPUCores: 1.5,
+ MemoryGB: 2,
+ DiskGB: 8,
+ WorkspacePath: &workspacePath,
+ RuntimeGeneration: 3,
+ }}
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 9: {
+ ID: 9,
+ RuntimeType: RuntimeTypeOpenClaw,
+ AgentEndpoint: &endpoint,
+ State: "ready",
+ Capacity: 1,
+ },
+ },
+ schedulable: []models.RuntimePod{{
+ ID: 9,
+ RuntimeType: RuntimeTypeOpenClaw,
+ AgentEndpoint: &endpoint,
+ State: "ready",
+ Capacity: 1,
+ }},
+ }
+ bindingRepo := newFakeRuntimeBindingRepo()
+ agent := &fakeRuntimeAgentClient{
+ createResponse: &RuntimeAgentCreateGatewayResponse{
+ GatewayID: "gw-17",
+ Port: 20017,
+ Status: "running",
+ },
+ }
+ events := &fakeRuntimeEventService{}
+ scheduler := NewRuntimeScheduler(
+ instanceRepo,
+ podRepo,
+ bindingRepo,
+ &fakeRuntimeRolloutRepo{},
+ agent,
+ events,
+ nil,
+ &fakeRuntimeDeploymentService{},
+ time.Second,
+ )
+
+ if err := scheduler.reconcile(ctx); err != nil {
+ t.Fatalf("reconcile returned error: %v", err)
+ }
+
+ if got := len(agent.createRequests); got != 1 {
+ t.Fatalf("CreateGateway calls = %d, want 1", got)
+ }
+ req := agent.createRequests[0]
+ if req.endpoint != endpoint {
+ t.Fatalf("CreateGateway endpoint = %q, want %q", req.endpoint, endpoint)
+ }
+ if req.req.WorkspacePath != "/workspaces/openclaw/user-45/instance-17" {
+ t.Fatalf("workspace path = %q", req.req.WorkspacePath)
+ }
+ if req.req.PortRange.Start != 20000 || req.req.PortRange.End != 20099 {
+ t.Fatalf("port range = %+v", req.req.PortRange)
+ }
+ if req.req.UID != RuntimeLinuxID(17) || req.req.GID != RuntimeLinuxID(17) {
+ t.Fatalf("linux IDs = %d/%d", req.req.UID, req.req.GID)
+ }
+ if req.req.MemoryMB != 2048 || req.req.DiskQuotaMB != 8192 {
+ t.Fatalf("resource MB = %d/%d", req.req.MemoryMB, req.req.DiskQuotaMB)
+ }
+
+ binding := bindingRepo.bindings[17]
+ if binding == nil {
+ t.Fatal("binding was not created")
+ }
+ if binding.RuntimePodID != 9 || binding.GatewayPort != 20017 {
+ t.Fatalf("binding pod/port = %d/%d", binding.RuntimePodID, binding.GatewayPort)
+ }
+ if instanceRepo.workspacePaths[17] != "/workspaces/openclaw/user-45/instance-17" {
+ t.Fatalf("instance workspace path = %q", instanceRepo.workspacePaths[17])
+ }
+ state := instanceRepo.runtimeStates[17]
+ if state.status != "running" || state.generation != 3 {
+ t.Fatalf("runtime state = %+v", state)
+ }
+ if podRepo.claims[9] != 1 {
+ t.Fatalf("pod claims = %d, want 1", podRepo.claims[9])
+ }
+ if got := len(events.published); got != 1 {
+ t.Fatalf("published events = %d, want 1", got)
+ }
+}
+
+func TestRuntimeSchedulerPassesGatewayEnvironmentToRuntimeAgent(t *testing.T) {
+ ctx := context.Background()
+ endpoint := "http://agent.runtime"
+ token := "igt_instance_68"
+ workspacePath := "/workspaces/openclaw/user-1/instance-68"
+ instanceRepo := newFakeRuntimeInstanceRepo()
+ instanceRepo.creating = []models.Instance{{
+ ID: 68,
+ UserID: 1,
+ Type: RuntimeTypeOpenClaw,
+ RuntimeType: RuntimeBackendGateway,
+ InstanceMode: InstanceModeLite,
+ Status: "creating",
+ AccessToken: &token,
+ WorkspacePath: &workspacePath,
+ RuntimeGeneration: 4,
+ }}
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 10: {
+ ID: 10,
+ RuntimeType: RuntimeTypeOpenClaw,
+ AgentEndpoint: &endpoint,
+ State: "ready",
+ Capacity: 10,
+ },
+ },
+ schedulable: []models.RuntimePod{{
+ ID: 10,
+ RuntimeType: RuntimeTypeOpenClaw,
+ AgentEndpoint: &endpoint,
+ State: "ready",
+ Capacity: 10,
+ }},
+ }
+ agent := &fakeRuntimeAgentClient{
+ createResponse: &RuntimeAgentCreateGatewayResponse{
+ GatewayID: "gw-68-4",
+ Port: 20068,
+ Status: "starting",
+ },
+ }
+ scheduler := NewRuntimeScheduler(
+ instanceRepo,
+ podRepo,
+ newFakeRuntimeBindingRepo(),
+ &fakeRuntimeRolloutRepo{},
+ agent,
+ NewRuntimeEventService(nil),
+ nil,
+ &fakeRuntimeDeploymentService{},
+ time.Second,
+ WithRuntimeSchedulerGatewayEnvBuilder(func(instance *models.Instance) (map[string]string, error) {
+ if instance == nil || instance.ID != 68 {
+ t.Fatalf("gateway env builder received instance %#v, want 68", instance)
+ }
+ return map[string]string{
+ "CLAWMANAGER_LLM_BASE_URL": "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001/api/v1/gateway/llm",
+ "CLAWMANAGER_LLM_API_KEY": token,
+ "CLAWMANAGER_LLM_MODEL": `["auto","gpt-5.5"]`,
+ "OPENAI_API_KEY": token,
+ }, nil
+ }),
+ )
+
+ if err := scheduler.reconcile(ctx); err != nil {
+ t.Fatalf("reconcile returned error: %v", err)
+ }
+
+ if got := len(agent.createRequests); got != 1 {
+ t.Fatalf("CreateGateway calls = %d, want 1", got)
+ }
+ env := agent.createRequests[0].req.Environment
+ if env["CLAWMANAGER_LLM_API_KEY"] != token || env["OPENAI_API_KEY"] != token {
+ t.Fatalf("gateway environment missing instance token aliases: %#v", env)
+ }
+ if env["CLAWMANAGER_LLM_BASE_URL"] != "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001/api/v1/gateway/llm" {
+ t.Fatalf("CLAWMANAGER_LLM_BASE_URL = %q", env["CLAWMANAGER_LLM_BASE_URL"])
+ }
+ if env["CLAWMANAGER_LLM_MODEL"] != `["auto","gpt-5.5"]` {
+ t.Fatalf("CLAWMANAGER_LLM_MODEL = %q", env["CLAWMANAGER_LLM_MODEL"])
+ }
+}
+
+func TestRuntimeSchedulerUsesTeamSharedGIDForTeamLiteGateway(t *testing.T) {
+ ctx := context.Background()
+ endpoint := "http://agent.runtime"
+ workspacePath := "/workspaces/openclaw/user-1/instance-130"
+ instanceRepo := newFakeRuntimeInstanceRepo()
+ instanceRepo.creating = []models.Instance{{
+ ID: 130,
+ UserID: 1,
+ Type: RuntimeTypeOpenClaw,
+ RuntimeType: RuntimeBackendGateway,
+ InstanceMode: InstanceModeLite,
+ Status: "creating",
+ WorkspacePath: &workspacePath,
+ RuntimeGeneration: 1,
+ MemoryGB: 1,
+ DiskGB: 1,
+ }}
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 11: {
+ ID: 11,
+ RuntimeType: RuntimeTypeOpenClaw,
+ AgentEndpoint: &endpoint,
+ State: "ready",
+ Capacity: 10,
+ },
+ },
+ schedulable: []models.RuntimePod{{
+ ID: 11,
+ RuntimeType: RuntimeTypeOpenClaw,
+ AgentEndpoint: &endpoint,
+ State: "ready",
+ Capacity: 10,
+ }},
+ }
+ agent := &fakeRuntimeAgentClient{
+ createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-130-1", Port: 20030, Status: "running"},
+ }
+ scheduler := NewRuntimeScheduler(
+ instanceRepo,
+ podRepo,
+ newFakeRuntimeBindingRepo(),
+ &fakeRuntimeRolloutRepo{},
+ agent,
+ NewRuntimeEventService(nil),
+ nil,
+ &fakeRuntimeDeploymentService{},
+ time.Second,
+ WithRuntimeSchedulerGatewayEnvBuilder(func(instance *models.Instance) (map[string]string, error) {
+ return map[string]string{
+ "CLAWMANAGER_TEAM_ENABLED": "true",
+ "CLAWMANAGER_TEAM_SHARED_GID": "1000",
+ }, nil
+ }),
+ )
+
+ if err := scheduler.reconcile(ctx); err != nil {
+ t.Fatalf("reconcile returned error: %v", err)
+ }
+ if got := len(agent.createRequests); got != 1 {
+ t.Fatalf("CreateGateway calls = %d, want 1", got)
+ }
+ req := agent.createRequests[0].req
+ if req.UID != RuntimeLinuxID(130) {
+ t.Fatalf("UID = %d, want isolated instance UID %d", req.UID, RuntimeLinuxID(130))
+ }
+ if req.GID != 1000 {
+ t.Fatalf("GID = %d, want Team shared GID 1000", req.GID)
+ }
+}
+
+func TestRuntimeSchedulerNoSchedulablePodReturnsErrorAndReleasesNothing(t *testing.T) {
+ ctx := context.Background()
+ instanceRepo := newFakeRuntimeInstanceRepo()
+ podRepo := &fakeRuntimePodRepo{}
+ scheduler := NewRuntimeScheduler(
+ instanceRepo,
+ podRepo,
+ newFakeRuntimeBindingRepo(),
+ &fakeRuntimeRolloutRepo{},
+ &fakeRuntimeAgentClient{},
+ NewRuntimeEventService(nil),
+ nil,
+ &fakeRuntimeDeploymentService{},
+ time.Second,
+ )
+
+ err := scheduler.assignInstance(ctx, models.Instance{
+ ID: 1,
+ UserID: 2,
+ Type: RuntimeTypeOpenClaw,
+ RuntimeType: RuntimeBackendGateway,
+ InstanceMode: InstanceModeLite,
+ WorkspacePath: ptrString("/workspaces/openclaw/user-2/instance-1"),
+ })
+ if err == nil {
+ t.Fatal("assignInstance returned nil error")
+ }
+ if podRepo.releaseCount != 0 {
+ t.Fatalf("release count = %d, want 0", podRepo.releaseCount)
+ }
+}
+
+func TestRuntimeSchedulerScalesOutWhenAllReadyPodsAtCapacity(t *testing.T) {
+ ctx := context.Background()
+ endpoint := "http://pod-1.runtime"
+ workspacePath := "/workspaces/openclaw/user-12/instance-27"
+ instanceRepo := newFakeRuntimeInstanceRepo()
+ instanceRepo.creating = []models.Instance{{
+ ID: 27,
+ UserID: 12,
+ Type: RuntimeTypeOpenClaw,
+ RuntimeType: RuntimeBackendGateway,
+ InstanceMode: InstanceModeLite,
+ Status: "creating",
+ WorkspacePath: &workspacePath,
+ RuntimeGeneration: 2,
+ }}
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 5: {
+ ID: 5,
+ RuntimeType: RuntimeTypeOpenClaw,
+ Namespace: "clawmanager-system",
+ PodName: "openclaw-runtime-abc",
+ DeploymentName: "openclaw-runtime",
+ AgentEndpoint: &endpoint,
+ State: "ready",
+ Capacity: 100,
+ UsedSlots: 100,
+ },
+ },
+ }
+ deployments := &fakeRuntimeDeploymentService{}
+ agent := &fakeRuntimeAgentClient{}
+ scheduler := NewRuntimeScheduler(
+ instanceRepo,
+ podRepo,
+ newFakeRuntimeBindingRepo(),
+ &fakeRuntimeRolloutRepo{},
+ agent,
+ NewRuntimeEventService(nil),
+ nil,
+ deployments,
+ time.Second,
+ )
+
+ if err := scheduler.reconcile(ctx); err != nil {
+ t.Fatalf("reconcile returned error: %v", err)
+ }
+ if got := len(deployments.scales); got != 1 {
+ t.Fatalf("deployment Scale calls = %d, want 1", got)
+ }
+ scale := deployments.scales[0]
+ if scale.namespace != "clawmanager-system" || scale.name != "openclaw-runtime" || scale.replicas != 2 {
+ t.Fatalf("deployment Scale call = %+v, want clawmanager-system/openclaw-runtime replicas 2", scale)
+ }
+ if got := len(agent.createRequests); got != 0 {
+ t.Fatalf("CreateGateway calls = %d, want 0 while waiting for scale-out pod", got)
+ }
+ if state, ok := instanceRepo.runtimeStates[27]; ok {
+ t.Fatalf("instance runtime state = %+v, want unchanged while waiting for scale-out pod", state)
+ }
+}
+
+func TestRuntimeSchedulerDoesNotScaleOutWhenGatewayCreateFailsWithFreeSlot(t *testing.T) {
+ ctx := context.Background()
+ endpoint := "http://pod-1.runtime"
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 5: {
+ ID: 5,
+ RuntimeType: RuntimeTypeOpenClaw,
+ Namespace: "clawmanager-system",
+ DeploymentName: "openclaw-runtime",
+ AgentEndpoint: &endpoint,
+ State: "ready",
+ Capacity: 100,
+ UsedSlots: 1,
+ },
+ },
+ schedulable: []models.RuntimePod{{
+ ID: 5,
+ RuntimeType: RuntimeTypeOpenClaw,
+ Namespace: "clawmanager-system",
+ DeploymentName: "openclaw-runtime",
+ AgentEndpoint: &endpoint,
+ State: "ready",
+ Capacity: 100,
+ UsedSlots: 1,
+ }},
+ }
+ deployments := &fakeRuntimeDeploymentService{}
+ scheduler := NewRuntimeScheduler(
+ newFakeRuntimeInstanceRepo(),
+ podRepo,
+ newFakeRuntimeBindingRepo(),
+ &fakeRuntimeRolloutRepo{},
+ &fakeRuntimeAgentClient{createErr: errors.New("agent timeout")},
+ NewRuntimeEventService(nil),
+ nil,
+ deployments,
+ time.Second,
+ )
+
+ err := scheduler.assignInstance(ctx, models.Instance{
+ ID: 28,
+ UserID: 12,
+ Type: RuntimeTypeOpenClaw,
+ RuntimeType: RuntimeBackendGateway,
+ InstanceMode: InstanceModeLite,
+ WorkspacePath: ptrString("/workspaces/openclaw/user-12/instance-28"),
+ RuntimeGeneration: 1,
+ })
+ if err == nil {
+ t.Fatal("assignInstance returned nil error")
+ }
+ if got := len(deployments.scales); got != 0 {
+ t.Fatalf("deployment Scale calls = %d, want 0", got)
+ }
+}
+
+func TestRuntimeSchedulerReconcileAssignsDesiredRunningInstanceWithMissingBinding(t *testing.T) {
+ ctx := context.Background()
+ endpoint := "http://agent.runtime"
+ workspacePath := "/workspaces/openclaw/user-46/instance-18"
+ instanceRepo := newFakeRuntimeInstanceRepo()
+ instanceRepo.desiredRunning = []models.Instance{{
+ ID: 18,
+ UserID: 46,
+ Type: RuntimeTypeOpenClaw,
+ RuntimeType: RuntimeBackendGateway,
+ InstanceMode: InstanceModeLite,
+ Status: "running",
+ MemoryGB: 1,
+ DiskGB: 1,
+ WorkspacePath: &workspacePath,
+ RuntimeGeneration: 4,
+ }}
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1},
+ },
+ schedulable: []models.RuntimePod{
+ {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1},
+ },
+ }
+ agent := &fakeRuntimeAgentClient{
+ createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-18", Port: 20018, Status: "running"},
+ }
+ scheduler := NewRuntimeScheduler(
+ instanceRepo,
+ podRepo,
+ newFakeRuntimeBindingRepo(),
+ &fakeRuntimeRolloutRepo{},
+ agent,
+ NewRuntimeEventService(nil),
+ nil,
+ &fakeRuntimeDeploymentService{},
+ time.Second,
+ )
+
+ if err := scheduler.reconcile(ctx); err != nil {
+ t.Fatalf("reconcile returned error: %v", err)
+ }
+ if got := len(agent.createRequests); got != 1 {
+ t.Fatalf("CreateGateway calls = %d, want 1", got)
+ }
+}
+
+func TestRuntimeSchedulerReconcileRetriesRecoverableNoSchedulableError(t *testing.T) {
+ ctx := context.Background()
+ endpoint := "http://agent.runtime"
+ workspacePath := "/workspaces/openclaw/user-46/instance-68"
+ errorMessage := "no schedulable openclaw runtime pod"
+ instanceRepo := newFakeRuntimeInstanceRepo()
+ instanceRepo.desiredRunning = []models.Instance{{
+ ID: 68,
+ UserID: 46,
+ Type: RuntimeTypeOpenClaw,
+ RuntimeType: RuntimeBackendGateway,
+ InstanceMode: InstanceModeLite,
+ Status: "error",
+ RuntimeErrorMessage: &errorMessage,
+ MemoryGB: 1,
+ DiskGB: 1,
+ WorkspacePath: &workspacePath,
+ RuntimeGeneration: 29,
+ }}
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 50: {ID: 50, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 100, UsedSlots: 0},
+ },
+ schedulable: []models.RuntimePod{
+ {ID: 50, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 100, UsedSlots: 0},
+ },
+ }
+ agent := &fakeRuntimeAgentClient{
+ createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-68", Port: 20068, Status: "running"},
+ }
+ scheduler := NewRuntimeScheduler(
+ instanceRepo,
+ podRepo,
+ newFakeRuntimeBindingRepo(),
+ &fakeRuntimeRolloutRepo{},
+ agent,
+ NewRuntimeEventService(nil),
+ nil,
+ &fakeRuntimeDeploymentService{},
+ time.Second,
+ )
+
+ if err := scheduler.reconcile(ctx); err != nil {
+ t.Fatalf("reconcile returned error: %v", err)
+ }
+ if got := len(agent.createRequests); got != 1 {
+ t.Fatalf("CreateGateway calls = %d, want 1", got)
+ }
+ state := instanceRepo.runtimeStates[68]
+ if state.status != "running" || state.generation != 29 || state.message != nil {
+ t.Fatalf("runtime state = %+v, want running generation 29 without error message", state)
+ }
+}
+
+func TestRuntimeSchedulerReconcileSkipsNonRecoverableErrorInstance(t *testing.T) {
+ ctx := context.Background()
+ endpoint := "http://agent.runtime"
+ workspacePath := "/workspaces/openclaw/user-46/instance-69"
+ errorMessage := "create gateway failed: invalid runtime response"
+ instanceRepo := newFakeRuntimeInstanceRepo()
+ instanceRepo.desiredRunning = []models.Instance{{
+ ID: 69,
+ UserID: 46,
+ Type: RuntimeTypeOpenClaw,
+ RuntimeType: RuntimeBackendGateway,
+ InstanceMode: InstanceModeLite,
+ Status: "error",
+ RuntimeErrorMessage: &errorMessage,
+ MemoryGB: 1,
+ DiskGB: 1,
+ WorkspacePath: &workspacePath,
+ RuntimeGeneration: 24,
+ }}
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 50: {ID: 50, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 100, UsedSlots: 0},
+ },
+ schedulable: []models.RuntimePod{
+ {ID: 50, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 100, UsedSlots: 0},
+ },
+ }
+ agent := &fakeRuntimeAgentClient{
+ createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-69", Port: 20069, Status: "running"},
+ }
+ scheduler := NewRuntimeScheduler(
+ instanceRepo,
+ podRepo,
+ newFakeRuntimeBindingRepo(),
+ &fakeRuntimeRolloutRepo{},
+ agent,
+ NewRuntimeEventService(nil),
+ nil,
+ &fakeRuntimeDeploymentService{},
+ time.Second,
+ )
+
+ if err := scheduler.reconcile(ctx); err != nil {
+ t.Fatalf("reconcile returned error: %v", err)
+ }
+ if got := len(agent.createRequests); got != 0 {
+ t.Fatalf("CreateGateway calls = %d, want 0 for non-recoverable error", got)
+ }
+ if _, ok := instanceRepo.runtimeStates[69]; ok {
+ t.Fatalf("runtime state was changed for non-recoverable error: %+v", instanceRepo.runtimeStates[69])
+ }
+}
+
+func TestRuntimeSchedulerReconcileSkipsAssignmentWhenBindingLookupErrors(t *testing.T) {
+ ctx := context.Background()
+ endpoint := "http://agent.runtime"
+ lookupErr := errors.New("binding db unavailable")
+ workspacePath := "/workspaces/openclaw/user-47/instance-19"
+ instanceRepo := newFakeRuntimeInstanceRepo()
+ instanceRepo.desiredRunning = []models.Instance{{
+ ID: 19,
+ UserID: 47,
+ Type: RuntimeTypeOpenClaw,
+ RuntimeType: RuntimeBackendGateway,
+ InstanceMode: InstanceModeLite,
+ Status: "running",
+ MemoryGB: 1,
+ DiskGB: 1,
+ WorkspacePath: &workspacePath,
+ RuntimeGeneration: 5,
+ }}
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1},
+ },
+ schedulable: []models.RuntimePod{
+ {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1},
+ },
+ }
+ bindingRepo := newFakeRuntimeBindingRepo()
+ bindingRepo.runningErr = lookupErr
+ agent := &fakeRuntimeAgentClient{
+ createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-19", Port: 20019, Status: "running"},
+ }
+ scheduler := NewRuntimeScheduler(
+ instanceRepo,
+ podRepo,
+ bindingRepo,
+ &fakeRuntimeRolloutRepo{},
+ agent,
+ NewRuntimeEventService(nil),
+ nil,
+ &fakeRuntimeDeploymentService{},
+ time.Second,
+ )
+
+ err := scheduler.reconcile(ctx)
+ if err == nil {
+ t.Fatal("reconcile returned nil error")
+ }
+ if !errors.Is(err, lookupErr) {
+ t.Fatalf("reconcile error = %v, want lookup error", err)
+ }
+ if got := len(agent.createRequests); got != 0 {
+ t.Fatalf("CreateGateway calls = %d, want 0", got)
+ }
+ if bindingRepo.bindings[19] != nil {
+ t.Fatal("binding was created despite lookup error")
+ }
+ if podRepo.claims[9] != 0 {
+ t.Fatalf("pod claims = %d, want 0", podRepo.claims[9])
+ }
+}
+
+func TestRuntimeSchedulerReconcileSkipsLegacyInstanceWithoutWorkspacePath(t *testing.T) {
+ ctx := context.Background()
+ endpoint := "http://agent.runtime"
+ instanceRepo := newFakeRuntimeInstanceRepo()
+ instanceRepo.creating = []models.Instance{{
+ ID: 20,
+ UserID: 48,
+ Type: RuntimeTypeOpenClaw,
+ RuntimeType: RuntimeBackendGateway,
+ InstanceMode: InstanceModeLite,
+ Status: "creating",
+ MemoryGB: 1,
+ DiskGB: 1,
+ RuntimeGeneration: 1,
+ }}
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1},
+ },
+ schedulable: []models.RuntimePod{
+ {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1},
+ },
+ }
+ agent := &fakeRuntimeAgentClient{
+ createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-20", Port: 20020, Status: "running"},
+ }
+ scheduler := NewRuntimeScheduler(
+ instanceRepo,
+ podRepo,
+ newFakeRuntimeBindingRepo(),
+ &fakeRuntimeRolloutRepo{},
+ agent,
+ NewRuntimeEventService(nil),
+ nil,
+ &fakeRuntimeDeploymentService{},
+ time.Second,
+ )
+
+ if err := scheduler.reconcile(ctx); err != nil {
+ t.Fatalf("reconcile returned error: %v", err)
+ }
+ if got := len(agent.createRequests); got != 0 {
+ t.Fatalf("CreateGateway calls = %d, want 0", got)
+ }
+ if _, ok := instanceRepo.runtimeStates[20]; ok {
+ t.Fatalf("legacy instance was marked with runtime state: %+v", instanceRepo.runtimeStates[20])
+ }
+}
+
+func TestRuntimeSchedulerSkipsCreatingInstanceWithExistingBinding(t *testing.T) {
+ ctx := context.Background()
+ endpoint := "http://agent.runtime"
+ workspacePath := "/workspaces/openclaw/user-49/instance-21"
+ instanceRepo := newFakeRuntimeInstanceRepo()
+ instanceRepo.creating = []models.Instance{{
+ ID: 21,
+ UserID: 49,
+ Type: RuntimeTypeOpenClaw,
+ RuntimeType: RuntimeBackendGateway,
+ InstanceMode: InstanceModeLite,
+ Status: "creating",
+ MemoryGB: 1,
+ DiskGB: 1,
+ WorkspacePath: &workspacePath,
+ RuntimeGeneration: 1,
+ }}
+ bindingRepo := newFakeRuntimeBindingRepo()
+ bindingRepo.bindings[21] = &models.InstanceRuntimeBinding{InstanceID: 21, RuntimePodID: 9, State: "creating", Generation: 1}
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1},
+ },
+ schedulable: []models.RuntimePod{
+ {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1},
+ },
+ }
+ agent := &fakeRuntimeAgentClient{
+ createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-21", Port: 20021, Status: "running"},
+ }
+ scheduler := NewRuntimeScheduler(
+ instanceRepo,
+ podRepo,
+ bindingRepo,
+ &fakeRuntimeRolloutRepo{},
+ agent,
+ NewRuntimeEventService(nil),
+ nil,
+ &fakeRuntimeDeploymentService{},
+ time.Second,
+ )
+
+ if err := scheduler.reconcile(ctx); err != nil {
+ t.Fatalf("reconcile returned error: %v", err)
+ }
+ if got := len(agent.createRequests); got != 0 {
+ t.Fatalf("CreateGateway calls = %d, want 0", got)
+ }
+ if podRepo.claims[9] != 0 {
+ t.Fatalf("pod claims = %d, want 0", podRepo.claims[9])
+ }
+}
+
+func TestRuntimeSchedulerCleansUpGatewayAndReleasesSlotWhenBindingCreateFails(t *testing.T) {
+ ctx := context.Background()
+ endpoint := "http://agent.runtime"
+ createErr := errors.New("binding insert failed")
+ instanceRepo := newFakeRuntimeInstanceRepo()
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1},
+ },
+ schedulable: []models.RuntimePod{
+ {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1},
+ },
+ }
+ bindingRepo := newFakeRuntimeBindingRepo()
+ bindingRepo.createErr = createErr
+ agent := &fakeRuntimeAgentClient{
+ createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-22", Port: 20022, Status: "running"},
+ }
+ scheduler := NewRuntimeScheduler(
+ instanceRepo,
+ podRepo,
+ bindingRepo,
+ &fakeRuntimeRolloutRepo{},
+ agent,
+ NewRuntimeEventService(nil),
+ nil,
+ &fakeRuntimeDeploymentService{},
+ time.Second,
+ )
+
+ err := scheduler.assignInstance(ctx, models.Instance{
+ ID: 22,
+ UserID: 50,
+ Type: RuntimeTypeOpenClaw,
+ RuntimeType: RuntimeBackendGateway,
+ InstanceMode: InstanceModeLite,
+ WorkspacePath: ptrString("/workspaces/openclaw/user-50/instance-22"),
+ MemoryGB: 1,
+ DiskGB: 1,
+ RuntimeGeneration: 1,
+ })
+ if err == nil {
+ t.Fatal("assignInstance returned nil error")
+ }
+ if !errors.Is(err, createErr) {
+ t.Fatalf("assignInstance error = %v, want binding create error", err)
+ }
+ if got := len(agent.deleteRequests); got != 1 {
+ t.Fatalf("DeleteGateway calls = %d, want 1", got)
+ }
+ if agent.deleteRequests[0].gatewayID != "gw-22" {
+ t.Fatalf("deleted gateway = %q, want gw-22", agent.deleteRequests[0].gatewayID)
+ }
+ if podRepo.releases[9] != 1 {
+ t.Fatalf("pod releases = %d, want 1", podRepo.releases[9])
+ }
+ if bindingRepo.bindings[22] != nil {
+ t.Fatal("binding remains after failed create")
+ }
+}
+
+func TestRuntimeSchedulerCleansUpGatewayBindingAndSlotWhenWorkspacePathUpdateFails(t *testing.T) {
+ ctx := context.Background()
+ endpoint := "http://agent.runtime"
+ workspaceErr := errors.New("workspace path update failed")
+ instanceRepo := newFakeRuntimeInstanceRepo()
+ instanceRepo.setWorkspacePathErr = workspaceErr
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1},
+ },
+ schedulable: []models.RuntimePod{
+ {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1},
+ },
+ }
+ bindingRepo := newFakeRuntimeBindingRepo()
+ agent := &fakeRuntimeAgentClient{
+ createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-23", Port: 20023, Status: "running"},
+ }
+ scheduler := NewRuntimeScheduler(
+ instanceRepo,
+ podRepo,
+ bindingRepo,
+ &fakeRuntimeRolloutRepo{},
+ agent,
+ NewRuntimeEventService(nil),
+ nil,
+ &fakeRuntimeDeploymentService{},
+ time.Second,
+ )
+
+ err := scheduler.assignInstance(ctx, models.Instance{
+ ID: 23,
+ UserID: 51,
+ Type: RuntimeTypeOpenClaw,
+ RuntimeType: RuntimeBackendGateway,
+ InstanceMode: InstanceModeLite,
+ WorkspacePath: ptrString("/workspaces/openclaw/user-51/instance-23"),
+ MemoryGB: 1,
+ DiskGB: 1,
+ RuntimeGeneration: 1,
+ })
+ if err == nil {
+ t.Fatal("assignInstance returned nil error")
+ }
+ if !errors.Is(err, workspaceErr) {
+ t.Fatalf("assignInstance error = %v, want workspace update error", err)
+ }
+ if got := len(agent.deleteRequests); got != 1 {
+ t.Fatalf("DeleteGateway calls = %d, want 1", got)
+ }
+ if agent.deleteRequests[0].gatewayID != "gw-23" {
+ t.Fatalf("deleted gateway = %q, want gw-23", agent.deleteRequests[0].gatewayID)
+ }
+ if bindingRepo.deleteCalls[23] != 1 {
+ t.Fatalf("binding delete calls = %d, want 1", bindingRepo.deleteCalls[23])
+ }
+ if bindingRepo.bindings[23] != nil {
+ t.Fatal("binding remains after workspace update failure")
+ }
+ if podRepo.releases[9] != 1 {
+ t.Fatalf("pod releases = %d, want 1", podRepo.releases[9])
+ }
+}
+
+func TestRuntimeSchedulerDesiredRunningSkipsNonRunningExistingBinding(t *testing.T) {
+ ctx := context.Background()
+ endpoint := "http://agent.runtime"
+ workspacePath := "/workspaces/openclaw/user-52/instance-24"
+ instanceRepo := newFakeRuntimeInstanceRepo()
+ instanceRepo.desiredRunning = []models.Instance{{
+ ID: 24,
+ UserID: 52,
+ Type: RuntimeTypeOpenClaw,
+ RuntimeType: RuntimeBackendGateway,
+ InstanceMode: InstanceModeLite,
+ Status: "running",
+ MemoryGB: 1,
+ DiskGB: 1,
+ WorkspacePath: &workspacePath,
+ RuntimeGeneration: 1,
+ }}
+ bindingRepo := newFakeRuntimeBindingRepo()
+ bindingRepo.bindings[24] = &models.InstanceRuntimeBinding{
+ InstanceID: 24,
+ RuntimePodID: 9,
+ State: "creating",
+ Generation: 1,
+ }
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1},
+ },
+ schedulable: []models.RuntimePod{
+ {ID: 9, RuntimeType: RuntimeTypeOpenClaw, AgentEndpoint: &endpoint, State: "ready", Capacity: 1},
+ },
+ }
+ agent := &fakeRuntimeAgentClient{
+ createResponse: &RuntimeAgentCreateGatewayResponse{GatewayID: "gw-24", Port: 20024, Status: "running"},
+ }
+ scheduler := NewRuntimeScheduler(
+ instanceRepo,
+ podRepo,
+ bindingRepo,
+ &fakeRuntimeRolloutRepo{},
+ agent,
+ NewRuntimeEventService(nil),
+ nil,
+ &fakeRuntimeDeploymentService{},
+ time.Second,
+ )
+
+ if err := scheduler.reconcile(ctx); err != nil {
+ t.Fatalf("reconcile returned error: %v", err)
+ }
+ if got := len(agent.createRequests); got != 0 {
+ t.Fatalf("CreateGateway calls = %d, want 0", got)
+ }
+ if podRepo.claims[9] != 0 {
+ t.Fatalf("pod claims = %d, want 0", podRepo.claims[9])
+ }
+ if bindingRepo.bindings[24].GatewayID != "" {
+ t.Fatalf("binding was overwritten: %+v", bindingRepo.bindings[24])
+ }
+}
+
+func TestRuntimeSchedulerFailoverPodMarksUnhealthyDeletesBindingsReleasesSlotsAndRecreatesInstances(t *testing.T) {
+ ctx := context.Background()
+ reason := "agent heartbeat lost"
+ instanceRepo := newFakeRuntimeInstanceRepo()
+ instanceRepo.byID[17] = &models.Instance{ID: 17, RuntimeGeneration: 3}
+ bindingRepo := newFakeRuntimeBindingRepo()
+ bindingRepo.bindings[17] = &models.InstanceRuntimeBinding{
+ InstanceID: 17,
+ RuntimePodID: 9,
+ Generation: 3,
+ }
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, State: "ready"},
+ },
+ }
+ scheduler := NewRuntimeScheduler(
+ instanceRepo,
+ podRepo,
+ bindingRepo,
+ &fakeRuntimeRolloutRepo{},
+ &fakeRuntimeAgentClient{},
+ NewRuntimeEventService(nil),
+ nil,
+ &fakeRuntimeDeploymentService{},
+ time.Second,
+ )
+
+ if err := scheduler.FailoverPod(ctx, 9, reason); err != nil {
+ t.Fatalf("FailoverPod returned error: %v", err)
+ }
+
+ if got := podRepo.marked[9]; got.state != "unhealthy" || got.draining {
+ t.Fatalf("marked pod state = %+v", got)
+ }
+ if bindingRepo.bindings[17] != nil {
+ t.Fatal("binding was not deleted")
+ }
+ if podRepo.releases[9] != 1 {
+ t.Fatalf("pod releases = %d, want 1", podRepo.releases[9])
+ }
+ state := instanceRepo.runtimeStates[17]
+ if state.status != "creating" || state.generation != 4 {
+ t.Fatalf("runtime state = %+v, want creating generation 4", state)
+ }
+ if state.message == nil || *state.message != reason {
+ t.Fatalf("runtime state message = %v, want %q", state.message, reason)
+ }
+}
+
+func TestRuntimeSchedulerFailoverLeavesBindingAndSlotWhenInstanceUpdateFails(t *testing.T) {
+ ctx := context.Background()
+ reason := "agent heartbeat lost"
+ updateErr := errors.New("instance update failed")
+ instanceRepo := newFakeRuntimeInstanceRepo()
+ instanceRepo.byID[17] = &models.Instance{ID: 17, RuntimeGeneration: 3}
+ instanceRepo.updateRuntimeStateErr = updateErr
+ bindingRepo := newFakeRuntimeBindingRepo()
+ bindingRepo.bindings[17] = &models.InstanceRuntimeBinding{
+ InstanceID: 17,
+ RuntimePodID: 9,
+ Generation: 3,
+ }
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, State: "ready"},
+ },
+ }
+ scheduler := NewRuntimeScheduler(
+ instanceRepo,
+ podRepo,
+ bindingRepo,
+ &fakeRuntimeRolloutRepo{},
+ &fakeRuntimeAgentClient{},
+ NewRuntimeEventService(nil),
+ nil,
+ &fakeRuntimeDeploymentService{},
+ time.Second,
+ )
+
+ err := scheduler.FailoverPod(ctx, 9, reason)
+ if err == nil {
+ t.Fatal("FailoverPod returned nil error")
+ }
+ if !errors.Is(err, updateErr) {
+ t.Fatalf("FailoverPod error = %v, want update error", err)
+ }
+ if bindingRepo.bindings[17] == nil {
+ t.Fatal("binding was deleted despite instance update failure")
+ }
+ if podRepo.releases[9] != 0 {
+ t.Fatalf("pod releases = %d, want 0", podRepo.releases[9])
+ }
+}
+
+func TestRuntimeSchedulerReconcileFailoversStalePod(t *testing.T) {
+ ctx := context.Background()
+ staleSeen := time.Now().UTC().Add(-30 * time.Second)
+ instanceRepo := newFakeRuntimeInstanceRepo()
+ instanceRepo.byID[17] = &models.Instance{ID: 17, RuntimeGeneration: 3}
+ bindingRepo := newFakeRuntimeBindingRepo()
+ bindingRepo.bindings[17] = &models.InstanceRuntimeBinding{
+ InstanceID: 17,
+ RuntimePodID: 9,
+ Generation: 3,
+ State: "running",
+ }
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, State: "ready", LastSeenAt: &staleSeen},
+ },
+ }
+ scheduler := NewRuntimeScheduler(
+ instanceRepo,
+ podRepo,
+ bindingRepo,
+ &fakeRuntimeRolloutRepo{},
+ &fakeRuntimeAgentClient{},
+ NewRuntimeEventService(nil),
+ nil,
+ &fakeRuntimeDeploymentService{},
+ time.Second,
+ WithRuntimeSchedulerHeartbeatTimeout(10*time.Second),
+ )
+
+ if err := scheduler.reconcile(ctx); err != nil {
+ t.Fatalf("reconcile returned error: %v", err)
+ }
+
+ if got := podRepo.marked[9]; got.state != "unhealthy" || got.draining {
+ t.Fatalf("marked pod state = %+v", got)
+ }
+ if bindingRepo.bindings[17] != nil {
+ t.Fatal("binding was not deleted")
+ }
+ if podRepo.releases[9] != 1 {
+ t.Fatalf("pod releases = %d, want 1", podRepo.releases[9])
+ }
+ state := instanceRepo.runtimeStates[17]
+ if state.status != "creating" || state.generation != 4 {
+ t.Fatalf("runtime state = %+v, want creating generation 4", state)
+ }
+ if state.message == nil || !strings.Contains(*state.message, "runtime pod heartbeat lost") {
+ t.Fatalf("runtime state message = %v, want heartbeat lost reason", state.message)
+ }
+}
+
+func TestRuntimeSchedulerReconcileKeepsRecentPodBinding(t *testing.T) {
+ ctx := context.Background()
+ recentSeen := time.Now().UTC()
+ instanceRepo := newFakeRuntimeInstanceRepo()
+ instanceRepo.byID[17] = &models.Instance{ID: 17, RuntimeGeneration: 3}
+ bindingRepo := newFakeRuntimeBindingRepo()
+ bindingRepo.bindings[17] = &models.InstanceRuntimeBinding{
+ InstanceID: 17,
+ RuntimePodID: 9,
+ Generation: 3,
+ State: "running",
+ }
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, State: "ready", LastSeenAt: &recentSeen},
+ },
+ }
+ scheduler := NewRuntimeScheduler(
+ instanceRepo,
+ podRepo,
+ bindingRepo,
+ &fakeRuntimeRolloutRepo{},
+ &fakeRuntimeAgentClient{},
+ NewRuntimeEventService(nil),
+ nil,
+ &fakeRuntimeDeploymentService{},
+ time.Second,
+ WithRuntimeSchedulerHeartbeatTimeout(10*time.Second),
+ )
+
+ if err := scheduler.reconcile(ctx); err != nil {
+ t.Fatalf("reconcile returned error: %v", err)
+ }
+
+ if got := podRepo.marked[9]; got.state != "" || got.draining {
+ t.Fatalf("pod was unexpectedly marked: %+v", got)
+ }
+ if bindingRepo.bindings[17] == nil {
+ t.Fatal("binding was deleted")
+ }
+ if podRepo.releases[9] != 0 {
+ t.Fatalf("pod releases = %d, want 0", podRepo.releases[9])
+ }
+}
+
+func TestRuntimeSchedulerRolloutExistingDrainingPodsPreventAdditionalDrains(t *testing.T) {
+ ctx := context.Background()
+ readyEndpoint := "http://ready.runtime"
+ drainingEndpoint := "http://draining.runtime"
+ rolloutRepo := &fakeRuntimeRolloutRepo{
+ rollouts: map[int64]*models.RuntimeRollout{
+ 7: {ID: 7, RuntimeType: RuntimeTypeOpenClaw, Status: "pending", BatchSize: 3, MaxUnavailable: 1},
+ },
+ }
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 1: {ID: 1, RuntimeType: RuntimeTypeOpenClaw, State: "ready", Draining: true, AgentEndpoint: &drainingEndpoint},
+ 2: {ID: 2, RuntimeType: RuntimeTypeOpenClaw, State: "ready", AgentEndpoint: &readyEndpoint},
+ },
+ }
+ agent := &fakeRuntimeAgentClient{}
+ scheduler := NewRuntimeScheduler(
+ newFakeRuntimeInstanceRepo(),
+ podRepo,
+ newFakeRuntimeBindingRepo(),
+ rolloutRepo,
+ agent,
+ NewRuntimeEventService(nil),
+ nil,
+ &fakeRuntimeDeploymentService{},
+ time.Second,
+ )
+
+ if err := scheduler.StartRollout(ctx, 7); err != nil {
+ t.Fatalf("StartRollout returned error: %v", err)
+ }
+ if got := len(agent.drainEndpoints); got != 0 {
+ t.Fatalf("Drain calls = %d, want 0", got)
+ }
+ if len(rolloutRepo.statuses) != 1 || rolloutRepo.statuses[0].status != "running" {
+ t.Fatalf("rollout statuses = %+v, want only running", rolloutRepo.statuses)
+ }
+}
+
+func TestRuntimeSchedulerRolloutBatchLimitedByMaxUnavailable(t *testing.T) {
+ ctx := context.Background()
+ endpoint1 := "http://pod-1.runtime"
+ endpoint2 := "http://pod-2.runtime"
+ endpoint3 := "http://pod-3.runtime"
+ rolloutRepo := &fakeRuntimeRolloutRepo{
+ rollouts: map[int64]*models.RuntimeRollout{
+ 8: {ID: 8, RuntimeType: RuntimeTypeOpenClaw, Status: "pending", BatchSize: 3, MaxUnavailable: 2},
+ },
+ }
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 1: {ID: 1, RuntimeType: RuntimeTypeOpenClaw, State: "ready", AgentEndpoint: &endpoint1},
+ 2: {ID: 2, RuntimeType: RuntimeTypeOpenClaw, State: "ready", AgentEndpoint: &endpoint2},
+ 3: {ID: 3, RuntimeType: RuntimeTypeOpenClaw, State: "ready", AgentEndpoint: &endpoint3},
+ },
+ }
+ agent := &fakeRuntimeAgentClient{}
+ scheduler := NewRuntimeScheduler(
+ newFakeRuntimeInstanceRepo(),
+ podRepo,
+ newFakeRuntimeBindingRepo(),
+ rolloutRepo,
+ agent,
+ NewRuntimeEventService(nil),
+ nil,
+ &fakeRuntimeDeploymentService{},
+ time.Second,
+ )
+
+ if err := scheduler.StartRollout(ctx, 8); err != nil {
+ t.Fatalf("StartRollout returned error: %v", err)
+ }
+ if got := len(agent.drainEndpoints); got != 2 {
+ t.Fatalf("Drain calls = %d, want 2", got)
+ }
+}
+
+func TestRuntimeSchedulerRolloutUpdatesDeployments(t *testing.T) {
+ ctx := context.Background()
+ endpoint := "http://pod-1.runtime"
+ rolloutRepo := &fakeRuntimeRolloutRepo{
+ rollouts: map[int64]*models.RuntimeRollout{
+ 9: {ID: 9, RuntimeType: RuntimeTypeOpenClaw, TargetImageRef: "new-image", Status: "pending", BatchSize: 1, MaxUnavailable: 1},
+ },
+ }
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 1: {
+ ID: 1,
+ RuntimeType: RuntimeTypeOpenClaw,
+ State: "ready",
+ Namespace: "runtime-system",
+ DeploymentName: "openclaw-runtime",
+ AgentEndpoint: &endpoint,
+ },
+ },
+ }
+ deployments := &fakeRuntimeDeploymentService{}
+ scheduler := NewRuntimeScheduler(
+ newFakeRuntimeInstanceRepo(),
+ podRepo,
+ newFakeRuntimeBindingRepo(),
+ rolloutRepo,
+ &fakeRuntimeAgentClient{},
+ NewRuntimeEventService(nil),
+ nil,
+ deployments,
+ time.Second,
+ )
+
+ if err := scheduler.StartRollout(ctx, 9); err != nil {
+ t.Fatalf("StartRollout returned error: %v", err)
+ }
+ if got := len(deployments.rolloutImageCalls); got != 1 {
+ t.Fatalf("deployment RolloutImage calls = %d, want 1", got)
+ }
+ call := deployments.rolloutImageCalls[0]
+ if call.namespace != "runtime-system" || call.name != "openclaw-runtime" || call.image != "new-image" {
+ t.Fatalf("RolloutImage call = %+v, want runtime-system/openclaw-runtime new-image", call)
+ }
+ if call.maxUnavailable != 1 || call.maxSurge != 1 {
+ t.Fatalf("RolloutImage strategy = %+v, want maxUnavailable=1 maxSurge=1", call)
+ }
+}
+
+func TestRuntimeSchedulerRolloutUsesStalePodDeploymentRefWhenNoCurrentPods(t *testing.T) {
+ ctx := context.Background()
+ staleSeen := time.Now().UTC().Add(-5 * time.Minute)
+ rolloutRepo := &fakeRuntimeRolloutRepo{
+ rollouts: map[int64]*models.RuntimeRollout{
+ 12: {ID: 12, RuntimeType: RuntimeTypeOpenClaw, TargetImageRef: "registry/openclaw:v2", Status: "pending", BatchSize: 1, MaxUnavailable: 1},
+ },
+ }
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 41: {
+ ID: 41,
+ RuntimeType: RuntimeTypeOpenClaw,
+ State: "unhealthy",
+ Namespace: "runtime-system",
+ DeploymentName: "openclaw-runtime",
+ ImageRef: "registry/openclaw:v1",
+ LastSeenAt: &staleSeen,
+ },
+ },
+ }
+ agent := &fakeRuntimeAgentClient{}
+ deployments := &fakeRuntimeDeploymentService{}
+ scheduler := NewRuntimeScheduler(
+ newFakeRuntimeInstanceRepo(),
+ podRepo,
+ newFakeRuntimeBindingRepo(),
+ rolloutRepo,
+ agent,
+ NewRuntimeEventService(nil),
+ nil,
+ deployments,
+ time.Second,
+ WithRuntimeSchedulerHeartbeatTimeout(10*time.Second),
+ )
+
+ if err := scheduler.StartRollout(ctx, 12); err != nil {
+ t.Fatalf("StartRollout returned error: %v", err)
+ }
+ if got := len(deployments.rolloutImageCalls); got != 1 {
+ t.Fatalf("deployment RolloutImage calls = %d, want 1", got)
+ }
+ call := deployments.rolloutImageCalls[0]
+ if call.namespace != "runtime-system" || call.name != "openclaw-runtime" || call.image != "registry/openclaw:v2" {
+ t.Fatalf("RolloutImage call = %+v, want runtime-system/openclaw-runtime registry/openclaw:v2", call)
+ }
+ if got := len(agent.drainEndpoints); got != 0 {
+ t.Fatalf("Drain calls = %d, want 0 with no current runtime pods", got)
+ }
+ if len(rolloutRepo.statuses) != 1 || rolloutRepo.statuses[0].status != "running" {
+ t.Fatalf("rollout statuses = %+v, want only running", rolloutRepo.statuses)
+ }
+}
+
+func TestRuntimeSchedulerRolloutFallsBackToDefaultDeploymentWhenNoPodRows(t *testing.T) {
+ ctx := context.Background()
+ rolloutRepo := &fakeRuntimeRolloutRepo{
+ rollouts: map[int64]*models.RuntimeRollout{
+ 13: {ID: 13, RuntimeType: RuntimeTypeOpenClaw, TargetImageRef: "registry/openclaw:v3", Status: "pending", BatchSize: 2, MaxUnavailable: 1},
+ },
+ }
+ deployments := &fakeRuntimeDeploymentService{}
+ scheduler := NewRuntimeScheduler(
+ newFakeRuntimeInstanceRepo(),
+ &fakeRuntimePodRepo{pods: map[int64]*models.RuntimePod{}},
+ newFakeRuntimeBindingRepo(),
+ rolloutRepo,
+ &fakeRuntimeAgentClient{},
+ NewRuntimeEventService(nil),
+ nil,
+ deployments,
+ time.Second,
+ WithRuntimeSchedulerNamespace("runtime-system"),
+ )
+
+ if err := scheduler.StartRollout(ctx, 13); err != nil {
+ t.Fatalf("StartRollout returned error: %v", err)
+ }
+ if got := len(deployments.rolloutImageCalls); got != 1 {
+ t.Fatalf("deployment RolloutImage calls = %d, want 1", got)
+ }
+ call := deployments.rolloutImageCalls[0]
+ if call.namespace != "runtime-system" || call.name != "openclaw-runtime" || call.image != "registry/openclaw:v3" {
+ t.Fatalf("RolloutImage call = %+v, want runtime-system/openclaw-runtime registry/openclaw:v3", call)
+ }
+}
+
+func TestRuntimeSchedulerRolloutSameImageFinishesWithoutDrain(t *testing.T) {
+ ctx := context.Background()
+ recentSeen := time.Now().UTC()
+ endpoint := "http://pod-1.runtime"
+ rolloutRepo := &fakeRuntimeRolloutRepo{
+ rollouts: map[int64]*models.RuntimeRollout{
+ 11: {ID: 11, RuntimeType: RuntimeTypeHermes, TargetImageRef: "registry/hermes:v2", Status: "pending", BatchSize: 1, MaxUnavailable: 1},
+ },
+ }
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 4: {
+ ID: 4,
+ RuntimeType: RuntimeTypeHermes,
+ State: "ready",
+ Namespace: "runtime-system",
+ DeploymentName: "hermes-runtime",
+ ImageRef: "registry/hermes:v2",
+ AgentEndpoint: &endpoint,
+ LastSeenAt: &recentSeen,
+ },
+ },
+ }
+ agent := &fakeRuntimeAgentClient{}
+ deployments := &fakeRuntimeDeploymentService{}
+ scheduler := NewRuntimeScheduler(
+ newFakeRuntimeInstanceRepo(),
+ podRepo,
+ newFakeRuntimeBindingRepo(),
+ rolloutRepo,
+ agent,
+ NewRuntimeEventService(nil),
+ nil,
+ deployments,
+ time.Second,
+ )
+
+ if err := scheduler.StartRollout(ctx, 11); err != nil {
+ t.Fatalf("StartRollout returned error: %v", err)
+ }
+ if got := len(agent.drainEndpoints); got != 0 {
+ t.Fatalf("Drain calls = %d, want 0", got)
+ }
+ if got := len(deployments.rolloutImageCalls); got != 0 {
+ t.Fatalf("RolloutImage calls = %d, want 0", got)
+ }
+ if len(rolloutRepo.statuses) != 2 || rolloutRepo.statuses[0].status != "running" || rolloutRepo.statuses[1].status != "finished" {
+ t.Fatalf("rollout statuses = %+v, want running then finished", rolloutRepo.statuses)
+ }
+}
+
+func TestRuntimeSchedulerReconcileStartsPendingRollouts(t *testing.T) {
+ ctx := context.Background()
+ recentSeen := time.Now().UTC()
+ endpoint := "http://pod-1.runtime"
+ rolloutRepo := &fakeRuntimeRolloutRepo{
+ rollouts: map[int64]*models.RuntimeRollout{
+ 10: {
+ ID: 10,
+ RuntimeType: RuntimeTypeHermes,
+ TargetImageRef: "registry/hermes:v2",
+ Status: "pending",
+ BatchSize: 2,
+ MaxUnavailable: 1,
+ },
+ },
+ }
+ podRepo := &fakeRuntimePodRepo{
+ pods: map[int64]*models.RuntimePod{
+ 3: {
+ ID: 3,
+ RuntimeType: RuntimeTypeHermes,
+ State: "ready",
+ Namespace: "runtime-system",
+ DeploymentName: "hermes-runtime",
+ AgentEndpoint: &endpoint,
+ LastSeenAt: &recentSeen,
+ },
+ },
+ }
+ deployments := &fakeRuntimeDeploymentService{}
+ scheduler := NewRuntimeScheduler(
+ newFakeRuntimeInstanceRepo(),
+ podRepo,
+ newFakeRuntimeBindingRepo(),
+ rolloutRepo,
+ &fakeRuntimeAgentClient{},
+ NewRuntimeEventService(nil),
+ nil,
+ deployments,
+ time.Second,
+ )
+
+ if err := scheduler.reconcile(ctx); err != nil {
+ t.Fatalf("reconcile returned error: %v", err)
+ }
+ if got := len(deployments.rolloutImageCalls); got != 1 {
+ t.Fatalf("deployment RolloutImage calls = %d, want 1", got)
+ }
+ call := deployments.rolloutImageCalls[0]
+ if call.namespace != "runtime-system" || call.name != "hermes-runtime" || call.image != "registry/hermes:v2" {
+ t.Fatalf("RolloutImage call = %+v, want runtime-system/hermes-runtime registry/hermes:v2", call)
+ }
+ if len(rolloutRepo.statuses) == 0 || rolloutRepo.statuses[0].status != "running" {
+ t.Fatalf("rollout statuses = %+v, want running", rolloutRepo.statuses)
+ }
+}
+
+type fakeRuntimeInstanceRepo struct {
+ creating []models.Instance
+ desiredRunning []models.Instance
+ byID map[int]*models.Instance
+ workspacePaths map[int]string
+ runtimeStates map[int]fakeRuntimeState
+ updateRuntimeStateErr error
+ setWorkspacePathErr error
+}
+
+type fakeRuntimeState struct {
+ status string
+ generation int
+ message *string
+}
+
+func newFakeRuntimeInstanceRepo() *fakeRuntimeInstanceRepo {
+ return &fakeRuntimeInstanceRepo{
+ byID: map[int]*models.Instance{},
+ workspacePaths: map[int]string{},
+ runtimeStates: map[int]fakeRuntimeState{},
+ }
+}
+
+func (r *fakeRuntimeInstanceRepo) Create(instance *models.Instance) error { return nil }
+func (r *fakeRuntimeInstanceRepo) GetByID(id int) (*models.Instance, error) {
+ return r.byID[id], nil
+}
+func (r *fakeRuntimeInstanceRepo) GetByAccessToken(accessToken string) (*models.Instance, error) {
+ return nil, nil
+}
+func (r *fakeRuntimeInstanceRepo) GetByAgentBootstrapToken(bootstrapToken string) (*models.Instance, error) {
+ return nil, nil
+}
+func (r *fakeRuntimeInstanceRepo) GetAll(offset, limit int) ([]models.Instance, error) {
+ return nil, nil
+}
+func (r *fakeRuntimeInstanceRepo) CountAll() (int, error) { return 0, nil }
+func (r *fakeRuntimeInstanceRepo) GetByUserID(userID int, offset, limit int) ([]models.Instance, error) {
+ return nil, nil
+}
+func (r *fakeRuntimeInstanceRepo) CountByUserID(userID int) (int, error) { return 0, nil }
+func (r *fakeRuntimeInstanceRepo) CountActiveByMode(ctx context.Context, mode string) (int, error) {
+ return 0, nil
+}
+func (r *fakeRuntimeInstanceRepo) ExistsByUserIDAndName(userID int, name string) (bool, error) {
+ return false, nil
+}
+func (r *fakeRuntimeInstanceRepo) GetAllRunning() ([]models.Instance, error) { return nil, nil }
+func (r *fakeRuntimeInstanceRepo) GetV2DesiredRunning(ctx context.Context, limit int) ([]models.Instance, error) {
+ return r.desiredRunning, nil
+}
+func (r *fakeRuntimeInstanceRepo) GetV2Creating(ctx context.Context, limit int) ([]models.Instance, error) {
+ return r.creating, nil
+}
+func (r *fakeRuntimeInstanceRepo) UpdateRuntimeState(ctx context.Context, id int, status string, generation int, message *string) error {
+ if r.updateRuntimeStateErr != nil {
+ return r.updateRuntimeStateErr
+ }
+ r.runtimeStates[id] = fakeRuntimeState{status: status, generation: generation, message: message}
+ return nil
+}
+func (r *fakeRuntimeInstanceRepo) SetWorkspacePath(ctx context.Context, id int, workspacePath string) error {
+ if r.setWorkspacePathErr != nil {
+ return r.setWorkspacePathErr
+ }
+ r.workspacePaths[id] = workspacePath
+ return nil
+}
+func (r *fakeRuntimeInstanceRepo) UpdateWorkspaceUsage(ctx context.Context, id int, usageBytes int64) error {
+ return nil
+}
+func (r *fakeRuntimeInstanceRepo) Update(instance *models.Instance) error { return nil }
+func (r *fakeRuntimeInstanceRepo) Delete(id int) error { return nil }
+
+type fakeRuntimePodRepo struct {
+ pods map[int64]*models.RuntimePod
+ schedulable []models.RuntimePod
+ claims map[int64]int
+ releases map[int64]int
+ marked map[int64]fakePodMark
+ releaseCount int
+ getErr error
+}
+
+type fakePodMark struct {
+ state string
+ draining bool
+}
+
+func (r *fakeRuntimePodRepo) UpsertFromAgent(ctx context.Context, pod *models.RuntimePod) error {
+ return nil
+}
+func (r *fakeRuntimePodRepo) GetByID(ctx context.Context, id int64) (*models.RuntimePod, error) {
+ if r.getErr != nil {
+ return nil, r.getErr
+ }
+ return r.pods[id], nil
+}
+func (r *fakeRuntimePodRepo) GetByNamespaceName(ctx context.Context, namespace, podName string) (*models.RuntimePod, error) {
+ return nil, nil
+}
+func (r *fakeRuntimePodRepo) List(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) {
+ var pods []models.RuntimePod
+ for _, pod := range r.pods {
+ if runtimeType == "" || pod.RuntimeType == runtimeType {
+ pods = append(pods, *pod)
+ }
+ }
+ return pods, nil
+}
+func (r *fakeRuntimePodRepo) ListSchedulable(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) {
+ return r.schedulable, nil
+}
+func (r *fakeRuntimePodRepo) TryClaimSlot(ctx context.Context, podID int64) (bool, error) {
+ if r.claims == nil {
+ r.claims = map[int64]int{}
+ }
+ r.claims[podID]++
+ return true, nil
+}
+func (r *fakeRuntimePodRepo) ReleaseSlot(ctx context.Context, podID int64) error {
+ if r.releases == nil {
+ r.releases = map[int64]int{}
+ }
+ r.releases[podID]++
+ r.releaseCount++
+ return nil
+}
+func (r *fakeRuntimePodRepo) MarkState(ctx context.Context, podID int64, state string, draining bool) error {
+ if r.marked == nil {
+ r.marked = map[int64]fakePodMark{}
+ }
+ r.marked[podID] = fakePodMark{state: state, draining: draining}
+ return nil
+}
+func (r *fakeRuntimePodRepo) UpdateHeartbeat(ctx context.Context, podID int64, state string, usedSlots int, capacity int, draining bool, lastSeenAt time.Time) error {
+ if r.pods != nil {
+ if pod := r.pods[podID]; pod != nil {
+ pod.State = state
+ pod.UsedSlots = usedSlots
+ pod.Capacity = capacity
+ pod.Draining = draining
+ pod.LastSeenAt = &lastSeenAt
+ }
+ }
+ return nil
+}
+func (r *fakeRuntimePodRepo) MarkUnseenUnhealthy(ctx context.Context, cutoff time.Time) error {
+ return nil
+}
+func (r *fakeRuntimePodRepo) UpdateMetrics(ctx context.Context, podID int64, metrics repository.RuntimePodMetricsUpdate) error {
+ return nil
+}
+
+type fakeRuntimeBindingRepo struct {
+ bindings map[int]*models.InstanceRuntimeBinding
+ deleteCalls map[int]int
+ deleteAndReleaseCalls map[int]int
+ runningErr error
+ getErr error
+ createErr error
+}
+
+func newFakeRuntimeBindingRepo() *fakeRuntimeBindingRepo {
+ return &fakeRuntimeBindingRepo{
+ bindings: map[int]*models.InstanceRuntimeBinding{},
+ deleteCalls: map[int]int{},
+ deleteAndReleaseCalls: map[int]int{},
+ }
+}
+
+func (r *fakeRuntimeBindingRepo) Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error {
+ if r.createErr != nil {
+ return r.createErr
+ }
+ copy := *binding
+ r.bindings[binding.InstanceID] = ©
+ return nil
+}
+func (r *fakeRuntimeBindingRepo) GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) {
+ if r.getErr != nil {
+ return nil, r.getErr
+ }
+ return r.bindings[instanceID], nil
+}
+func (r *fakeRuntimeBindingRepo) GetRunningByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error) {
+ if r.runningErr != nil {
+ return nil, r.runningErr
+ }
+ binding := r.bindings[instanceID]
+ if binding == nil || binding.State != "running" {
+ return nil, nil
+ }
+ return binding, nil
+}
+func (r *fakeRuntimeBindingRepo) ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error) {
+ var bindings []models.InstanceRuntimeBinding
+ for _, binding := range r.bindings {
+ if binding.RuntimePodID == runtimePodID {
+ bindings = append(bindings, *binding)
+ }
+ }
+ return bindings, nil
+}
+func (r *fakeRuntimeBindingRepo) ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error) {
+ return nil, nil
+}
+func (r *fakeRuntimeBindingRepo) UpdateRunning(ctx context.Context, instanceID int, generation int, gatewayID string, port int, pid *int) error {
+ return nil
+}
+func (r *fakeRuntimeBindingRepo) UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error {
+ return nil
+}
+func (r *fakeRuntimeBindingRepo) DeleteByInstanceID(ctx context.Context, instanceID int) error {
+ r.deleteCalls[instanceID]++
+ delete(r.bindings, instanceID)
+ return nil
+}
+func (r *fakeRuntimeBindingRepo) DeleteByInstanceIDAndReleaseSlot(ctx context.Context, instanceID int, runtimePodID int64) error {
+ r.deleteAndReleaseCalls[instanceID]++
+ delete(r.bindings, instanceID)
+ return nil
+}
+
+type fakeRuntimeRolloutRepo struct {
+ rollouts map[int64]*models.RuntimeRollout
+ statuses []fakeRolloutStatus
+}
+
+type fakeRolloutStatus struct {
+ id int64
+ status string
+ startedAt *time.Time
+ finishedAt *time.Time
+ message *string
+}
+
+func (r *fakeRuntimeRolloutRepo) Create(ctx context.Context, rollout *models.RuntimeRollout) error {
+ return nil
+}
+func (r *fakeRuntimeRolloutRepo) GetByID(ctx context.Context, id int64) (*models.RuntimeRollout, error) {
+ if r.rollouts == nil {
+ return nil, nil
+ }
+ return r.rollouts[id], nil
+}
+func (r *fakeRuntimeRolloutRepo) ListActive(ctx context.Context, runtimeType string) ([]models.RuntimeRollout, error) {
+ var active []models.RuntimeRollout
+ for _, rollout := range r.rollouts {
+ if rollout == nil {
+ continue
+ }
+ if runtimeType != "" && rollout.RuntimeType != runtimeType {
+ continue
+ }
+ if rollout.Status == "pending" || rollout.Status == "running" {
+ active = append(active, *rollout)
+ }
+ }
+ return active, nil
+}
+func (r *fakeRuntimeRolloutRepo) UpdateStatus(ctx context.Context, id int64, status string, startedAt *time.Time, finishedAt *time.Time, message *string) error {
+ r.statuses = append(r.statuses, fakeRolloutStatus{id: id, status: status, startedAt: startedAt, finishedAt: finishedAt, message: message})
+ return nil
+}
+
+type fakeRuntimeAgentClient struct {
+ createResponse *RuntimeAgentCreateGatewayResponse
+ createErr error
+ deleteErr error
+ createRequests []fakeCreateGatewayRequest
+ deleteRequests []fakeDeleteGatewayRequest
+ drainEndpoints []string
+}
+
+type fakeCreateGatewayRequest struct {
+ endpoint string
+ req RuntimeAgentCreateGatewayRequest
+}
+
+type fakeDeleteGatewayRequest struct {
+ endpoint string
+ gatewayID string
+}
+
+func (c *fakeRuntimeAgentClient) Health(ctx context.Context, endpoint string) error { return nil }
+func (c *fakeRuntimeAgentClient) CreateGateway(ctx context.Context, endpoint string, req RuntimeAgentCreateGatewayRequest) (*RuntimeAgentCreateGatewayResponse, error) {
+ c.createRequests = append(c.createRequests, fakeCreateGatewayRequest{endpoint: endpoint, req: req})
+ if c.createErr != nil {
+ return nil, c.createErr
+ }
+ if c.createResponse == nil {
+ return nil, errors.New("missing create response")
+ }
+ return c.createResponse, nil
+}
+func (c *fakeRuntimeAgentClient) DeleteGateway(ctx context.Context, endpoint, gatewayID string) error {
+ c.deleteRequests = append(c.deleteRequests, fakeDeleteGatewayRequest{endpoint: endpoint, gatewayID: gatewayID})
+ return c.deleteErr
+}
+func (c *fakeRuntimeAgentClient) Drain(ctx context.Context, endpoint string) error {
+ c.drainEndpoints = append(c.drainEndpoints, endpoint)
+ return nil
+}
+
+type fakeRuntimeEventService struct {
+ published []fakeRuntimeEvent
+}
+
+type fakeRuntimeEvent struct {
+ eventType string
+ payload any
+}
+
+func (s *fakeRuntimeEventService) Publish(ctx context.Context, eventType string, payload any) error {
+ s.published = append(s.published, fakeRuntimeEvent{eventType: eventType, payload: payload})
+ return nil
+}
+func (s *fakeRuntimeEventService) Read(ctx context.Context, lastID string, block time.Duration) ([]redisStreamMessage, error) {
+ return nil, nil
+}
+
+type fakeRuntimeDeploymentService struct {
+ ensureCalls int
+ scaleCalls int
+ scales []fakeScaleCall
+ rolloutImageCalls []fakeRolloutImageCall
+ pods []k8s.RuntimeDeploymentPod
+}
+
+type fakeScaleCall struct {
+ namespace string
+ name string
+ replicas int32
+}
+
+type fakeRolloutImageCall struct {
+ namespace string
+ name string
+ image string
+ maxUnavailable int
+ maxSurge int
+}
+
+func (s *fakeRuntimeDeploymentService) Ensure(ctx context.Context, spec k8s.RuntimeDeploymentSpec) error {
+ s.ensureCalls++
+ return nil
+}
+func (s *fakeRuntimeDeploymentService) Scale(ctx context.Context, namespace, name string, replicas int32) error {
+ s.scaleCalls++
+ s.scales = append(s.scales, fakeScaleCall{namespace: namespace, name: name, replicas: replicas})
+ return nil
+}
+func (s *fakeRuntimeDeploymentService) RolloutImage(ctx context.Context, namespace, name, image string, maxUnavailable, maxSurge int) error {
+ s.rolloutImageCalls = append(s.rolloutImageCalls, fakeRolloutImageCall{
+ namespace: namespace,
+ name: name,
+ image: image,
+ maxUnavailable: maxUnavailable,
+ maxSurge: maxSurge,
+ })
+ return nil
+}
+func (s *fakeRuntimeDeploymentService) ListPods(ctx context.Context, namespace, runtimeType string) ([]k8s.RuntimeDeploymentPod, error) {
+ var pods []k8s.RuntimeDeploymentPod
+ for _, pod := range s.pods {
+ if namespace != "" && pod.Namespace != namespace {
+ continue
+ }
+ if runtimeType != "" && pod.RuntimeType != runtimeType {
+ continue
+ }
+ pods = append(pods, pod)
+ }
+ return pods, nil
+}
+
+func TestIsSchedulerManagedV2InstanceRejectsProDesktop(t *testing.T) {
+ workspacePath := "/workspaces/openclaw/user-7/instance-42"
+ instance := models.Instance{
+ ID: 42,
+ Type: RuntimeTypeOpenClaw,
+ RuntimeType: RuntimeBackendDesktop,
+ InstanceMode: InstanceModePro,
+ WorkspacePath: &workspacePath,
+ }
+ if isSchedulerManagedV2Instance(instance) {
+ t.Fatal("pro desktop instance must not be scheduler-managed")
+ }
+
+ instance.RuntimeType = RuntimeBackendGateway
+ instance.InstanceMode = InstanceModeLite
+ if !isSchedulerManagedV2Instance(instance) {
+ t.Fatal("lite gateway instance should be scheduler-managed")
+ }
+}
+
+func ptrString(value string) *string {
+ return &value
+}
diff --git a/backend/internal/services/runtime_workspace_file_service.go b/backend/internal/services/runtime_workspace_file_service.go
new file mode 100644
index 0000000..7af41fe
--- /dev/null
+++ b/backend/internal/services/runtime_workspace_file_service.go
@@ -0,0 +1,660 @@
+package services
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "mime"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+ "time"
+
+ "clawreef/internal/models"
+ "clawreef/internal/repository"
+ "clawreef/internal/services/k8s"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/client-go/kubernetes/scheme"
+ "k8s.io/client-go/tools/remotecommand"
+)
+
+const (
+ runtimeWorkspaceErrPrefix = "CLAW_WORKSPACE_ERR:"
+ runtimeWorkspaceBaseDir = "/config"
+)
+
+type runtimeWorkspaceExecutor interface {
+ exec(ctx context.Context, userID, instanceID int, command []string, stdin io.Reader, stdout, stderr io.Writer) error
+}
+
+type runtimeWorkspaceFileService struct {
+ auditRepo repository.WorkspaceFileAuditRepository
+ executor runtimeWorkspaceExecutor
+}
+
+type k8sRuntimeWorkspaceExecutor struct {
+ deploymentService *k8s.InstanceDeploymentService
+}
+
+type runtimeWorkspaceEntryPayload struct {
+ Name string `json:"name"`
+ Path string `json:"path"`
+ IsDir bool `json:"is_dir"`
+ Size int64 `json:"size"`
+ ModifiedAt time.Time `json:"modified_at"`
+}
+
+func NewRuntimeWorkspaceFileService(auditRepo repository.WorkspaceFileAuditRepository) WorkspaceFileService {
+ return &runtimeWorkspaceFileService{
+ auditRepo: auditRepo,
+ executor: &k8sRuntimeWorkspaceExecutor{
+ deploymentService: k8s.NewInstanceDeploymentService(),
+ },
+ }
+}
+
+func (s *runtimeWorkspaceFileService) List(ctx context.Context, scope WorkspaceFileScope, relativePath string) ([]WorkspaceEntry, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ relative, err := cleanWorkspaceRelativePath(relativePath)
+ if err != nil {
+ return nil, err
+ }
+ var response struct {
+ Entries []runtimeWorkspaceEntryPayload `json:"entries"`
+ }
+ if err := s.runJSON(ctx, scope, []string{"python3", "-c", runtimeWorkspaceListScript, runtimeWorkspaceBase(scope), relative}, nil, &response); err != nil {
+ return nil, err
+ }
+ entries := make([]WorkspaceEntry, 0, len(response.Entries))
+ for _, payload := range response.Entries {
+ entry := runtimeWorkspaceEntryFromPayload(payload)
+ entries = append(entries, entry)
+ }
+ sort.Slice(entries, func(i, j int) bool {
+ if entries[i].IsDir != entries[j].IsDir {
+ return entries[i].IsDir
+ }
+ left := strings.ToLower(entries[i].Name)
+ right := strings.ToLower(entries[j].Name)
+ if left == right {
+ return entries[i].Name < entries[j].Name
+ }
+ return left < right
+ })
+ return entries, nil
+}
+
+func (s *runtimeWorkspaceFileService) Preview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspacePreview, error) {
+ relative, entry, err := s.statFile(ctx, scope, relativePath)
+ if err != nil {
+ return nil, err
+ }
+ kind, contentType, previewable, maxBytes := workspacePreviewKind(entry.Name, entry.Size)
+ downloadURL := workspaceDownloadURL(scope.InstanceID, relative)
+ if !previewable {
+ return &WorkspacePreview{
+ Kind: "binary",
+ ContentType: "application/octet-stream",
+ DownloadURL: downloadURL,
+ }, nil
+ }
+ if maxBytes > 0 && entry.Size > maxBytes {
+ return nil, ErrWorkspacePreviewTooLarge
+ }
+ if kind == "text" {
+ file, _, _, err := s.openRemoteFile(ctx, scope, relative, false)
+ if err != nil {
+ return nil, err
+ }
+ defer file.Close()
+ data, err := io.ReadAll(io.LimitReader(file, textPreviewMaxBytes+1))
+ if err != nil {
+ return nil, err
+ }
+ if int64(len(data)) > textPreviewMaxBytes {
+ return nil, ErrWorkspacePreviewTooLarge
+ }
+ return &WorkspacePreview{
+ Kind: "text",
+ ContentType: contentType,
+ Text: string(data),
+ DownloadURL: downloadURL,
+ }, nil
+ }
+
+ return &WorkspacePreview{
+ Kind: kind,
+ ContentType: contentType,
+ PreviewURL: workspacePreviewURL(scope.InstanceID, relative),
+ DownloadURL: downloadURL,
+ }, nil
+}
+
+func (s *runtimeWorkspaceFileService) OpenPreview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) {
+ _, entry, err := s.statFile(ctx, scope, relativePath)
+ if err != nil {
+ return nil, "", 0, err
+ }
+ _, contentType, previewable, maxBytes := workspacePreviewKind(entry.Name, entry.Size)
+ if !previewable {
+ return nil, "", 0, ErrWorkspaceFileExpected
+ }
+ if maxBytes > 0 && entry.Size > maxBytes {
+ return nil, "", 0, ErrWorkspacePreviewTooLarge
+ }
+ file, _, size, err := s.openRemoteFile(ctx, scope, relativePath, false)
+ if err != nil {
+ return nil, "", 0, err
+ }
+ return file, contentType, size, nil
+}
+
+func (s *runtimeWorkspaceFileService) OpenDownload(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) {
+ relative, entry, err := s.statFile(ctx, scope, relativePath)
+ if err != nil {
+ return nil, "", 0, err
+ }
+ file, filename, size, err := s.openRemoteFile(ctx, scope, relative, true)
+ if err != nil {
+ return nil, "", 0, err
+ }
+ if filename == "" {
+ filename = entry.Name
+ }
+ return file, filename, size, nil
+}
+
+func (s *runtimeWorkspaceFileService) Upload(ctx context.Context, scope WorkspaceFileScope, relativeDir string, filename string, reader io.Reader, size int64) (*WorkspaceEntry, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ maxBytes := WorkspaceUploadMaxBytes()
+ if size > maxBytes {
+ return nil, ErrWorkspaceUploadTooLarge
+ }
+ cleanName, err := sanitizeWorkspaceFilename(filename)
+ if err != nil {
+ return nil, err
+ }
+ dir, err := cleanWorkspaceRelativePath(relativeDir)
+ if err != nil {
+ return nil, err
+ }
+ targetRelative := joinWorkspaceRelative(dir, cleanName)
+ limited := io.LimitReader(reader, maxBytes+1)
+ var payload runtimeWorkspaceEntryPayload
+ if err := s.runJSON(ctx, scope, []string{
+ "python3",
+ "-c",
+ runtimeWorkspaceUploadScript,
+ runtimeWorkspaceBase(scope),
+ dir,
+ cleanName,
+ fmt.Sprintf("%d", maxBytes),
+ }, limited, &payload); err != nil {
+ return nil, err
+ }
+ if err := s.recordAudit(ctx, scope, "upload", targetRelative, payload.Size); err != nil {
+ return nil, err
+ }
+ entry := runtimeWorkspaceEntryFromPayload(payload)
+ return &entry, nil
+}
+
+func (s *runtimeWorkspaceFileService) Mkdir(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspaceEntry, error) {
+ if isWorkspaceRootRelative(relativePath) {
+ return nil, ErrWorkspaceRootOperation
+ }
+ relative, err := cleanWorkspaceRelativePath(relativePath)
+ if err != nil {
+ return nil, err
+ }
+ var payload runtimeWorkspaceEntryPayload
+ if err := s.runJSON(ctx, scope, []string{"python3", "-c", runtimeWorkspaceMkdirScript, runtimeWorkspaceBase(scope), relative}, nil, &payload); err != nil {
+ return nil, err
+ }
+ if err := s.recordAudit(ctx, scope, "mkdir", relative, 0); err != nil {
+ return nil, err
+ }
+ entry := runtimeWorkspaceEntryFromPayload(payload)
+ return &entry, nil
+}
+
+func (s *runtimeWorkspaceFileService) Rename(ctx context.Context, scope WorkspaceFileScope, oldPath, newPath string) (*WorkspaceEntry, error) {
+ if isWorkspaceRootRelative(oldPath) || isWorkspaceRootRelative(newPath) {
+ return nil, ErrWorkspaceRootOperation
+ }
+ oldRelative, err := cleanWorkspaceRelativePath(oldPath)
+ if err != nil {
+ return nil, err
+ }
+ newRelative, err := cleanWorkspaceRelativePath(newPath)
+ if err != nil {
+ return nil, err
+ }
+ var payload runtimeWorkspaceEntryPayload
+ if err := s.runJSON(ctx, scope, []string{"python3", "-c", runtimeWorkspaceRenameScript, runtimeWorkspaceBase(scope), oldRelative, newRelative}, nil, &payload); err != nil {
+ return nil, err
+ }
+ if err := s.recordAudit(ctx, scope, "rename", oldRelative+" -> "+newRelative, 0); err != nil {
+ return nil, err
+ }
+ entry := runtimeWorkspaceEntryFromPayload(payload)
+ return &entry, nil
+}
+
+func (s *runtimeWorkspaceFileService) Delete(ctx context.Context, scope WorkspaceFileScope, relativePath string) error {
+ if isWorkspaceRootRelative(relativePath) {
+ return ErrWorkspaceRootOperation
+ }
+ relative, err := cleanWorkspaceRelativePath(relativePath)
+ if err != nil {
+ return err
+ }
+ if err := s.run(ctx, scope, []string{"python3", "-c", runtimeWorkspaceDeleteScript, runtimeWorkspaceBase(scope), relative}, nil, io.Discard); err != nil {
+ return err
+ }
+ return s.recordAudit(ctx, scope, "delete", relative, 0)
+}
+
+func (s *runtimeWorkspaceFileService) statFile(ctx context.Context, scope WorkspaceFileScope, relativePath string) (string, WorkspaceEntry, error) {
+ relative, err := cleanWorkspaceRelativePath(relativePath)
+ if err != nil {
+ return "", WorkspaceEntry{}, err
+ }
+ var payload runtimeWorkspaceEntryPayload
+ if err := s.runJSON(ctx, scope, []string{"python3", "-c", runtimeWorkspaceStatFileScript, runtimeWorkspaceBase(scope), relative}, nil, &payload); err != nil {
+ return "", WorkspaceEntry{}, err
+ }
+ entry := runtimeWorkspaceEntryFromPayload(payload)
+ return relative, entry, nil
+}
+
+func (s *runtimeWorkspaceFileService) openRemoteFile(ctx context.Context, scope WorkspaceFileScope, relativePath string, audit bool) (*os.File, string, int64, error) {
+ relative, entry, err := s.statFile(ctx, scope, relativePath)
+ if err != nil {
+ return nil, "", 0, err
+ }
+ file, err := os.CreateTemp("", "clawmanager-runtime-workspace-*")
+ if err != nil {
+ return nil, "", 0, err
+ }
+ ok := false
+ defer func() {
+ if !ok {
+ name := file.Name()
+ _ = file.Close()
+ _ = os.Remove(name)
+ }
+ }()
+ if err := s.run(ctx, scope, []string{"python3", "-c", runtimeWorkspaceStreamFileScript, runtimeWorkspaceBase(scope), relative}, nil, file); err != nil {
+ return nil, "", 0, err
+ }
+ if _, err := file.Seek(0, io.SeekStart); err != nil {
+ return nil, "", 0, err
+ }
+ if audit {
+ if err := s.recordAudit(ctx, scope, "download", relative, entry.Size); err != nil {
+ return nil, "", 0, err
+ }
+ }
+ ok = true
+ return file, entry.Name, entry.Size, nil
+}
+
+func (s *runtimeWorkspaceFileService) runJSON(ctx context.Context, scope WorkspaceFileScope, command []string, stdin io.Reader, target any) error {
+ var stdout bytes.Buffer
+ if err := s.run(ctx, scope, command, stdin, &stdout); err != nil {
+ return err
+ }
+ if err := json.Unmarshal(stdout.Bytes(), target); err != nil {
+ return fmt.Errorf("failed to parse runtime workspace response: %w", err)
+ }
+ return nil
+}
+
+func (s *runtimeWorkspaceFileService) run(ctx context.Context, scope WorkspaceFileScope, command []string, stdin io.Reader, stdout io.Writer) error {
+ if s == nil || s.executor == nil {
+ return fmt.Errorf("runtime workspace executor is not configured")
+ }
+ var stderr bytes.Buffer
+ if err := s.executor.exec(ctx, scope.UserID, scope.InstanceID, command, stdin, stdout, &stderr); err != nil {
+ return mapRuntimeWorkspaceError(err, stderr.String())
+ }
+ return nil
+}
+
+func (s *runtimeWorkspaceFileService) recordAudit(ctx context.Context, scope WorkspaceFileScope, action, relativePath string, bytes int64) error {
+ if s.auditRepo == nil {
+ return nil
+ }
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ return s.auditRepo.Create(ctx, &models.WorkspaceFileAudit{
+ InstanceID: scope.InstanceID,
+ UserID: scope.UserID,
+ Action: action,
+ RelativePath: relativePath,
+ Bytes: bytes,
+ CreatedAt: time.Now().UTC(),
+ })
+}
+
+func (e *k8sRuntimeWorkspaceExecutor) exec(ctx context.Context, userID, instanceID int, command []string, stdin io.Reader, stdout, stderr io.Writer) error {
+ client := k8s.GetClient()
+ if client == nil || client.Clientset == nil || client.Config == nil {
+ return fmt.Errorf("k8s client not initialized")
+ }
+ deploymentService := e.deploymentService
+ if deploymentService == nil {
+ deploymentService = k8s.NewInstanceDeploymentService()
+ }
+ pod, err := deploymentService.GetActivePod(ctx, userID, instanceID)
+ if err != nil {
+ return fmt.Errorf("failed to get active instance pod: %w", err)
+ }
+ req := client.Clientset.CoreV1().RESTClient().Post().
+ Resource("pods").
+ Name(pod.Name).
+ Namespace(pod.Namespace).
+ SubResource("exec")
+ req.VersionedParams(&corev1.PodExecOptions{
+ Container: "desktop",
+ Command: command,
+ Stdin: stdin != nil,
+ Stdout: stdout != nil,
+ Stderr: stderr != nil,
+ TTY: false,
+ }, scheme.ParameterCodec)
+ exec, err := remotecommand.NewSPDYExecutor(client.Config, "POST", req.URL())
+ if err != nil {
+ return fmt.Errorf("failed to initialize runtime workspace exec stream: %w", err)
+ }
+ return exec.StreamWithContext(ctx, remotecommand.StreamOptions{
+ Stdin: stdin,
+ Stdout: stdout,
+ Stderr: stderr,
+ Tty: false,
+ })
+}
+
+func runtimeWorkspaceBase(scope WorkspaceFileScope) string {
+ if base := strings.TrimSpace(scope.WorkspacePath); base != "" {
+ return base
+ }
+ return runtimeWorkspaceBaseDir
+}
+
+func runtimeWorkspaceEntryFromPayload(payload runtimeWorkspaceEntryPayload) WorkspaceEntry {
+ isDir := payload.IsDir
+ return WorkspaceEntry{
+ Name: payload.Name,
+ Path: filepath.ToSlash(payload.Path),
+ IsDir: isDir,
+ Size: payload.Size,
+ ModifiedAt: payload.ModifiedAt.UTC(),
+ Previewable: workspaceEntryPreviewable(payload.Name, payload.Size, isDir),
+ Downloadable: !isDir,
+ }
+}
+
+func mapRuntimeWorkspaceError(execErr error, stderr string) error {
+ code := runtimeWorkspaceErrorCode(stderr)
+ switch code {
+ case "not_found":
+ return ErrWorkspacePathNotFound
+ case "escape":
+ return ErrWorkspacePathEscape
+ case "dir_required":
+ return ErrWorkspaceDirectoryExpected
+ case "file_required":
+ return ErrWorkspaceFileExpected
+ case "exists":
+ return ErrWorkspaceEntryExists
+ case "upload_too_large":
+ return ErrWorkspaceUploadTooLarge
+ case "filename_invalid", "invalid":
+ return ErrWorkspacePathInvalid
+ }
+ if strings.TrimSpace(stderr) != "" {
+ return fmt.Errorf("runtime workspace operation failed: %s", strings.TrimSpace(stderr))
+ }
+ if execErr != nil {
+ return execErr
+ }
+ return errors.New("runtime workspace operation failed")
+}
+
+func runtimeWorkspaceErrorCode(stderr string) string {
+ for _, line := range strings.Split(stderr, "\n") {
+ line = strings.TrimSpace(line)
+ if strings.HasPrefix(line, runtimeWorkspaceErrPrefix) {
+ remainder := strings.TrimPrefix(line, runtimeWorkspaceErrPrefix)
+ if index := strings.Index(remainder, ":"); index >= 0 {
+ return remainder[:index]
+ }
+ return remainder
+ }
+ }
+ return ""
+}
+
+const runtimeWorkspacePythonPrelude = `
+import datetime
+import json
+import os
+import shutil
+import stat
+import sys
+import tempfile
+
+ERR_PREFIX = "CLAW_WORKSPACE_ERR:"
+
+def fail(code, message=""):
+ print(ERR_PREFIX + code + ":" + str(message), file=sys.stderr)
+ sys.exit(1)
+
+def clean_rel(value):
+ raw = (value or "").strip().replace("\\", "/")
+ if raw in ("", "."):
+ return ""
+ if "\x00" in raw:
+ fail("invalid", "path contains null byte")
+ if raw.startswith("/"):
+ fail("escape", "absolute paths are not allowed")
+ parts = []
+ for part in raw.split("/"):
+ if part in ("", "."):
+ continue
+ if part == "..":
+ fail("escape", "traversal is not allowed")
+ parts.append(part)
+ return "/".join(parts)
+
+def safe_base(value):
+ base = os.path.realpath(value or "/config")
+ if not os.path.isdir(base):
+ fail("not_found", "workspace root")
+ return base
+
+def is_subpath(base, target):
+ try:
+ return os.path.commonpath([base, target]) == base
+ except ValueError:
+ return False
+
+def joined_path(base, rel):
+ target = os.path.abspath(os.path.join(base, rel)) if rel else base
+ if not is_subpath(base, target):
+ fail("escape", rel)
+ return target
+
+def target_for(base, rel, allow_missing=False):
+ rel = clean_rel(rel)
+ target = joined_path(base, rel)
+ if os.path.lexists(target):
+ real = os.path.realpath(target)
+ if not is_subpath(base, real):
+ fail("escape", rel)
+ return rel, target
+ if allow_missing:
+ parent = os.path.realpath(os.path.dirname(target))
+ if not os.path.isdir(parent):
+ fail("invalid", "parent is not a directory")
+ if not is_subpath(base, parent):
+ fail("escape", rel)
+ return rel, target
+ fail("not_found", rel)
+
+def entry_payload(base, rel, target):
+ try:
+ st = os.lstat(target)
+ except FileNotFoundError:
+ fail("not_found", rel)
+ name = os.path.basename(target) if rel else os.path.basename(base)
+ modified = datetime.datetime.fromtimestamp(st.st_mtime, datetime.timezone.utc).isoformat().replace("+00:00", "Z")
+ return {
+ "name": name,
+ "path": rel,
+ "is_dir": stat.S_ISDIR(st.st_mode),
+ "size": int(st.st_size),
+ "modified_at": modified,
+ }
+
+def json_out(value):
+ print(json.dumps(value, separators=(",", ":")))
+
+def chown_abc(path):
+ try:
+ import grp
+ import pwd
+ os.chown(path, pwd.getpwnam("abc").pw_uid, grp.getgrnam("abc").gr_gid)
+ except Exception:
+ pass
+`
+
+var runtimeWorkspaceListScript = runtimeWorkspacePythonPrelude + `
+base = safe_base(sys.argv[1])
+rel, target = target_for(base, sys.argv[2] if len(sys.argv) > 2 else "")
+if not os.path.isdir(target):
+ fail("dir_required", rel)
+entries = []
+for entry in os.scandir(target):
+ child_rel = (rel + "/" + entry.name) if rel else entry.name
+ child_path = os.path.join(target, entry.name)
+ try:
+ real = os.path.realpath(child_path)
+ if not is_subpath(base, real):
+ continue
+ except Exception:
+ continue
+ entries.append(entry_payload(base, child_rel, child_path))
+json_out({"entries": entries})
+`
+
+var runtimeWorkspaceStatFileScript = runtimeWorkspacePythonPrelude + `
+base = safe_base(sys.argv[1])
+rel, target = target_for(base, sys.argv[2] if len(sys.argv) > 2 else "")
+if not os.path.isfile(target):
+ fail("file_required", rel)
+json_out(entry_payload(base, rel, target))
+`
+
+var runtimeWorkspaceStreamFileScript = runtimeWorkspacePythonPrelude + `
+base = safe_base(sys.argv[1])
+rel, target = target_for(base, sys.argv[2] if len(sys.argv) > 2 else "")
+if not os.path.isfile(target):
+ fail("file_required", rel)
+with open(target, "rb") as source:
+ shutil.copyfileobj(source, sys.stdout.buffer)
+`
+
+var runtimeWorkspaceUploadScript = runtimeWorkspacePythonPrelude + `
+base = safe_base(sys.argv[1])
+dir_rel, dir_path = target_for(base, sys.argv[2] if len(sys.argv) > 2 else "")
+name = sys.argv[3] if len(sys.argv) > 3 else ""
+max_bytes = int(sys.argv[4]) if len(sys.argv) > 4 else 524288000
+if not name or name in (".", "..") or "/" in name or "\\" in name or "\x00" in name:
+ fail("filename_invalid", name)
+if not os.path.isdir(dir_path):
+ fail("dir_required", dir_rel)
+target_rel = (dir_rel + "/" + name) if dir_rel else name
+target_rel, target = target_for(base, target_rel, allow_missing=True)
+if os.path.isdir(target):
+ fail("file_required", target_rel)
+tmp_fd, tmp_name = tempfile.mkstemp(prefix="." + name + ".tmp-", dir=dir_path)
+written = 0
+try:
+ with os.fdopen(tmp_fd, "wb") as out:
+ while True:
+ chunk = sys.stdin.buffer.read(1024 * 1024)
+ if not chunk:
+ break
+ written += len(chunk)
+ if written > max_bytes:
+ fail("upload_too_large", target_rel)
+ out.write(chunk)
+ chown_abc(tmp_name)
+ os.replace(tmp_name, target)
+ chown_abc(target)
+ json_out(entry_payload(base, target_rel, target))
+finally:
+ try:
+ if os.path.exists(tmp_name):
+ os.unlink(tmp_name)
+ except Exception:
+ pass
+`
+
+var runtimeWorkspaceMkdirScript = runtimeWorkspacePythonPrelude + `
+base = safe_base(sys.argv[1])
+rel = clean_rel(sys.argv[2] if len(sys.argv) > 2 else "")
+if not rel:
+ fail("invalid", "workspace root cannot be modified")
+rel, target = target_for(base, rel, allow_missing=True)
+if os.path.lexists(target):
+ fail("exists", rel)
+os.makedirs(target, mode=0o750, exist_ok=False)
+chown_abc(target)
+json_out(entry_payload(base, rel, target))
+`
+
+var runtimeWorkspaceRenameScript = runtimeWorkspacePythonPrelude + `
+base = safe_base(sys.argv[1])
+old_rel, old_target = target_for(base, sys.argv[2] if len(sys.argv) > 2 else "")
+new_rel, new_target = target_for(base, sys.argv[3] if len(sys.argv) > 3 else "", allow_missing=True)
+if not old_rel or not new_rel:
+ fail("invalid", "workspace root cannot be modified")
+if os.path.lexists(new_target):
+ fail("exists", new_rel)
+os.rename(old_target, new_target)
+json_out(entry_payload(base, new_rel, new_target))
+`
+
+var runtimeWorkspaceDeleteScript = runtimeWorkspacePythonPrelude + `
+base = safe_base(sys.argv[1])
+rel, target = target_for(base, sys.argv[2] if len(sys.argv) > 2 else "")
+if not rel:
+ fail("invalid", "workspace root cannot be modified")
+if os.path.islink(target) or os.path.isfile(target):
+ os.unlink(target)
+elif os.path.isdir(target):
+ shutil.rmtree(target)
+else:
+ fail("not_found", rel)
+`
+
+func runtimeWorkspaceContentType(name string) string {
+ contentType := mime.TypeByExtension(strings.ToLower(filepath.Ext(name)))
+ if contentType == "" {
+ return "application/octet-stream"
+ }
+ return contentType
+}
diff --git a/backend/internal/services/sync_service.go b/backend/internal/services/sync_service.go
index da90987..937e22d 100644
--- a/backend/internal/services/sync_service.go
+++ b/backend/internal/services/sync_service.go
@@ -9,6 +9,7 @@ import (
"clawreef/internal/repository"
"clawreef/internal/services/k8s"
+ appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
)
@@ -17,6 +18,7 @@ type SyncService struct {
instanceRepo repository.InstanceRepository
runtimeStatusService InstanceRuntimeStatusService
podService *k8s.PodService
+ deploymentService *k8s.InstanceDeploymentService
interval time.Duration
stopChan chan struct{}
}
@@ -27,6 +29,7 @@ func NewSyncService(instanceRepo repository.InstanceRepository, runtimeStatusSer
instanceRepo: instanceRepo,
runtimeStatusService: runtimeStatusService,
podService: k8s.NewPodService(),
+ deploymentService: k8s.NewInstanceDeploymentService(),
interval: 5 * time.Second, // Sync every 5 seconds for more responsive status updates
stopChan: make(chan struct{}),
}
@@ -96,6 +99,15 @@ func (s *SyncService) syncAllInstances() {
// syncInstance synchronizes a single instance's state
func (s *SyncService) syncInstance(ctx context.Context, instance *models.Instance) {
+ if _, ok := v2RuntimeTypeForInstance(instance); ok {
+ s.updateInfraStatus(instance.ID, instance.Status)
+ return
+ }
+ if instanceUsesDesktopRuntime(instance) {
+ s.syncDeploymentInstance(ctx, instance)
+ return
+ }
+
// Check if pod exists in K8s
pod, err := s.podService.GetPod(ctx, instance.UserID, instance.ID)
if err != nil {
@@ -196,6 +208,73 @@ func (s *SyncService) syncInstance(ctx context.Context, instance *models.Instanc
}
}
+func (s *SyncService) syncDeploymentInstance(ctx context.Context, instance *models.Instance) {
+ deployment, err := s.deploymentService.GetDeployment(ctx, instance.UserID, instance.ID)
+ if err != nil {
+ if instance.Status == "running" || instance.Status == "creating" {
+ nextStatus := "stopped"
+ if instance.Status == "creating" {
+ nextStatus = "error"
+ }
+ fmt.Printf("Instance %d marked as %s but deployment not found in K8s, updating status to %s\n",
+ instance.ID, instance.Status, nextStatus)
+ instance.Status = nextStatus
+ instance.PodName = nil
+ instance.PodNamespace = nil
+ instance.PodIP = nil
+ instance.UpdatedAt = time.Now()
+ if err := s.instanceRepo.Update(instance); err != nil {
+ fmt.Printf("Error updating instance %d status: %v\n", instance.ID, err)
+ } else {
+ s.updateInfraStatus(instance.ID, nextStatus)
+ GetHub().BroadcastInstanceStatus(instance.UserID, instance)
+ }
+ }
+ return
+ }
+
+ desiredStatus := mapDeploymentToInstanceStatus(deployment)
+ needsUpdate := false
+ if instance.Status != desiredStatus {
+ fmt.Printf("Instance %d: Deployment replicas=%d available=%d but instance status is %s, updating to %s\n",
+ instance.ID, desiredReplicas(deployment), deployment.Status.AvailableReplicas, instance.Status, desiredStatus)
+ instance.Status = desiredStatus
+ needsUpdate = true
+ }
+ s.updateInfraStatus(instance.ID, desiredStatus)
+
+ if pod, podErr := s.deploymentService.GetActivePod(ctx, instance.UserID, instance.ID); podErr == nil && pod != nil {
+ if pod.Status.PodIP != "" && (instance.PodIP == nil || *instance.PodIP != pod.Status.PodIP) {
+ instance.PodIP = &pod.Status.PodIP
+ needsUpdate = true
+ }
+ if instance.PodName == nil || *instance.PodName != pod.Name {
+ instance.PodName = &pod.Name
+ needsUpdate = true
+ }
+ if instance.PodNamespace == nil || *instance.PodNamespace != pod.Namespace {
+ instance.PodNamespace = &pod.Namespace
+ needsUpdate = true
+ }
+ } else if desiredStatus == "stopped" {
+ if instance.PodName != nil || instance.PodNamespace != nil || instance.PodIP != nil {
+ instance.PodName = nil
+ instance.PodNamespace = nil
+ instance.PodIP = nil
+ needsUpdate = true
+ }
+ }
+
+ if needsUpdate {
+ instance.UpdatedAt = time.Now()
+ if err := s.instanceRepo.Update(instance); err != nil {
+ fmt.Printf("Error updating instance %d: %v\n", instance.ID, err)
+ } else {
+ GetHub().BroadcastInstanceStatus(instance.UserID, instance)
+ }
+ }
+}
+
func (s *SyncService) updateInfraStatus(instanceID int, instanceStatus string) {
if s.runtimeStatusService == nil {
return
@@ -243,6 +322,26 @@ func mapPodToInstanceStatus(pod *corev1.Pod) string {
}
}
+func mapDeploymentToInstanceStatus(deployment *appsv1.Deployment) string {
+ if deployment == nil {
+ return "error"
+ }
+ if desiredReplicas(deployment) == 0 {
+ return "stopped"
+ }
+ if deployment.Status.AvailableReplicas > 0 {
+ return "running"
+ }
+ return "creating"
+}
+
+func desiredReplicas(deployment *appsv1.Deployment) int32 {
+ if deployment == nil || deployment.Spec.Replicas == nil {
+ return 1
+ }
+ return *deployment.Spec.Replicas
+}
+
func isPodReady(pod *corev1.Pod) bool {
for _, condition := range pod.Status.Conditions {
if condition.Type == corev1.PodReady {
diff --git a/backend/internal/services/system_image_setting_service.go b/backend/internal/services/system_image_setting_service.go
index 93f535d..7a564c5 100644
--- a/backend/internal/services/system_image_setting_service.go
+++ b/backend/internal/services/system_image_setting_service.go
@@ -20,10 +20,10 @@ var orderedSystemImageTypes = []string{
}
var supportedSystemImageTypes = map[string]string{
- "openclaw": "OpenClaw Desktop",
+ "openclaw": "OpenClaw Pro",
"ubuntu": "Ubuntu Desktop",
"webtop": "Webtop Desktop",
- "hermes": "Hermes Runtime",
+ "hermes": "Hermes Pro",
"debian": "Debian Desktop",
"centos": "CentOS Desktop",
"custom": "Custom Image",
@@ -39,11 +39,11 @@ var defaultSystemImageSettings = map[string]string{
"custom": "registry.example.com/your-custom-image:latest",
}
-var defaultShellSystemImageSettings = map[string]string{
- "openclaw": "ghcr.io/yuan-lab-llm/agentsruntime/openclaw-shell:latest",
+var defaultGatewaySystemImageSettings = map[string]string{
+ "openclaw": "ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest",
"ubuntu": "ubuntu:22.04",
"webtop": "ubuntu:22.04",
- "hermes": "ghcr.io/yuan-lab-llm/agentsruntime/hermes-shell:latest",
+ "hermes": "ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest",
"debian": "debian:12",
"centos": "quay.io/centos/centos:stream9",
"custom": "registry.example.com/your-custom-shell-image:latest",
@@ -55,8 +55,9 @@ var defaultEnabledSystemImageTypes = map[string]bool{
"hermes": true,
}
-var defaultEnabledShellSystemImageTypes = map[string]bool{
+var defaultEnabledGatewaySystemImageTypes = map[string]bool{
"openclaw": true,
+ "hermes": true,
}
// RuntimeImageConfig is the runtime card selected for an instance type.
@@ -320,10 +321,16 @@ func displayNameForSystemImageType(instanceType string) string {
func displayNameForSystemImagePreset(instanceType, runtimeType string) string {
normalizedRuntimeType := normalizeSystemImageRuntimeType(runtimeType)
if instanceType == "openclaw" {
- if normalizedRuntimeType == "shell" {
- return "OpenClaw Shell"
+ if normalizedRuntimeType == "gateway" {
+ return "OpenClaw Lite"
}
- return "OpenClaw Desktop"
+ return "OpenClaw Pro"
+ }
+ if instanceType == "hermes" {
+ if normalizedRuntimeType == "gateway" {
+ return "Hermes Lite"
+ }
+ return "Hermes Pro"
}
return displayNameForSystemImageType(instanceType)
}
@@ -337,12 +344,12 @@ func defaultSystemImagePresetsForType(instanceType string) []models.SystemImageS
IsEnabled: defaultEnabledSystemImageTypes[instanceType],
}}
- if image := strings.TrimSpace(defaultShellSystemImageSettings[instanceType]); image != "" {
- if defaultEnabledShellSystemImageTypes[instanceType] {
+ if image := strings.TrimSpace(defaultGatewaySystemImageSettings[instanceType]); image != "" {
+ if defaultEnabledGatewaySystemImageTypes[instanceType] {
settings = append(settings, models.SystemImageSetting{
InstanceType: instanceType,
- RuntimeType: "shell",
- DisplayName: displayNameForSystemImagePreset(instanceType, "shell"),
+ RuntimeType: "gateway",
+ DisplayName: displayNameForSystemImagePreset(instanceType, "gateway"),
Image: image,
IsEnabled: true,
})
@@ -359,8 +366,8 @@ func isSupportedSystemImageType(instanceType string) bool {
func normalizeSystemImageRuntimeType(runtimeType string) string {
normalized := strings.TrimSpace(strings.ToLower(runtimeType))
- if normalized == "shell" {
- return "shell"
+ if normalized == "gateway" || normalized == "shell" {
+ return "gateway"
}
return "desktop"
}
@@ -370,7 +377,10 @@ func validateSystemImageRuntimeType(runtimeType string) (string, error) {
if normalized == "" {
return "desktop", nil
}
- if normalized != "desktop" && normalized != "shell" {
+ if normalized == "shell" {
+ return "gateway", nil
+ }
+ if normalized != "desktop" && normalized != "gateway" {
return "", errors.New("unsupported runtime type")
}
return normalized, nil
diff --git a/backend/internal/services/system_image_setting_service_test.go b/backend/internal/services/system_image_setting_service_test.go
index 6e562d6..3fcd060 100644
--- a/backend/internal/services/system_image_setting_service_test.go
+++ b/backend/internal/services/system_image_setting_service_test.go
@@ -150,7 +150,7 @@ func TestSystemImageSettingServiceGetRuntimeImageFallsBackToDefaultWhenNoRowsExi
}
}
-func TestSystemImageSettingServiceListIncludesOpenClawShellDefault(t *testing.T) {
+func TestSystemImageSettingServiceListIncludesOpenClawGatewayDefault(t *testing.T) {
service := NewSystemImageSettingService(&stubSystemImageSettingRepository{})
items, err := service.List()
@@ -160,34 +160,63 @@ func TestSystemImageSettingServiceListIncludesOpenClawShellDefault(t *testing.T)
found := false
for _, item := range items {
- if item.InstanceType == "openclaw" && item.RuntimeType == "shell" {
+ if item.InstanceType == "openclaw" && item.RuntimeType == "gateway" {
found = true
- if item.Image != defaultShellSystemImageSettings["openclaw"] {
- t.Fatalf("expected default openclaw shell image %q, got %q", defaultShellSystemImageSettings["openclaw"], item.Image)
+ if item.Image != defaultGatewaySystemImageSettings["openclaw"] {
+ t.Fatalf("expected default openclaw gateway image %q, got %q", defaultGatewaySystemImageSettings["openclaw"], item.Image)
}
if !item.IsEnabled {
- t.Fatalf("expected default openclaw shell image to be enabled")
+ t.Fatalf("expected default openclaw gateway image to be enabled")
}
}
}
if !found {
- t.Fatalf("expected default openclaw shell runtime image")
+ t.Fatalf("expected default openclaw gateway runtime image")
}
}
-func TestSystemImageSettingServiceGetRuntimeImageForImageFallsBackToOpenClawShellDefault(t *testing.T) {
+func TestSystemImageSettingServiceListIncludesHermesLiteDefault(t *testing.T) {
service := NewSystemImageSettingService(&stubSystemImageSettingRepository{})
- selection, ok := service.GetRuntimeImageForImage("openclaw", defaultShellSystemImageSettings["openclaw"])
+ items, err := service.List()
+ if err != nil {
+ t.Fatalf("List returned error: %v", err)
+ }
+
+ found := false
+ for _, item := range items {
+ if item.InstanceType == "hermes" && item.RuntimeType == "gateway" {
+ found = true
+ if item.Image != defaultGatewaySystemImageSettings["hermes"] {
+ t.Fatalf("expected default hermes lite image %q, got %q", defaultGatewaySystemImageSettings["hermes"], item.Image)
+ }
+ if item.DisplayName != "Hermes Lite" {
+ t.Fatalf("expected Hermes Lite display name, got %q", item.DisplayName)
+ }
+ if !item.IsEnabled {
+ t.Fatalf("expected default hermes lite image to be enabled")
+ }
+ }
+ }
+
+ if !found {
+ t.Fatalf("expected default hermes lite runtime image")
+ }
+}
+
+func TestSystemImageSettingServiceGetRuntimeImageForImageFallsBackToOpenClawGatewayDefault(t *testing.T) {
+ service := NewSystemImageSettingService(&stubSystemImageSettingRepository{})
+
+ selection, ok := service.GetRuntimeImageForImage("openclaw", defaultGatewaySystemImageSettings["openclaw"])
if !ok {
- t.Fatalf("expected default openclaw shell runtime image to resolve")
+ t.Fatalf("expected default openclaw gateway runtime image to resolve")
}
- if selection.RuntimeType != "shell" {
- t.Fatalf("expected runtime type shell, got %q", selection.RuntimeType)
+ if selection.RuntimeType != "gateway" {
+ t.Fatalf("expected runtime type gateway, got %q", selection.RuntimeType)
}
- if selection.Image != defaultShellSystemImageSettings["openclaw"] {
- t.Fatalf("expected shell image %q, got %q", defaultShellSystemImageSettings["openclaw"], selection.Image)
+ if selection.Image != defaultGatewaySystemImageSettings["openclaw"] {
+ t.Fatalf("expected gateway image %q, got %q", defaultGatewaySystemImageSettings["openclaw"], selection.Image)
}
}
@@ -195,20 +224,70 @@ func TestSystemImageSettingServiceGetRuntimeImageForImageUsesCardRuntimeType(t *
repo := &stubSystemImageSettingRepository{
items: []models.SystemImageSetting{
{ID: 1, InstanceType: "openclaw", RuntimeType: "desktop", DisplayName: "OpenClaw Desktop", Image: "registry/openclaw-desktop:latest", IsEnabled: true},
- {ID: 2, InstanceType: "openclaw", RuntimeType: "shell", DisplayName: "OpenClaw Shell", Image: "registry/openclaw-shell:latest", IsEnabled: true},
+ {ID: 2, InstanceType: "openclaw", RuntimeType: "gateway", DisplayName: "OpenClaw Lite", Image: "registry/openclaw-lite:latest", IsEnabled: true},
},
nextID: 2,
}
service := NewSystemImageSettingService(repo)
- selection, ok := service.GetRuntimeImageForImage("openclaw", "registry/openclaw-shell:latest")
+ selection, ok := service.GetRuntimeImageForImage("openclaw", "registry/openclaw-lite:latest")
if !ok {
- t.Fatalf("expected selected shell runtime image to resolve")
+ t.Fatalf("expected selected gateway runtime image to resolve")
}
- if selection.RuntimeType != "shell" {
- t.Fatalf("expected runtime type shell, got %q", selection.RuntimeType)
+ if selection.RuntimeType != "gateway" {
+ t.Fatalf("expected runtime type gateway, got %q", selection.RuntimeType)
}
- if selection.Image != "registry/openclaw-shell:latest" {
- t.Fatalf("expected shell image to resolve, got %q", selection.Image)
+ if selection.Image != "registry/openclaw-lite:latest" {
+ t.Fatalf("expected gateway image to resolve, got %q", selection.Image)
+ }
+}
+
+func TestSystemImageSettingServiceSaveAcceptsGatewayRuntimeType(t *testing.T) {
+ repo := &stubSystemImageSettingRepository{}
+ service := NewSystemImageSettingService(repo)
+
+ saved, err := service.Save(&models.SystemImageSetting{
+ InstanceType: "OpenClaw",
+ RuntimeType: "gateway",
+ DisplayName: "OpenClaw Lite",
+ Image: "registry/openclaw-lite:v2",
+ })
+ if err != nil {
+ t.Fatalf("Save returned error: %v", err)
+ }
+
+ if saved.InstanceType != "openclaw" {
+ t.Fatalf("expected normalized instance type openclaw, got %q", saved.InstanceType)
+ }
+ if saved.RuntimeType != "gateway" {
+ t.Fatalf("expected gateway runtime type, got %q", saved.RuntimeType)
+ }
+}
+
+func TestSystemImageSettingServiceNormalizesLegacyShellRuntimeTypeToGateway(t *testing.T) {
+ repo := &stubSystemImageSettingRepository{
+ items: []models.SystemImageSetting{
+ {ID: 1, InstanceType: "hermes", RuntimeType: "shell", DisplayName: "Hermes Lite", Image: "registry/hermes-lite:legacy", IsEnabled: true},
+ },
+ nextID: 1,
+ }
+ service := NewSystemImageSettingService(repo)
+
+ items, err := service.List()
+ if err != nil {
+ t.Fatalf("List returned error: %v", err)
+ }
+ for _, item := range items {
+ if item.ID == 1 && item.RuntimeType != "gateway" {
+ t.Fatalf("expected legacy shell row to be exposed as gateway, got %q", item.RuntimeType)
+ }
+ }
+
+ selection, ok := service.GetRuntimeImageForImage("hermes", "registry/hermes-lite:legacy")
+ if !ok {
+ t.Fatalf("expected legacy shell image to resolve")
+ }
+ if selection.RuntimeType != "gateway" {
+ t.Fatalf("expected legacy shell image to resolve as gateway, got %q", selection.RuntimeType)
}
}
diff --git a/backend/internal/services/team_redis.go b/backend/internal/services/team_redis.go
index 4e426eb..7856551 100644
--- a/backend/internal/services/team_redis.go
+++ b/backend/internal/services/team_redis.go
@@ -70,6 +70,25 @@ func (b *redisBus) XAdd(ctx context.Context, key string, fields map[string]strin
return id, nil
}
+func (b *redisBus) SetNX(ctx context.Context, key, value string, ttl time.Duration) (bool, error) {
+ if ttl <= 0 {
+ ttl = time.Second
+ }
+ reply, err := b.do(ctx, "SET", key, value, "NX", "PX", fmt.Sprintf("%d", ttl.Milliseconds()))
+ if err != nil {
+ return false, err
+ }
+ if reply == nil {
+ return false, nil
+ }
+ return reply == "OK", nil
+}
+
+func (b *redisBus) Del(ctx context.Context, key string) error {
+ _, err := b.do(ctx, "DEL", key)
+ return err
+}
+
func (b *redisBus) XRead(ctx context.Context, key, lastID string, block time.Duration) ([]redisStreamMessage, error) {
blockMillis := int(block / time.Millisecond)
if blockMillis <= 0 {
diff --git a/backend/internal/services/team_service.go b/backend/internal/services/team_service.go
index f85738c..a0fa5aa 100644
--- a/backend/internal/services/team_service.go
+++ b/backend/internal/services/team_service.go
@@ -31,6 +31,8 @@ const (
teamTaskStaleSweepInterval = 30 * time.Second
initialLeaderTaskIntent = "team_bootstrap_introduction"
+ teamTaskCompletionTool = "team_complete_task"
+ teamTaskReplyTarget = "clawmanager"
)
var (
@@ -66,6 +68,8 @@ type CreateTeamMemberRequest struct {
MemberID string `json:"member_id,omitempty"`
Name string `json:"name,omitempty"`
Role string `json:"role"`
+ Mode string `json:"mode,omitempty"`
+ InstanceMode string `json:"instance_mode,omitempty"`
RuntimeType string `json:"runtime_type,omitempty"`
Description *string `json:"description,omitempty"`
CPUCores float64 `json:"cpu_cores,omitempty"`
@@ -130,36 +134,60 @@ type teamService struct {
secretService *k8s.SecretService
configMapService *k8s.ConfigMapService
- ctx context.Context
- cancel context.CancelFunc
- mu sync.Mutex
- consumers map[int]struct{}
- staleMonitorStarted bool
+ ctx context.Context
+ cancel context.CancelFunc
+ mu sync.Mutex
+ consumers map[int]struct{}
+ staleMonitorStarted bool
+ runtimeWorkspaceRoot string
}
type plannedTeamMember struct {
- Request CreateTeamMemberRequest
- MemberKey string
- DisplayName string
- Role string
- RuntimeType string
- IsLeader bool
+ Request CreateTeamMemberRequest
+ MemberKey string
+ DisplayName string
+ Role string
+ RuntimeType string
+ InstanceMode string
+ IsLeader bool
}
-func NewTeamService(repo repository.TeamRepository, instanceService InstanceService) TeamService {
- ctx, cancel := context.WithCancel(context.Background())
- return &teamService{
- repo: repo,
- instanceService: instanceService,
- pvcService: k8s.NewPVCService(),
- secretService: k8s.NewSecretService(),
- configMapService: k8s.NewConfigMapService(),
- ctx: ctx,
- cancel: cancel,
- consumers: map[int]struct{}{},
+type teamRuntimeSecrets struct {
+ RedisURL string
+ Token string
+}
+
+type TeamServiceOption func(*teamService)
+
+func WithTeamRuntimeWorkspaceRoot(root string) TeamServiceOption {
+ return func(s *teamService) {
+ if strings.TrimSpace(root) != "" {
+ s.runtimeWorkspaceRoot = strings.TrimSpace(root)
+ }
}
}
+func NewTeamService(repo repository.TeamRepository, instanceService InstanceService, opts ...TeamServiceOption) TeamService {
+ ctx, cancel := context.WithCancel(context.Background())
+ service := &teamService{
+ repo: repo,
+ instanceService: instanceService,
+ pvcService: k8s.NewPVCService(),
+ secretService: k8s.NewSecretService(),
+ configMapService: k8s.NewConfigMapService(),
+ ctx: ctx,
+ cancel: cancel,
+ consumers: map[int]struct{}{},
+ runtimeWorkspaceRoot: "/workspaces",
+ }
+ for _, opt := range opts {
+ if opt != nil {
+ opt(service)
+ }
+ }
+ return service
+}
+
func (s *teamService) Start() {
teams, err := s.repo.ListActiveTeams()
if err != nil {
@@ -252,15 +280,17 @@ func (s *teamService) CreateTeam(userID int, req CreateTeamRequest) (*TeamDetail
return nil, s.rollbackTeamCreation(userID, team, err)
}
- if err := s.provisionTeamK8s(userID, team, redisURL, sharedStorageGB, strings.TrimSpace(req.StorageClass)); err != nil {
+ runtimeSecrets, err := s.provisionTeamK8s(userID, team, redisURL, sharedStorageGB, strings.TrimSpace(req.StorageClass))
+ if err != nil {
return nil, s.rollbackTeamCreation(userID, team, err)
}
- if err := s.upsertTeamRosterConfig(userID, team, memberPlans); err != nil {
+ rosterJSON, err := s.upsertTeamRosterConfig(userID, team, memberPlans)
+ if err != nil {
return nil, s.rollbackTeamCreation(userID, team, err)
}
for _, memberPlan := range memberPlans {
- member, err := s.createTeamMemberInstance(userID, team, memberPlan)
+ member, err := s.createTeamMemberInstance(userID, team, memberPlan, runtimeSecrets, rosterJSON)
if err != nil {
return nil, s.rollbackTeamCreation(userID, team, err)
}
@@ -339,16 +369,84 @@ func (s *teamService) recordInitialLeaderTaskDispatchFailure(teamID int, cause e
})
}
-func (s *teamService) provisionTeamK8s(userID int, team *models.Team, redisURL string, sharedStorageGB int, storageClass string) error {
+func buildTeamTaskEnvelope(teamID int, memberKey string, task *models.TeamTask, messageID string, taskPayload map[string]interface{}, now time.Time) map[string]interface{} {
+ if taskPayload == nil {
+ taskPayload = map[string]interface{}{}
+ }
+ taskID := 0
+ if task != nil {
+ taskID = task.ID
+ }
+ taskRef := fmt.Sprintf("team-%d-task-%d", teamID, taskID)
+ prompt := eventString(taskPayload, "prompt", "goal", "instruction", "instructions")
+ if prompt == "" {
+ rawPayload, _ := marshalJSON(taskPayload)
+ prompt = rawPayload
+ }
+ envelope := map[string]interface{}{
+ "v": 1,
+ "messageId": messageID,
+ "teamId": strconv.Itoa(teamID),
+ "from": "clawmanager",
+ "to": memberKey,
+ "replyTo": teamTaskReplyTarget,
+ "requiresCompletion": true,
+ "completionTool": teamTaskCompletionTool,
+ "resultSink": map[string]interface{}{
+ "type": "redis_stream",
+ "eventsKey": teamEventsKey(teamID),
+ "successEvent": "task_completed",
+ "failureEvent": "task_failed",
+ "replyEvent": "reply",
+ "resultField": "resultMarkdown",
+ "summaryField": "summary",
+ "artifactField": "artifactRefs",
+ "completionTool": teamTaskCompletionTool,
+ },
+ "intent": eventString(taskPayload, "intent"),
+ "taskId": taskRef,
+ "title": eventString(taskPayload, "title"),
+ "prompt": appendTeamTaskCompletionInstruction(prompt),
+ "contextRefs": normalizeContextRefs(taskPayload["contextRefs"]),
+ "metadata": taskPayload,
+ "createdAt": now.Format(time.RFC3339Nano),
+ }
+ if envelope["intent"] == "" {
+ envelope["intent"] = "run_task"
+ }
+ if envelope["title"] == "" {
+ envelope["title"] = fmt.Sprintf("Team task %d", taskID)
+ }
+ return envelope
+}
+
+func appendTeamTaskCompletionInstruction(prompt string) string {
+ base := strings.TrimSpace(prompt)
+ if strings.Contains(base, teamTaskCompletionTool) && strings.Contains(base, "task_completed") {
+ return base
+ }
+ instruction := strings.Join([]string{
+ "Completion contract:",
+ "- When the final result is ready, call team_complete_task with status=\"succeeded\", summary, and resultMarkdown.",
+ "- If the task fails, call team_complete_task with status=\"failed\" and an error message.",
+ "- Do not send the final answer as a normal message to clawmanager; ClawManager consumes task_completed/task_failed events from the Team Redis event stream.",
+ }, "\n")
+ if base == "" {
+ return instruction
+ }
+ return base + "\n\n" + instruction
+}
+
+func (s *teamService) provisionTeamK8s(userID int, team *models.Team, redisURL string, sharedStorageGB int, storageClass string) (*teamRuntimeSecrets, error) {
ctx := context.Background()
pvc, err := s.pvcService.CreateTeamSharedPVC(ctx, userID, team.ID, sharedStorageGB, storageClass)
if err != nil {
- return err
+ return nil, err
}
secretName := s.pvcService.GetClient().GetTeamSecretName(team.ID)
teamToken, err := generatePrefixedToken("team")
if err != nil {
- return fmt.Errorf("failed to generate Team token: %w", err)
+ return nil, fmt.Errorf("failed to generate Team token: %w", err)
}
if err := s.secretService.UpsertSecret(ctx, userID, secretName, map[string]string{
teamRedisURLSecretKey: redisURL,
@@ -358,7 +456,7 @@ func (s *teamService) provisionTeamK8s(userID int, team *models.Team, redisURL s
"managed-by": "clawreef",
"team-id": strconv.Itoa(team.ID),
}); err != nil {
- return err
+ return nil, err
}
team.RedisURLSecretName = &secretName
@@ -368,21 +466,27 @@ func (s *teamService) provisionTeamK8s(userID int, team *models.Team, redisURL s
team.SharedPVCName = &pvc.Name
team.SharedPVCNamespace = &pvc.Namespace
team.UpdatedAt = time.Now().UTC()
- return s.repo.UpdateTeam(team)
+ if err := s.repo.UpdateTeam(team); err != nil {
+ return nil, err
+ }
+ return &teamRuntimeSecrets{RedisURL: redisURL, Token: teamToken}, nil
}
-func (s *teamService) upsertTeamRosterConfig(userID int, team *models.Team, members []plannedTeamMember) error {
+func (s *teamService) upsertTeamRosterConfig(userID int, team *models.Team, members []plannedTeamMember) (string, error) {
rosterJSON, err := buildTeamRosterConfig(team, members)
if err != nil {
- return err
+ return "", err
}
- return s.configMapService.UpsertConfigMap(context.Background(), userID, s.teamConfigMapName(team.ID), map[string]string{
+ if err := s.configMapService.UpsertConfigMap(context.Background(), userID, s.teamConfigMapName(team.ID), map[string]string{
teamConfigFileName: rosterJSON,
}, map[string]string{
"app": "clawreef",
"managed-by": "clawreef",
"team-id": strconv.Itoa(team.ID),
- })
+ }); err != nil {
+ return "", err
+ }
+ return rosterJSON, nil
}
func (s *teamService) teamConfigMapName(teamID int) string {
@@ -393,7 +497,7 @@ func (s *teamService) teamConfigMapName(teamID int) string {
return client.GetTeamConfigMapName(teamID)
}
-func (s *teamService) createTeamMemberInstance(userID int, team *models.Team, memberPlan plannedTeamMember) (*models.TeamMember, error) {
+func (s *teamService) createTeamMemberInstance(userID int, team *models.Team, memberPlan plannedTeamMember, runtimeSecrets *teamRuntimeSecrets, rosterJSON string) (*models.TeamMember, error) {
now := time.Now().UTC()
member := &models.TeamMember{
TeamID: team.ID,
@@ -402,6 +506,7 @@ func (s *teamService) createTeamMemberInstance(userID int, team *models.Team, me
DisplayName: memberPlan.DisplayName,
Role: memberPlan.Role,
RuntimeType: memberPlan.RuntimeType,
+ InstanceMode: memberPlan.InstanceMode,
Description: optionalString(strings.TrimSpace(derefTeamString(memberPlan.Request.Description))),
Status: models.TeamMemberStatusCreating,
Availability: models.TeamMemberAvailabilityUnknown,
@@ -412,7 +517,7 @@ func (s *teamService) createTeamMemberInstance(userID int, team *models.Team, me
return nil, err
}
- createReq := s.buildTeamMemberInstanceRequest(team, memberPlan)
+ createReq := s.buildTeamMemberInstanceRequestWithSecrets(team, memberPlan, runtimeSecrets, rosterJSON)
instance, err := s.instanceService.Create(userID, createReq)
if err != nil {
member.Status = models.TeamMemberStatusFailed
@@ -437,10 +542,36 @@ func (s *teamService) buildTeamMemberInstanceRequests(team *models.Team, memberP
}
func (s *teamService) buildTeamMemberInstanceRequest(team *models.Team, memberPlan plannedTeamMember) CreateInstanceRequest {
+ return s.buildTeamMemberInstanceRequestWithSecrets(team, memberPlan, nil, "")
+}
+
+func (s *teamService) buildTeamMemberInstanceRequestWithSecrets(team *models.Team, memberPlan plannedTeamMember, runtimeSecrets *teamRuntimeSecrets, rosterJSON string) CreateInstanceRequest {
req := memberPlan.Request
+ instanceMode := memberPlan.InstanceMode
+ if instanceMode == "" {
+ instanceMode = InstanceModeLite
+ }
+ runtimeBackendType, _ := RuntimeTypeForInstanceMode(instanceMode)
+ memberEnv := s.teamMemberEnv(team, memberPlan.MemberKey, memberPlan.Role)
+ if instanceMode == InstanceModeLite {
+ memberEnv["CLAWMANAGER_TEAM_SHARED_DIR"] = s.teamRuntimeSharedPath(team)
+ }
+ environmentOverrides := mergeEnvMaps(req.EnvironmentOverrides, memberEnv)
+ if instanceMode == InstanceModeLite && runtimeSecrets != nil {
+ environmentOverrides = mergeEnvMaps(environmentOverrides, map[string]string{
+ teamRedisURLSecretKey: runtimeSecrets.RedisURL,
+ teamTokenSecretKey: runtimeSecrets.Token,
+ })
+ if strings.TrimSpace(rosterJSON) != "" {
+ environmentOverrides["CLAWMANAGER_TEAM_CONFIG_JSON"] = rosterJSONWithSharedDir(rosterJSON, s.teamRuntimeSharedPath(team))
+ }
+ }
return CreateInstanceRequest{
Name: teamMemberInstanceName(team.Name, team.ID, memberPlan.MemberKey),
Type: memberPlan.RuntimeType,
+ Mode: instanceMode,
+ InstanceMode: instanceMode,
+ RuntimeType: runtimeBackendType,
CPUCores: defaultFloat(req.CPUCores, 2),
MemoryGB: defaultInt(req.MemoryGB, 4),
DiskGB: defaultInt(req.DiskGB, 20),
@@ -450,7 +581,7 @@ func (s *teamService) buildTeamMemberInstanceRequest(team *models.Team, memberPl
OSVersion: "latest",
ImageRegistry: req.ImageRegistry,
ImageTag: req.ImageTag,
- EnvironmentOverrides: req.EnvironmentOverrides,
+ EnvironmentOverrides: environmentOverrides,
StorageClass: derefTeamString(team.StorageClass),
OpenClawConfigPlan: req.OpenClawConfigPlan,
Team: &TeamInstanceConfig{
@@ -467,6 +598,13 @@ func (s *teamService) buildTeamMemberInstanceRequest(team *models.Team, memberPl
}
}
+func (s *teamService) teamRuntimeSharedPath(team *models.Team) string {
+ if team == nil {
+ return k8s.TeamSharedWorkspacePath(s.runtimeWorkspaceRoot, 0, 0)
+ }
+ return k8s.TeamSharedWorkspacePath(s.runtimeWorkspaceRoot, team.UserID, team.ID)
+}
+
func (s *teamService) teamMemberEnv(team *models.Team, memberKey, role string) map[string]string {
managerBaseURL, _ := defaultTeamManagerBaseURL()
return map[string]string{
@@ -651,30 +789,7 @@ func (s *teamService) DispatchTask(userID, teamID int, req DispatchTeamTaskReque
}
}
now := time.Now().UTC()
- envelope := map[string]interface{}{
- "v": 1,
- "messageId": messageID,
- "teamId": strconv.Itoa(teamID),
- "from": "clawmanager",
- "to": member.MemberKey,
- "intent": eventString(taskPayload, "intent"),
- "taskId": fmt.Sprintf("team-%d-task-%d", teamID, task.ID),
- "title": eventString(taskPayload, "title"),
- "prompt": eventString(taskPayload, "prompt", "goal", "instruction", "instructions"),
- "contextRefs": normalizeContextRefs(taskPayload["contextRefs"]),
- "metadata": taskPayload,
- "createdAt": now.Format(time.RFC3339Nano),
- }
- if envelope["intent"] == "" {
- envelope["intent"] = "run_task"
- }
- if envelope["title"] == "" {
- envelope["title"] = fmt.Sprintf("Team task %d", task.ID)
- }
- if envelope["prompt"] == "" {
- rawPayload, _ := marshalJSON(taskPayload)
- envelope["prompt"] = rawPayload
- }
+ envelope := buildTeamTaskEnvelope(teamID, member.MemberKey, task, messageID, taskPayload, now)
envelopeJSON, err := marshalJSON(envelope)
if err != nil {
return nil, fmt.Errorf("failed to encode task envelope: %w", err)
@@ -1090,6 +1205,7 @@ func (s *teamService) projectTeamEvent(team *models.Team, bus *redisBus, message
}
task = found
}
+ eventType = normalizeFinalReplyTaskEvent(eventType, payload, task)
payloadJSON, err := marshalOptionalJSON(payload)
if err != nil {
@@ -1184,6 +1300,35 @@ func (s *teamService) projectTeamEvent(team *models.Team, bus *redisBus, message
return nil
}
+func normalizeFinalReplyTaskEvent(eventType string, payload map[string]interface{}, task *models.TeamTask) string {
+ if task == nil || !strings.EqualFold(strings.TrimSpace(eventType), "reply") {
+ return eventType
+ }
+ if !eventBool(payload, "final", "isFinal", "complete", "completed", "taskCompleted") {
+ return eventType
+ }
+ if !teamEventHasBody(payload) {
+ return eventType
+ }
+ payload["originalEvent"] = eventType
+ payload["event"] = "task_completed"
+ payload["type"] = "task_completed"
+ payload["status"] = "succeeded"
+ payload["availability"] = models.TeamMemberAvailabilityIdle
+ payload["runtimeStatus"] = "succeeded"
+ if eventString(payload, "resultMarkdown") == "" {
+ if text := eventString(payload, "text", "result", "summary"); text != "" {
+ payload["resultMarkdown"] = text
+ }
+ }
+ if eventString(payload, "summary") == "" {
+ if text := eventString(payload, "text", "resultMarkdown", "result"); text != "" {
+ payload["summary"] = text
+ }
+ }
+ return "task_completed"
+}
+
func (s *teamService) enrichOutboundEventFromInbox(teamID int, bus *redisBus, payload map[string]interface{}, messageID string) (map[string]interface{}, error) {
targetMember := eventString(payload, "to", "recipient", "target", "targetMemberId", "target_member_id")
if targetMember == "" {
@@ -1429,6 +1574,36 @@ func eventString(payload map[string]interface{}, keys ...string) string {
return ""
}
+func eventBool(payload map[string]interface{}, keys ...string) bool {
+ for _, key := range keys {
+ value, ok := payload[key]
+ if !ok || value == nil {
+ continue
+ }
+ switch typed := value.(type) {
+ case bool:
+ return typed
+ case string:
+ switch strings.ToLower(strings.TrimSpace(typed)) {
+ case "1", "true", "yes", "y", "on":
+ return true
+ case "0", "false", "no", "n", "off":
+ return false
+ }
+ case float64:
+ return typed != 0
+ case int:
+ return typed != 0
+ default:
+ text := strings.ToLower(strings.TrimSpace(fmt.Sprintf("%v", typed)))
+ if text == "true" || text == "yes" || text == "1" {
+ return true
+ }
+ }
+ }
+ return false
+}
+
func applyTeamMemberRuntimeProjection(member *models.TeamMember, payload map[string]interface{}, eventType string) {
if member == nil {
return
@@ -1557,6 +1732,10 @@ func planTeamMembers(teamName string, members []CreateTeamMemberRequest) ([]plan
if err != nil {
return nil, err
}
+ instanceMode, err := normalizeTeamMemberInstanceMode(memberReq.Mode, memberReq.InstanceMode)
+ if err != nil {
+ return nil, err
+ }
isLeader := memberReq.IsLeader || isTeamLeaderRole(role)
if isLeader {
@@ -1568,12 +1747,13 @@ func planTeamMembers(teamName string, members []CreateTeamMemberRequest) ([]plan
displayName = fmt.Sprintf("%s-%s", teamName, memberKey)
}
plans = append(plans, plannedTeamMember{
- Request: memberReq,
- MemberKey: memberKey,
- DisplayName: displayName,
- Role: role,
- RuntimeType: runtimeType,
- IsLeader: isLeader,
+ Request: memberReq,
+ MemberKey: memberKey,
+ DisplayName: displayName,
+ Role: role,
+ RuntimeType: runtimeType,
+ InstanceMode: instanceMode,
+ IsLeader: isLeader,
})
}
if leaderCount != 1 {
@@ -1636,6 +1816,22 @@ func normalizeTeamMemberRuntimeType(raw string) (string, error) {
}
}
+func normalizeTeamMemberInstanceMode(rawMode, rawInstanceMode string) (string, error) {
+ if mode, ok := NormalizeInstanceMode(rawMode); ok {
+ return mode, nil
+ }
+ if strings.TrimSpace(rawMode) != "" {
+ return "", fmt.Errorf("unsupported team member instance mode: %s", rawMode)
+ }
+ if mode, ok := NormalizeInstanceMode(rawInstanceMode); ok {
+ return mode, nil
+ }
+ if strings.TrimSpace(rawInstanceMode) != "" {
+ return "", fmt.Errorf("unsupported team member instance mode: %s", rawInstanceMode)
+ }
+ return InstanceModeLite, nil
+}
+
func normalizeTeamMemberRole(raw string, isLeader bool) string {
role := strings.TrimSpace(raw)
if isLeader || isTeamLeaderRole(role) {
@@ -1682,12 +1878,13 @@ type teamRosterConfig struct {
}
type teamRosterMember struct {
- MemberID string `json:"memberId"`
- Role string `json:"role"`
- RuntimeType string `json:"runtimeType"`
- DisplayName string `json:"displayName"`
- Description string `json:"description,omitempty"`
- IsLeader bool `json:"isLeader"`
+ MemberID string `json:"memberId"`
+ Role string `json:"role"`
+ RuntimeType string `json:"runtimeType"`
+ InstanceMode string `json:"instanceMode"`
+ DisplayName string `json:"displayName"`
+ Description string `json:"description,omitempty"`
+ IsLeader bool `json:"isLeader"`
}
type teamRosterRedis struct {
@@ -1714,12 +1911,13 @@ func buildTeamRosterConfig(team *models.Team, members []plannedTeamMember) (stri
config.LeaderMemberID = member.MemberKey
}
config.Members = append(config.Members, teamRosterMember{
- MemberID: member.MemberKey,
- Role: member.Role,
- RuntimeType: member.RuntimeType,
- DisplayName: member.DisplayName,
- Description: derefTeamString(member.Request.Description),
- IsLeader: member.IsLeader,
+ MemberID: member.MemberKey,
+ Role: member.Role,
+ RuntimeType: member.RuntimeType,
+ InstanceMode: member.InstanceMode,
+ DisplayName: member.DisplayName,
+ Description: derefTeamString(member.Request.Description),
+ IsLeader: member.IsLeader,
})
}
if config.LeaderMemberID == "" {
@@ -1728,6 +1926,23 @@ func buildTeamRosterConfig(team *models.Team, members []plannedTeamMember) (stri
return marshalJSON(config)
}
+func rosterJSONWithSharedDir(rosterJSON, sharedDir string) string {
+ sharedDir = strings.TrimSpace(sharedDir)
+ if sharedDir == "" {
+ return rosterJSON
+ }
+ var payload map[string]interface{}
+ if err := json.Unmarshal([]byte(rosterJSON), &payload); err != nil {
+ return rosterJSON
+ }
+ payload["sharedDir"] = sharedDir
+ raw, err := json.Marshal(payload)
+ if err != nil {
+ return rosterJSON
+ }
+ return string(raw)
+}
+
func buildTeamRosterConfigFromMembers(team *models.Team, members []models.TeamMember) (string, error) {
config := teamRosterConfig{
Version: 1,
@@ -1747,16 +1962,21 @@ func buildTeamRosterConfigFromMembers(team *models.Team, members []models.TeamMe
if runtimeType == "" {
runtimeType = "openclaw"
}
+ instanceMode := strings.TrimSpace(member.InstanceMode)
+ if instanceMode == "" {
+ instanceMode = InstanceModeLite
+ }
if isLeader {
config.LeaderMemberID = member.MemberKey
}
config.Members = append(config.Members, teamRosterMember{
- MemberID: member.MemberKey,
- Role: member.Role,
- RuntimeType: runtimeType,
- DisplayName: member.DisplayName,
- Description: derefTeamString(member.Description),
- IsLeader: isLeader,
+ MemberID: member.MemberKey,
+ Role: member.Role,
+ RuntimeType: runtimeType,
+ InstanceMode: instanceMode,
+ DisplayName: member.DisplayName,
+ Description: derefTeamString(member.Description),
+ IsLeader: isLeader,
})
}
if config.LeaderMemberID == "" {
diff --git a/backend/internal/services/team_service_test.go b/backend/internal/services/team_service_test.go
index 2c27515..2d3374e 100644
--- a/backend/internal/services/team_service_test.go
+++ b/backend/internal/services/team_service_test.go
@@ -1,6 +1,7 @@
package services
import (
+ "encoding/json"
"strings"
"testing"
"time"
@@ -85,6 +86,81 @@ func TestBuildTeamMemberInstanceRequestUsesSharedPermissionDefaults(t *testing.T
}
}
+func TestBuildTeamMemberInstanceRequestSupportsLiteMode(t *testing.T) {
+ service := &teamService{runtimeWorkspaceRoot: "/workspaces"}
+ team := &models.Team{
+ UserID: 1,
+ ID: 8,
+ Name: "Lite Team",
+ SharedMountPath: "/team",
+ }
+ memberPlan := plannedTeamMember{
+ Request: CreateTeamMemberRequest{
+ Mode: "Lite",
+ InstanceMode: "Lite",
+ },
+ MemberKey: "lite-worker",
+ DisplayName: "lite worker",
+ Role: "developer",
+ RuntimeType: "openclaw",
+ InstanceMode: InstanceModeLite,
+ }
+ req := service.buildTeamMemberInstanceRequest(team, memberPlan)
+
+ if req.Mode != InstanceModeLite || req.InstanceMode != InstanceModeLite {
+ t.Fatalf("expected Team member instance request to preserve lite mode, got mode=%q instance_mode=%q", req.Mode, req.InstanceMode)
+ }
+ if req.RuntimeType != RuntimeBackendGateway {
+ t.Fatalf("expected lite Team member to target gateway runtime, got %q", req.RuntimeType)
+ }
+
+ rosterJSON := `{"teamId":"8","members":[{"memberId":"lite-worker"}]}`
+ liteReq := service.buildTeamMemberInstanceRequestWithSecrets(team, memberPlan, &teamRuntimeSecrets{
+ RedisURL: "redis://team-redis:6379/0",
+ Token: "team_test_token",
+ }, rosterJSON)
+ if liteReq.EnvironmentOverrides[teamRedisURLSecretKey] != "redis://team-redis:6379/0" ||
+ liteReq.EnvironmentOverrides[teamTokenSecretKey] != "team_test_token" {
+ t.Fatalf("expected Lite Team runtime secrets in gateway env overrides, got %#v", liteReq.EnvironmentOverrides)
+ }
+ if !strings.Contains(liteReq.EnvironmentOverrides["CLAWMANAGER_TEAM_CONFIG_JSON"], `"sharedDir":"/workspaces/teams/user-1/team-8-shared"`) {
+ t.Fatalf("expected Lite Team roster JSON to include runtime shared dir, got %#v", liteReq.EnvironmentOverrides)
+ }
+}
+
+func TestBuildTeamMemberInstanceRequestPointsLiteSharedDirAtRuntimeWorkspace(t *testing.T) {
+ service := &teamService{runtimeWorkspaceRoot: "/workspaces"}
+ team := &models.Team{
+ UserID: 1,
+ ID: 28,
+ Name: "Mixed Team",
+ SharedMountPath: "/team",
+ }
+ memberPlan := plannedTeamMember{
+ MemberKey: "backend",
+ DisplayName: "backend",
+ Role: "developer",
+ RuntimeType: "openclaw",
+ InstanceMode: InstanceModeLite,
+ }
+
+ req := service.buildTeamMemberInstanceRequestWithSecrets(team, memberPlan, &teamRuntimeSecrets{
+ RedisURL: "redis://team-redis:6379/0",
+ Token: "team_test_token",
+ }, `{"sharedDir":"/team"}`)
+
+ wantSharedDir := "/workspaces/teams/user-1/team-28-shared"
+ if req.EnvironmentOverrides["CLAWMANAGER_TEAM_SHARED_DIR"] != wantSharedDir {
+ t.Fatalf("expected Lite shared dir %q, got %#v", wantSharedDir, req.EnvironmentOverrides)
+ }
+ if !strings.Contains(req.EnvironmentOverrides["CLAWMANAGER_TEAM_CONFIG_JSON"], `"sharedDir":"`+wantSharedDir+`"`) {
+ t.Fatalf("expected Lite roster JSON to use runtime shared dir, got %s", req.EnvironmentOverrides["CLAWMANAGER_TEAM_CONFIG_JSON"])
+ }
+ if req.Team.SharedMountPath != "/team" {
+ t.Fatalf("Pro Team mount path should remain /team, got %q", req.Team.SharedMountPath)
+ }
+}
+
func TestNewRedisBusParsesURLWithoutNetwork(t *testing.T) {
bus, err := newRedisBus("redis://:pass@redis.example:6380/3")
if err != nil {
@@ -141,7 +217,7 @@ func TestPlanTeamMembersRequiresExactlyOneLeader(t *testing.T) {
func TestPlanTeamMembersSupportsHermesRuntime(t *testing.T) {
plans, err := planTeamMembers("team", []CreateTeamMemberRequest{
{MemberID: "lead", Role: "leader"},
- {MemberID: "hermes-writer", Role: "writer", RuntimeType: "Hermes"},
+ {MemberID: "hermes-writer", Role: "writer", RuntimeType: "Hermes", InstanceMode: "Pro"},
})
if err != nil {
t.Fatalf("planTeamMembers returned error: %v", err)
@@ -149,6 +225,9 @@ func TestPlanTeamMembersSupportsHermesRuntime(t *testing.T) {
if plans[1].RuntimeType != "hermes" {
t.Fatalf("expected Hermes runtime to be normalized, got %#v", plans[1])
}
+ if plans[1].InstanceMode != InstanceModePro {
+ t.Fatalf("expected Pro instance mode to be normalized, got %#v", plans[1])
+ }
_, err = planTeamMembers("team", []CreateTeamMemberRequest{
{MemberID: "lead", Role: "leader"},
@@ -157,6 +236,14 @@ func TestPlanTeamMembersSupportsHermesRuntime(t *testing.T) {
if err == nil || !strings.Contains(err.Error(), "unsupported team member runtime type") {
t.Fatalf("expected unsupported runtime validation error, got %v", err)
}
+
+ _, err = planTeamMembers("team", []CreateTeamMemberRequest{
+ {MemberID: "lead", Role: "leader"},
+ {MemberID: "worker", Role: "developer", Mode: "mini"},
+ })
+ if err == nil || !strings.Contains(err.Error(), "unsupported team member instance mode") {
+ t.Fatalf("expected unsupported instance mode validation error, got %v", err)
+ }
}
func TestTeamMemberInstanceNameUsesTeamIDAndMemberKey(t *testing.T) {
@@ -205,6 +292,9 @@ func TestBuildTeamRosterConfigOmitsSecrets(t *testing.T) {
if !strings.Contains(roster, `"runtimeType":"openclaw"`) {
t.Fatalf("roster missing member runtime type: %s", roster)
}
+ if !strings.Contains(roster, `"instanceMode":"lite"`) {
+ t.Fatalf("roster missing member instance mode: %s", roster)
+ }
}
func TestBuildInitialLeaderTaskPayloadDescribesRosterAndTeamSend(t *testing.T) {
@@ -236,6 +326,107 @@ func TestBuildInitialLeaderTaskPayloadDescribesRosterAndTeamSend(t *testing.T) {
}
}
+func TestBuildTeamTaskEnvelopeIncludesCompletionContract(t *testing.T) {
+ task := &models.TeamTask{ID: 67}
+ payload := map[string]interface{}{
+ "intent": "team_bootstrap_introduction",
+ "title": "Introduce the team",
+ "prompt": "Generate the team report.",
+ }
+
+ envelope := buildTeamTaskEnvelope(31, "leader", task, "team-31-bootstrap-introduction", payload, time.Unix(123, 0).UTC())
+
+ if envelope["replyTo"] != "clawmanager" {
+ t.Fatalf("expected replyTo clawmanager, got %#v", envelope["replyTo"])
+ }
+ if envelope["requiresCompletion"] != true {
+ t.Fatalf("expected requiresCompletion=true, got %#v", envelope["requiresCompletion"])
+ }
+ if envelope["completionTool"] != "team_complete_task" {
+ t.Fatalf("expected completion tool team_complete_task, got %#v", envelope["completionTool"])
+ }
+ resultSink, ok := envelope["resultSink"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("expected resultSink map, got %#v", envelope["resultSink"])
+ }
+ if resultSink["type"] != "redis_stream" || resultSink["eventsKey"] != "claw:team:31:events" {
+ t.Fatalf("unexpected resultSink: %#v", resultSink)
+ }
+ if resultSink["successEvent"] != "task_completed" || resultSink["failureEvent"] != "task_failed" {
+ t.Fatalf("unexpected resultSink events: %#v", resultSink)
+ }
+ prompt, ok := envelope["prompt"].(string)
+ if !ok {
+ t.Fatalf("expected prompt string, got %#v", envelope["prompt"])
+ }
+ for _, expected := range []string{"Generate the team report.", "team_complete_task", "resultMarkdown", "task_completed"} {
+ if !strings.Contains(prompt, expected) {
+ t.Fatalf("completion prompt missing %q: %s", expected, prompt)
+ }
+ }
+}
+
+func TestProjectTeamEventTreatsFinalReplyAsTaskCompleted(t *testing.T) {
+ taskID := 67
+ messageID := "team-31-bootstrap-introduction"
+ task := &models.TeamTask{
+ ID: taskID,
+ TeamID: 31,
+ TargetMemberID: 120,
+ MessageID: messageID,
+ Status: models.TeamTaskStatusRunning,
+ UpdatedAt: time.Now().UTC(),
+ }
+ member := &models.TeamMember{
+ ID: 120,
+ TeamID: 31,
+ MemberKey: "leader",
+ Status: models.TeamMemberStatusBusy,
+ CurrentTaskID: &taskID,
+ Availability: models.TeamMemberAvailabilityBusy,
+ }
+ repo := &teamRepositoryStub{
+ tasksByMessageID: map[string]*models.TeamTask{messageID: task},
+ membersByKey: map[string]*models.TeamMember{"leader": member},
+ }
+ service := &teamService{repo: repo}
+ payload := map[string]interface{}{
+ "event": "reply",
+ "messageId": messageID,
+ "memberId": "leader",
+ "taskId": "team-31-task-67",
+ "final": true,
+ "summary": "Team report ready",
+ "resultMarkdown": "Full report",
+ "text": "Full report",
+ }
+ payloadJSON, err := json.Marshal(payload)
+ if err != nil {
+ t.Fatalf("marshal payload: %v", err)
+ }
+
+ err = service.projectTeamEvent(&models.Team{ID: 31}, nil, redisStreamMessage{
+ ID: "1781171178655-0",
+ Fields: map[string]string{"payload": string(payloadJSON)},
+ })
+ if err != nil {
+ t.Fatalf("projectTeamEvent returned error: %v", err)
+ }
+
+ if repo.updatedTask == nil || repo.updatedTask.Status != models.TeamTaskStatusSucceeded {
+ t.Fatalf("expected final reply to mark task succeeded, got %#v", repo.updatedTask)
+ }
+ if repo.updatedTask.ResultJSON == nil || !strings.Contains(*repo.updatedTask.ResultJSON, "Full report") {
+ t.Fatalf("expected reply payload to become task result, got %#v", repo.updatedTask.ResultJSON)
+ }
+ if repo.updatedMember == nil || repo.updatedMember.Status != models.TeamMemberStatusIdle || repo.updatedMember.Availability != models.TeamMemberAvailabilityIdle {
+ t.Fatalf("expected member to become idle, got %#v", repo.updatedMember)
+ }
+ if len(repo.createdEvents) != 1 || repo.createdEvents[0].EventType != "task_completed" {
+ t.Fatalf("expected stored event type task_completed, got %#v", repo.createdEvents)
+ }
+}
+
func TestActiveTeamMembersFiltersDeletedMembers(t *testing.T) {
members := activeTeamMembers([]models.TeamMember{
{MemberKey: "leader", Status: models.TeamMemberStatusIdle},
@@ -343,3 +534,111 @@ func TestMergeMissingEventFieldsEnrichesOutboundPayload(t *testing.T) {
t.Fatalf("expected enriched event to have displayable body: %#v", merged)
}
}
+
+type teamRepositoryStub struct {
+ teamsByID map[int]*models.Team
+ membersByID map[int]*models.TeamMember
+ membersByKey map[string]*models.TeamMember
+ tasksByID map[int]*models.TeamTask
+ tasksByMessageID map[string]*models.TeamTask
+ createdEvents []models.TeamEvent
+ updatedTask *models.TeamTask
+ updatedMember *models.TeamMember
+ updatedTeam *models.Team
+}
+
+func (s *teamRepositoryStub) CreateTeam(team *models.Team) error { return nil }
+func (s *teamRepositoryStub) UpdateTeam(team *models.Team) error {
+ clone := *team
+ s.updatedTeam = &clone
+ return nil
+}
+func (s *teamRepositoryStub) GetTeamByID(id int) (*models.Team, error) {
+ if s.teamsByID != nil {
+ return s.teamsByID[id], nil
+ }
+ return nil, nil
+}
+func (s *teamRepositoryStub) GetTeamByUserIDAndName(userID int, name string) (*models.Team, error) {
+ return nil, nil
+}
+func (s *teamRepositoryStub) ExistsByUserIDAndName(userID int, name string) (bool, error) {
+ return false, nil
+}
+func (s *teamRepositoryStub) ListTeamsByUserID(userID int, offset, limit int) ([]models.Team, error) {
+ return nil, nil
+}
+func (s *teamRepositoryStub) ListActiveTeams() ([]models.Team, error) { return nil, nil }
+func (s *teamRepositoryStub) CountTeamsByUserID(userID int) (int, error) {
+ return 0, nil
+}
+func (s *teamRepositoryStub) CreateMember(member *models.TeamMember) error { return nil }
+func (s *teamRepositoryStub) UpdateMember(member *models.TeamMember) error {
+ clone := *member
+ s.updatedMember = &clone
+ return nil
+}
+func (s *teamRepositoryStub) GetMemberByID(id int) (*models.TeamMember, error) {
+ if s.membersByID != nil {
+ return s.membersByID[id], nil
+ }
+ return nil, nil
+}
+func (s *teamRepositoryStub) GetMemberByTeamKey(teamID int, memberKey string) (*models.TeamMember, error) {
+ if s.membersByKey == nil {
+ return nil, nil
+ }
+ member := s.membersByKey[memberKey]
+ if member == nil || member.TeamID != teamID {
+ return nil, nil
+ }
+ return member, nil
+}
+func (s *teamRepositoryStub) ListMembersByTeamID(teamID int) ([]models.TeamMember, error) {
+ return nil, nil
+}
+func (s *teamRepositoryStub) CreateTask(task *models.TeamTask) error { return nil }
+func (s *teamRepositoryStub) UpdateTask(task *models.TeamTask) error {
+ clone := *task
+ s.updatedTask = &clone
+ return nil
+}
+func (s *teamRepositoryStub) GetTaskByID(id int) (*models.TeamTask, error) {
+ if s.tasksByID != nil {
+ return s.tasksByID[id], nil
+ }
+ return nil, nil
+}
+func (s *teamRepositoryStub) GetTaskByMessageID(teamID int, messageID string) (*models.TeamTask, error) {
+ if s.tasksByMessageID == nil {
+ return nil, nil
+ }
+ task := s.tasksByMessageID[messageID]
+ if task == nil || task.TeamID != teamID {
+ return nil, nil
+ }
+ return task, nil
+}
+func (s *teamRepositoryStub) ListTasksByTeamID(teamID int, limit int) ([]models.TeamTask, error) {
+ return nil, nil
+}
+func (s *teamRepositoryStub) ListTasksBeforeID(teamID, beforeID, limit int) ([]models.TeamTask, error) {
+ return nil, nil
+}
+func (s *teamRepositoryStub) ListStaleCandidateTasks(cutoff time.Time, limit int) ([]models.TeamTask, error) {
+ return nil, nil
+}
+func (s *teamRepositoryStub) CreateEvent(event *models.TeamEvent) error {
+ clone := *event
+ s.createdEvents = append(s.createdEvents, clone)
+ return nil
+}
+func (s *teamRepositoryStub) EventExistsByStreamID(teamID int, streamID string) (bool, error) {
+ return false, nil
+}
+func (s *teamRepositoryStub) ListEventsByTeamID(teamID int, limit int) ([]models.TeamEvent, error) {
+ return nil, nil
+}
+func (s *teamRepositoryStub) ListEventsBeforeID(teamID, beforeID, limit int) ([]models.TeamEvent, error) {
+ return nil, nil
+}
diff --git a/backend/internal/services/websocket_service.go b/backend/internal/services/websocket_service.go
index 255d464..5f13491 100644
--- a/backend/internal/services/websocket_service.go
+++ b/backend/internal/services/websocket_service.go
@@ -1,9 +1,11 @@
package services
import (
+ "context"
"encoding/json"
"log"
"net/http"
+ "strings"
"sync"
"time"
@@ -24,11 +26,20 @@ var upgrader = websocket.Upgrader{
type Client struct {
ID int
UserID int
+ Role string
+ Topic WebSocketTopic
Conn *websocket.Conn
Send chan []byte
hub *Hub
}
+type WebSocketTopic string
+
+const (
+ WebSocketTopicUser WebSocketTopic = "user"
+ WebSocketTopicRuntimeAdmin WebSocketTopic = "runtime_admin"
+)
+
// Hub maintains the set of active clients and broadcasts messages
type Hub struct {
clients map[*Client]bool
@@ -181,16 +192,59 @@ func (h *Hub) BroadcastToAll(msgType string, data interface{}) {
h.broadcast <- msg
}
+func (h *Hub) BroadcastRuntimeAdmin(msgType string, data interface{}) {
+ msg := &Message{
+ Type: msgType,
+ Data: data,
+ Timestamp: time.Now(),
+ }
+ h.broadcastRuntimeAdminMessage(msg)
+}
+
+func (h *Hub) broadcastRuntimeAdminMessage(msg *Message) {
+ h.mu.RLock()
+ clients := make([]*Client, 0, len(h.clients))
+ for client := range h.clients {
+ if client.Role == "admin" && client.Topic == WebSocketTopicRuntimeAdmin {
+ clients = append(clients, client)
+ }
+ }
+ h.mu.RUnlock()
+
+ encoded := mustEncode(msg)
+ for _, client := range clients {
+ select {
+ case client.Send <- encoded:
+ default:
+ h.mu.Lock()
+ if _, ok := h.clients[client]; ok {
+ close(client.Send)
+ delete(h.clients, client)
+ }
+ h.mu.Unlock()
+ }
+ }
+}
+
// ServeWS handles WebSocket connections
func ServeWS(hub *Hub, w http.ResponseWriter, r *http.Request, userID int) {
+ ServeWSWithOptions(hub, w, r, userID, "", WebSocketTopicUser)
+}
+
+func ServeWSWithOptions(hub *Hub, w http.ResponseWriter, r *http.Request, userID int, role string, topic WebSocketTopic) {
conn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
log.Printf("Failed to upgrade WebSocket: %v", err)
return
}
+ if topic == "" {
+ topic = WebSocketTopicUser
+ }
client := &Client{
UserID: userID,
+ Role: strings.TrimSpace(role),
+ Topic: topic,
Conn: conn,
Send: make(chan []byte, 256),
hub: hub,
@@ -203,6 +257,92 @@ func ServeWS(hub *Hub, w http.ResponseWriter, r *http.Request, userID int) {
go client.readPump()
}
+func StartRuntimeAdminEventBridge(ctx context.Context, events RuntimeEventService, hub *Hub) {
+ if events == nil || hub == nil {
+ return
+ }
+ go func() {
+ lastID := "$"
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ default:
+ }
+
+ messages, err := events.Read(ctx, lastID, 5*time.Second)
+ if err != nil {
+ if ctx.Err() != nil {
+ return
+ }
+ log.Printf("runtime admin event bridge read failed: %v", err)
+ sleepOrDone(ctx, time.Second)
+ continue
+ }
+ if len(messages) == 0 {
+ sleepOrDone(ctx, 200*time.Millisecond)
+ continue
+ }
+ for _, message := range messages {
+ if strings.TrimSpace(message.ID) != "" {
+ lastID = message.ID
+ }
+ if msg, ok := runtimeEventWebSocketMessage(message); ok {
+ hub.broadcastRuntimeAdminMessage(msg)
+ }
+ }
+ }
+ }()
+}
+
+func runtimeEventWebSocketMessage(message redisStreamMessage) (*Message, bool) {
+ eventType := strings.TrimSpace(message.Fields["type"])
+ if eventType == "" {
+ var event RuntimeEvent
+ if raw := strings.TrimSpace(message.Fields["event"]); raw != "" && json.Unmarshal([]byte(raw), &event) == nil {
+ eventType = strings.TrimSpace(event.Type)
+ if eventType == "" {
+ return nil, false
+ }
+ return &Message{
+ Type: eventType,
+ Data: json.RawMessage(event.Payload),
+ Timestamp: event.CreatedAt,
+ }, true
+ }
+ return nil, false
+ }
+
+ createdAt := time.Now().UTC()
+ if rawCreatedAt := strings.TrimSpace(message.Fields["created_at"]); rawCreatedAt != "" {
+ if parsed, err := time.Parse(time.RFC3339Nano, rawCreatedAt); err == nil {
+ createdAt = parsed
+ }
+ }
+ var data interface{} = map[string]any{}
+ if rawPayload := strings.TrimSpace(message.Fields["payload"]); rawPayload != "" {
+ if json.Valid([]byte(rawPayload)) {
+ data = json.RawMessage(rawPayload)
+ } else {
+ data = rawPayload
+ }
+ }
+ return &Message{
+ Type: eventType,
+ Data: data,
+ Timestamp: createdAt,
+ }, true
+}
+
+func sleepOrDone(ctx context.Context, d time.Duration) {
+ timer := time.NewTimer(d)
+ defer timer.Stop()
+ select {
+ case <-ctx.Done():
+ case <-timer.C:
+ }
+}
+
// readPump pumps messages from the websocket connection to the hub
func (c *Client) readPump() {
defer func() {
@@ -278,7 +418,6 @@ func (h *Hub) GetClientCount() int {
return len(h.clients)
}
-
// Stop gracefully shuts down the hub, closing all client connections.
func (h *Hub) Stop() {
close(h.stop)
diff --git a/backend/internal/services/websocket_service_test.go b/backend/internal/services/websocket_service_test.go
index d19c018..06b088c 100644
--- a/backend/internal/services/websocket_service_test.go
+++ b/backend/internal/services/websocket_service_test.go
@@ -1,6 +1,7 @@
package services
import (
+ "encoding/json"
"sync"
"testing"
"time"
@@ -86,3 +87,57 @@ func TestHubStopClosesClients(t *testing.T) {
}
}
+func TestHubBroadcastRuntimeAdminFiltersToAdminRuntimeClients(t *testing.T) {
+ h := NewHub()
+ adminRuntime := &Client{
+ UserID: 1,
+ Role: "admin",
+ Topic: WebSocketTopicRuntimeAdmin,
+ Send: make(chan []byte, 1),
+ hub: h,
+ }
+ adminUserTopic := &Client{
+ UserID: 2,
+ Role: "admin",
+ Topic: WebSocketTopicUser,
+ Send: make(chan []byte, 1),
+ hub: h,
+ }
+ normalRuntime := &Client{
+ UserID: 3,
+ Role: "user",
+ Topic: WebSocketTopicRuntimeAdmin,
+ Send: make(chan []byte, 1),
+ hub: h,
+ }
+ h.clients[adminRuntime] = true
+ h.clients[adminUserTopic] = true
+ h.clients[normalRuntime] = true
+
+ h.BroadcastRuntimeAdmin("runtime_pod_metrics", map[string]any{"pod_id": int64(9)})
+
+ select {
+ case raw := <-adminRuntime.Send:
+ var msg Message
+ if err := json.Unmarshal(raw, &msg); err != nil {
+ t.Fatalf("runtime admin message is not json: %v", err)
+ }
+ if msg.Type != "runtime_pod_metrics" {
+ t.Fatalf("message type = %q, want runtime_pod_metrics", msg.Type)
+ }
+ default:
+ t.Fatalf("admin runtime client did not receive runtime admin message")
+ }
+
+ select {
+ case raw := <-adminUserTopic.Send:
+ t.Fatalf("admin user-topic client received runtime admin message: %s", string(raw))
+ default:
+ }
+
+ select {
+ case raw := <-normalRuntime.Send:
+ t.Fatalf("normal runtime client received runtime admin message: %s", string(raw))
+ default:
+ }
+}
diff --git a/backend/internal/services/workspace_file_service.go b/backend/internal/services/workspace_file_service.go
new file mode 100644
index 0000000..cea67c9
--- /dev/null
+++ b/backend/internal/services/workspace_file_service.go
@@ -0,0 +1,645 @@
+package services
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "mime"
+ "net/url"
+ "os"
+ "path"
+ "path/filepath"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+ "unicode"
+
+ "clawreef/internal/models"
+ "clawreef/internal/repository"
+)
+
+const (
+ WorkspaceUploadMaxBytesEnv = "WORKSPACE_UPLOAD_MAX_BYTES"
+ defaultWorkspaceUploadMax = int64(524288000)
+ textPreviewMaxBytes = int64(1024 * 1024)
+ imagePreviewMaxBytes = int64(10 * 1024 * 1024)
+)
+
+var (
+ ErrWorkspaceDirectoryExpected = errors.New("workspace directory is required")
+ ErrWorkspaceFileExpected = errors.New("workspace file is required")
+ ErrWorkspacePreviewTooLarge = errors.New("workspace preview exceeds maximum size")
+ ErrWorkspaceUploadTooLarge = errors.New("workspace upload exceeds maximum size")
+ ErrWorkspaceRootOperation = errors.New("workspace root cannot be modified")
+ ErrWorkspaceFileNameInvalid = errors.New("workspace filename is invalid")
+ ErrWorkspaceEntryExists = errors.New("workspace entry already exists")
+)
+
+type WorkspaceFileScope struct {
+ InstanceID int
+ UserID int
+ WorkspacePath string
+}
+
+type WorkspaceEntry struct {
+ Name string `json:"name"`
+ Path string `json:"path"`
+ IsDir bool `json:"is_dir"`
+ Size int64 `json:"size"`
+ ModifiedAt time.Time `json:"modified_at"`
+ Previewable bool `json:"previewable"`
+ Downloadable bool `json:"downloadable"`
+}
+
+type WorkspacePreview struct {
+ Kind string `json:"kind"`
+ ContentType string `json:"content_type"`
+ Text string `json:"text,omitempty"`
+ PreviewURL string `json:"preview_url,omitempty"`
+ DownloadURL string `json:"download_url,omitempty"`
+}
+
+type WorkspaceFileService interface {
+ List(ctx context.Context, scope WorkspaceFileScope, relativePath string) ([]WorkspaceEntry, error)
+ Preview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspacePreview, error)
+ OpenPreview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error)
+ OpenDownload(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error)
+ Upload(ctx context.Context, scope WorkspaceFileScope, relativeDir string, filename string, reader io.Reader, size int64) (*WorkspaceEntry, error)
+ Mkdir(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspaceEntry, error)
+ Rename(ctx context.Context, scope WorkspaceFileScope, oldPath, newPath string) (*WorkspaceEntry, error)
+ Delete(ctx context.Context, scope WorkspaceFileScope, relativePath string) error
+}
+
+type workspaceFileService struct {
+ auditRepo repository.WorkspaceFileAuditRepository
+}
+
+func NewWorkspaceFileService(auditRepo repository.WorkspaceFileAuditRepository) WorkspaceFileService {
+ return &workspaceFileService{auditRepo: auditRepo}
+}
+
+func WorkspaceUploadMaxBytes() int64 {
+ raw := strings.TrimSpace(os.Getenv(WorkspaceUploadMaxBytesEnv))
+ if raw == "" {
+ return defaultWorkspaceUploadMax
+ }
+ value, err := strconv.ParseInt(raw, 10, 64)
+ if err != nil || value <= 0 {
+ return defaultWorkspaceUploadMax
+ }
+ return value
+}
+
+func (s *workspaceFileService) List(ctx context.Context, scope WorkspaceFileScope, relativePath string) ([]WorkspaceEntry, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ resolved, err := ResolveWorkspacePath(scope.WorkspacePath, relativePath, false)
+ if err != nil {
+ return nil, err
+ }
+ root, err := openWorkspaceRoot(scope.WorkspacePath)
+ if err != nil {
+ return nil, err
+ }
+ defer root.Close()
+
+ dir, err := root.Open(workspaceRootName(resolved.RelativePath))
+ if err != nil {
+ return nil, err
+ }
+ defer dir.Close()
+ info, err := dir.Stat()
+ if err != nil {
+ return nil, err
+ }
+ if !info.IsDir() {
+ return nil, ErrWorkspaceDirectoryExpected
+ }
+
+ items, err := dir.ReadDir(-1)
+ if err != nil {
+ return nil, err
+ }
+ entries := make([]WorkspaceEntry, 0, len(items))
+ for _, item := range items {
+ itemInfo, infoErr := item.Info()
+ if infoErr != nil {
+ return nil, infoErr
+ }
+ childRelative := joinWorkspaceRelative(resolved.RelativePath, item.Name())
+ entries = append(entries, buildWorkspaceEntry(childRelative, itemInfo))
+ }
+ sort.Slice(entries, func(i, j int) bool {
+ if entries[i].IsDir != entries[j].IsDir {
+ return entries[i].IsDir
+ }
+ left := strings.ToLower(entries[i].Name)
+ right := strings.ToLower(entries[j].Name)
+ if left == right {
+ return entries[i].Name < entries[j].Name
+ }
+ return left < right
+ })
+ return entries, nil
+}
+
+func (s *workspaceFileService) Preview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspacePreview, error) {
+ resolved, file, info, err := s.openExistingFile(ctx, scope, relativePath)
+ if err != nil {
+ return nil, err
+ }
+ defer file.Close()
+
+ kind, contentType, previewable, maxBytes := workspacePreviewKind(info.Name(), info.Size())
+ downloadURL := workspaceDownloadURL(scope.InstanceID, resolved.RelativePath)
+ if !previewable {
+ return &WorkspacePreview{
+ Kind: "binary",
+ ContentType: "application/octet-stream",
+ DownloadURL: downloadURL,
+ }, nil
+ }
+ if maxBytes > 0 && info.Size() > maxBytes {
+ return nil, ErrWorkspacePreviewTooLarge
+ }
+ if kind == "text" {
+ data, err := io.ReadAll(io.LimitReader(file, textPreviewMaxBytes+1))
+ if err != nil {
+ return nil, err
+ }
+ if int64(len(data)) > textPreviewMaxBytes {
+ return nil, ErrWorkspacePreviewTooLarge
+ }
+ return &WorkspacePreview{
+ Kind: "text",
+ ContentType: contentType,
+ Text: string(data),
+ DownloadURL: downloadURL,
+ }, nil
+ }
+
+ return &WorkspacePreview{
+ Kind: kind,
+ ContentType: contentType,
+ PreviewURL: workspacePreviewURL(scope.InstanceID, resolved.RelativePath),
+ DownloadURL: downloadURL,
+ }, nil
+}
+
+func (s *workspaceFileService) OpenPreview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) {
+ file, info, contentType, err := s.openPreviewableFile(ctx, scope, relativePath)
+ if err != nil {
+ return nil, "", 0, err
+ }
+ return file, contentType, info.Size(), nil
+}
+
+func (s *workspaceFileService) openPreviewableFile(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, os.FileInfo, string, error) {
+ _, file, info, err := s.openExistingFile(ctx, scope, relativePath)
+ if err != nil {
+ return nil, nil, "", err
+ }
+ _, contentType, previewable, maxBytes := workspacePreviewKind(info.Name(), info.Size())
+ if !previewable {
+ file.Close()
+ return nil, nil, "", ErrWorkspaceFileExpected
+ }
+ if maxBytes > 0 && info.Size() > maxBytes {
+ file.Close()
+ return nil, nil, "", ErrWorkspacePreviewTooLarge
+ }
+ return file, info, contentType, nil
+}
+
+func (s *workspaceFileService) OpenDownload(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error) {
+ resolved, file, info, err := s.openExistingFile(ctx, scope, relativePath)
+ if err != nil {
+ return nil, "", 0, err
+ }
+ if err := s.recordAudit(ctx, scope, "download", resolved.RelativePath, info.Size()); err != nil {
+ file.Close()
+ return nil, "", 0, err
+ }
+ return file, info.Name(), info.Size(), nil
+}
+
+func (s *workspaceFileService) Upload(ctx context.Context, scope WorkspaceFileScope, relativeDir string, filename string, reader io.Reader, size int64) (*WorkspaceEntry, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ maxBytes := WorkspaceUploadMaxBytes()
+ if size > maxBytes {
+ return nil, ErrWorkspaceUploadTooLarge
+ }
+ cleanName, err := sanitizeWorkspaceFilename(filename)
+ if err != nil {
+ return nil, err
+ }
+ dir, err := ResolveWorkspacePath(scope.WorkspacePath, relativeDir, false)
+ if err != nil {
+ return nil, err
+ }
+ root, err := openWorkspaceRoot(scope.WorkspacePath)
+ if err != nil {
+ return nil, err
+ }
+ defer root.Close()
+ dirInfo, err := root.Stat(workspaceRootName(dir.RelativePath))
+ if err != nil {
+ return nil, err
+ }
+ if !dirInfo.IsDir() {
+ return nil, ErrWorkspaceDirectoryExpected
+ }
+
+ targetRelative := joinWorkspaceRelative(dir.RelativePath, cleanName)
+ target, err := ResolveWorkspacePath(scope.WorkspacePath, targetRelative, true)
+ if err != nil {
+ return nil, err
+ }
+ if err := ensureWorkspaceRuntimeOwnership(scope, dir.RelativePath); err != nil {
+ return nil, err
+ }
+ targetName := workspaceRootName(target.RelativePath)
+ if targetInfo, statErr := root.Stat(targetName); statErr == nil && targetInfo.IsDir() {
+ return nil, ErrWorkspaceFileExpected
+ } else if statErr != nil && !os.IsNotExist(statErr) {
+ return nil, statErr
+ }
+
+ tempRelative := joinWorkspaceRelative(dir.RelativePath, fmt.Sprintf(".%s.tmp-%d", cleanName, time.Now().UnixNano()))
+ tempName := workspaceRootName(tempRelative)
+ file, err := root.OpenFile(tempName, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0640)
+ if err != nil {
+ return nil, err
+ }
+ written, copyErr := io.Copy(file, io.LimitReader(reader, maxBytes+1))
+ closeErr := file.Close()
+ if copyErr != nil {
+ _ = root.Remove(tempName)
+ return nil, copyErr
+ }
+ if closeErr != nil {
+ _ = root.Remove(tempName)
+ return nil, closeErr
+ }
+ if written > maxBytes {
+ _ = root.Remove(tempName)
+ return nil, ErrWorkspaceUploadTooLarge
+ }
+ if err := applyWorkspaceRuntimeOwnership(scope, tempRelative, 0640); err != nil {
+ _ = root.Remove(tempName)
+ return nil, err
+ }
+ if err := s.recordAudit(ctx, scope, "upload", target.RelativePath, written); err != nil {
+ _ = root.Remove(tempName)
+ return nil, err
+ }
+ if err := root.Rename(tempName, targetName); err != nil {
+ _ = root.Remove(tempName)
+ return nil, err
+ }
+ info, err := root.Stat(targetName)
+ if err != nil {
+ return nil, err
+ }
+ entry := buildWorkspaceEntry(target.RelativePath, info)
+ return &entry, nil
+}
+
+func (s *workspaceFileService) Mkdir(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspaceEntry, error) {
+ if isWorkspaceRootRelative(relativePath) {
+ return nil, ErrWorkspaceRootOperation
+ }
+ resolved, err := ResolveWorkspacePath(scope.WorkspacePath, relativePath, true)
+ if err != nil {
+ return nil, err
+ }
+ root, err := openWorkspaceRoot(scope.WorkspacePath)
+ if err != nil {
+ return nil, err
+ }
+ defer root.Close()
+ resolvedName := workspaceRootName(resolved.RelativePath)
+ if _, err := root.Lstat(resolvedName); err == nil {
+ return nil, ErrWorkspaceEntryExists
+ } else if !os.IsNotExist(err) {
+ return nil, err
+ }
+ if err := s.recordAudit(ctx, scope, "mkdir", resolved.RelativePath, 0); err != nil {
+ return nil, err
+ }
+ if err := root.MkdirAll(resolvedName, 0750); err != nil {
+ return nil, err
+ }
+ if err := ensureWorkspaceRuntimeOwnership(scope, resolved.RelativePath); err != nil {
+ _ = root.RemoveAll(resolvedName)
+ return nil, err
+ }
+ info, err := root.Stat(resolvedName)
+ if err != nil {
+ return nil, err
+ }
+ entry := buildWorkspaceEntry(resolved.RelativePath, info)
+ return &entry, nil
+}
+
+func (s *workspaceFileService) Rename(ctx context.Context, scope WorkspaceFileScope, oldPath, newPath string) (*WorkspaceEntry, error) {
+ if isWorkspaceRootRelative(oldPath) || isWorkspaceRootRelative(newPath) {
+ return nil, ErrWorkspaceRootOperation
+ }
+ oldResolved, err := ResolveWorkspacePath(scope.WorkspacePath, oldPath, false)
+ if err != nil {
+ return nil, err
+ }
+ newResolved, err := ResolveWorkspacePath(scope.WorkspacePath, newPath, true)
+ if err != nil {
+ return nil, err
+ }
+ root, err := openWorkspaceRoot(scope.WorkspacePath)
+ if err != nil {
+ return nil, err
+ }
+ defer root.Close()
+ oldName := workspaceRootName(oldResolved.RelativePath)
+ newName := workspaceRootName(newResolved.RelativePath)
+ if _, err := root.Lstat(newName); err == nil {
+ return nil, ErrWorkspaceEntryExists
+ } else if !os.IsNotExist(err) {
+ return nil, err
+ }
+ if _, err := root.Lstat(oldName); err != nil {
+ return nil, err
+ }
+ auditPath := oldResolved.RelativePath + " -> " + newResolved.RelativePath
+ if err := s.recordAudit(ctx, scope, "rename", auditPath, 0); err != nil {
+ return nil, err
+ }
+ if err := root.Rename(oldName, newName); err != nil {
+ return nil, err
+ }
+ if err := ensureWorkspaceRuntimeOwnership(scope, newResolved.RelativePath); err != nil {
+ return nil, err
+ }
+ info, err := root.Stat(newName)
+ if err != nil {
+ return nil, err
+ }
+ entry := buildWorkspaceEntry(newResolved.RelativePath, info)
+ return &entry, nil
+}
+
+func (s *workspaceFileService) Delete(ctx context.Context, scope WorkspaceFileScope, relativePath string) error {
+ if isWorkspaceRootRelative(relativePath) {
+ return ErrWorkspaceRootOperation
+ }
+ resolved, err := ResolveWorkspacePath(scope.WorkspacePath, relativePath, false)
+ if err != nil {
+ return err
+ }
+ if err := s.recordAudit(ctx, scope, "delete", resolved.RelativePath, 0); err != nil {
+ return err
+ }
+ root, err := openWorkspaceRoot(scope.WorkspacePath)
+ if err != nil {
+ return err
+ }
+ defer root.Close()
+ return root.RemoveAll(workspaceRootName(resolved.RelativePath))
+}
+
+func (s *workspaceFileService) openExistingFile(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*ResolvedWorkspacePath, *os.File, os.FileInfo, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, nil, nil, err
+ }
+ resolved, err := ResolveWorkspacePath(scope.WorkspacePath, relativePath, false)
+ if err != nil {
+ return nil, nil, nil, err
+ }
+ root, err := openWorkspaceRoot(scope.WorkspacePath)
+ if err != nil {
+ return nil, nil, nil, err
+ }
+ defer root.Close()
+ file, err := root.Open(workspaceRootName(resolved.RelativePath))
+ if err != nil {
+ return nil, nil, nil, err
+ }
+ info, err := file.Stat()
+ if err != nil {
+ file.Close()
+ return nil, nil, nil, err
+ }
+ if info.IsDir() {
+ file.Close()
+ return nil, nil, nil, ErrWorkspaceFileExpected
+ }
+ return resolved, file, info, nil
+}
+
+func (s *workspaceFileService) recordAudit(ctx context.Context, scope WorkspaceFileScope, action, relativePath string, bytes int64) error {
+ if s.auditRepo == nil {
+ return nil
+ }
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ return s.auditRepo.Create(ctx, &models.WorkspaceFileAudit{
+ InstanceID: scope.InstanceID,
+ UserID: scope.UserID,
+ Action: action,
+ RelativePath: relativePath,
+ Bytes: bytes,
+ CreatedAt: time.Now().UTC(),
+ })
+}
+
+func buildWorkspaceEntry(relativePath string, info os.FileInfo) WorkspaceEntry {
+ isDir := info.IsDir()
+ return WorkspaceEntry{
+ Name: info.Name(),
+ Path: filepath.ToSlash(relativePath),
+ IsDir: isDir,
+ Size: info.Size(),
+ ModifiedAt: info.ModTime().UTC(),
+ Previewable: workspaceEntryPreviewable(info.Name(), info.Size(), isDir),
+ Downloadable: !isDir,
+ }
+}
+
+func workspaceEntryPreviewable(name string, size int64, isDir bool) bool {
+ if isDir {
+ return false
+ }
+ kind, _, previewable, maxBytes := workspacePreviewKind(name, size)
+ if !previewable {
+ return false
+ }
+ return kind == "pdf" || maxBytes <= 0 || size <= maxBytes
+}
+
+func workspacePreviewKind(name string, size int64) (string, string, bool, int64) {
+ ext := strings.ToLower(filepath.Ext(name))
+ switch ext {
+ case ".txt", ".md", ".json", ".yaml", ".yml", ".log", ".py", ".js", ".ts", ".go", ".sh":
+ return "text", "text/plain; charset=utf-8", true, textPreviewMaxBytes
+ case ".png", ".jpg", ".jpeg", ".gif", ".webp":
+ contentType := mime.TypeByExtension(ext)
+ if contentType == "" {
+ contentType = "application/octet-stream"
+ }
+ return "image", contentType, true, imagePreviewMaxBytes
+ case ".pdf":
+ return "pdf", "application/pdf", true, 0
+ default:
+ return "binary", "application/octet-stream", false, 0
+ }
+}
+
+func sanitizeWorkspaceFilename(name string) (string, error) {
+ name = strings.TrimSpace(name)
+ if name == "" || name == "." || name == ".." {
+ return "", ErrWorkspaceFileNameInvalid
+ }
+ if strings.ContainsAny(name, `/\`) || strings.Contains(name, "\x00") {
+ return "", ErrWorkspaceFileNameInvalid
+ }
+ cleaned := strings.Map(func(r rune) rune {
+ if unicode.IsControl(r) {
+ return -1
+ }
+ switch r {
+ case ':', '*', '?', '"', '<', '>', '|':
+ return '-'
+ default:
+ return r
+ }
+ }, name)
+ cleaned = strings.TrimSpace(cleaned)
+ if cleaned == "" || cleaned == "." || cleaned == ".." {
+ return "", ErrWorkspaceFileNameInvalid
+ }
+ return cleaned, nil
+}
+
+func joinWorkspaceRelative(base, name string) string {
+ base = filepath.ToSlash(strings.TrimSpace(base))
+ name = filepath.ToSlash(strings.TrimSpace(name))
+ if base == "" {
+ return path.Clean(name)
+ }
+ return path.Clean(base + "/" + name)
+}
+
+func ensureWorkspaceRuntimeOwnership(scope WorkspaceFileScope, relativePath string) error {
+ if _, _, ok := runtimeWorkspaceOwner(scope); !ok {
+ return nil
+ }
+ relative, err := cleanWorkspaceRelativePath(relativePath)
+ if err != nil {
+ return err
+ }
+ if relative == "" {
+ return nil
+ }
+
+ parts := strings.Split(relative, "/")
+ for idx := range parts {
+ current := strings.Join(parts[:idx+1], "/")
+ resolved, err := ResolveWorkspacePath(scope.WorkspacePath, current, false)
+ if err != nil {
+ return err
+ }
+ info, err := os.Stat(resolved.RealPath)
+ if err != nil {
+ return err
+ }
+ mode := os.FileMode(0640)
+ if info.IsDir() {
+ mode = 0750
+ }
+ if err := applyWorkspaceRuntimeOwnershipToPath(scope, resolved.RealPath, mode); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func applyWorkspaceRuntimeOwnership(scope WorkspaceFileScope, relativePath string, mode os.FileMode) error {
+ if _, _, ok := runtimeWorkspaceOwner(scope); !ok {
+ return nil
+ }
+ resolved, err := ResolveWorkspacePath(scope.WorkspacePath, relativePath, false)
+ if err != nil {
+ return err
+ }
+ return applyWorkspaceRuntimeOwnershipToPath(scope, resolved.RealPath, mode)
+}
+
+func applyWorkspaceRuntimeOwnershipToPath(scope WorkspaceFileScope, targetPath string, mode os.FileMode) error {
+ uid, gid, ok := runtimeWorkspaceOwner(scope)
+ if !ok {
+ return nil
+ }
+ if err := os.Chown(targetPath, uid, gid); err != nil {
+ return err
+ }
+ if err := os.Chmod(targetPath, mode); err != nil {
+ return err
+ }
+ return nil
+}
+
+func runtimeWorkspaceOwner(scope WorkspaceFileScope) (int, int, bool) {
+ if scope.InstanceID <= 0 || scope.UserID <= 0 {
+ return 0, 0, false
+ }
+ cleaned := filepath.ToSlash(filepath.Clean(strings.TrimSpace(scope.WorkspacePath)))
+ parts := strings.Split(strings.Trim(cleaned, "/"), "/")
+ if len(parts) < 3 {
+ return 0, 0, false
+ }
+ instancePart := fmt.Sprintf("instance-%d", scope.InstanceID)
+ userPart := fmt.Sprintf("user-%d", scope.UserID)
+ runtimeType := parts[len(parts)-3]
+ if parts[len(parts)-1] != instancePart || parts[len(parts)-2] != userPart {
+ return 0, 0, false
+ }
+ if runtimeType != RuntimeTypeOpenClaw && runtimeType != RuntimeTypeHermes {
+ return 0, 0, false
+ }
+ linuxID := RuntimeLinuxID(scope.InstanceID)
+ return linuxID, linuxID, true
+}
+
+func isWorkspaceRootRelative(relativePath string) bool {
+ cleaned, err := cleanWorkspaceRelativePath(relativePath)
+ return err == nil && cleaned == ""
+}
+
+func openWorkspaceRoot(workspacePath string) (*os.Root, error) {
+ resolved, err := ResolveWorkspacePath(workspacePath, "", false)
+ if err != nil {
+ return nil, err
+ }
+ return os.OpenRoot(resolved.Root)
+}
+
+func workspaceRootName(relativePath string) string {
+ relativePath = filepath.ToSlash(strings.TrimSpace(relativePath))
+ if relativePath == "" {
+ return "."
+ }
+ return filepath.FromSlash(relativePath)
+}
+
+func workspaceDownloadURL(instanceID int, relativePath string) string {
+ return fmt.Sprintf("/api/v1/instances/%d/workspace/download?path=%s", instanceID, url.QueryEscape(filepath.ToSlash(relativePath)))
+}
+
+func workspacePreviewURL(instanceID int, relativePath string) string {
+ return fmt.Sprintf("/api/v1/instances/%d/workspace/preview?path=%s&raw=1", instanceID, url.QueryEscape(filepath.ToSlash(relativePath)))
+}
diff --git a/backend/internal/services/workspace_file_service_test.go b/backend/internal/services/workspace_file_service_test.go
new file mode 100644
index 0000000..2c9d7fd
--- /dev/null
+++ b/backend/internal/services/workspace_file_service_test.go
@@ -0,0 +1,302 @@
+package services
+
+import (
+ "context"
+ "errors"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "clawreef/internal/models"
+)
+
+type recordingWorkspaceFileAuditRepo struct {
+ items []models.WorkspaceFileAudit
+ err error
+}
+
+func (r *recordingWorkspaceFileAuditRepo) Create(ctx context.Context, audit *models.WorkspaceFileAudit) error {
+ if r.err != nil {
+ return r.err
+ }
+ r.items = append(r.items, *audit)
+ return nil
+}
+
+func workspaceTestScope(root string) WorkspaceFileScope {
+ return WorkspaceFileScope{
+ InstanceID: 12,
+ UserID: 34,
+ WorkspacePath: root,
+ }
+}
+
+func TestWorkspaceFileServiceListSortsDirectoriesFirstAndMarksCapabilities(t *testing.T) {
+ root := t.TempDir()
+ if err := os.Mkdir(filepath.Join(root, "docs"), 0750); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(root, "alpha.log"), []byte("hello"), 0640); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(root, "binary.dat"), []byte{0, 1, 2}, 0640); err != nil {
+ t.Fatal(err)
+ }
+
+ service := NewWorkspaceFileService(&recordingWorkspaceFileAuditRepo{})
+ entries, err := service.List(context.Background(), workspaceTestScope(root), "")
+ if err != nil {
+ t.Fatalf("List returned error: %v", err)
+ }
+
+ names := make([]string, 0, len(entries))
+ for _, entry := range entries {
+ names = append(names, entry.Name)
+ }
+ if strings.Join(names, ",") != "docs,alpha.log,binary.dat" {
+ t.Fatalf("entry order = %v, want dirs first then names", names)
+ }
+ if entries[1].Path != "alpha.log" || !entries[1].Previewable || !entries[1].Downloadable {
+ t.Fatalf("alpha.log capabilities = %#v, want previewable downloadable file", entries[1])
+ }
+ if entries[2].Previewable || !entries[2].Downloadable {
+ t.Fatalf("binary.dat capabilities = %#v, want download-only file", entries[2])
+ }
+}
+
+func TestWorkspaceFileServicePreviewTextAndDoesNotAudit(t *testing.T) {
+ root := t.TempDir()
+ if err := os.WriteFile(filepath.Join(root, "notes.md"), []byte("# hello"), 0640); err != nil {
+ t.Fatal(err)
+ }
+ auditRepo := &recordingWorkspaceFileAuditRepo{}
+
+ service := NewWorkspaceFileService(auditRepo)
+ preview, err := service.Preview(context.Background(), workspaceTestScope(root), "notes.md")
+ if err != nil {
+ t.Fatalf("Preview returned error: %v", err)
+ }
+
+ if preview.Kind != "text" || preview.Text != "# hello" {
+ t.Fatalf("preview = %#v, want text preview", preview)
+ }
+ if len(auditRepo.items) != 0 {
+ t.Fatalf("preview audited %d events, want none", len(auditRepo.items))
+ }
+}
+
+func TestWorkspaceFileServiceTreatsSVGAsDownloadOnly(t *testing.T) {
+ root := t.TempDir()
+ if err := os.WriteFile(filepath.Join(root, "icon.svg"), []byte(``), 0640); err != nil {
+ t.Fatal(err)
+ }
+
+ service := NewWorkspaceFileService(&recordingWorkspaceFileAuditRepo{})
+ entries, err := service.List(context.Background(), workspaceTestScope(root), "")
+ if err != nil {
+ t.Fatalf("List returned error: %v", err)
+ }
+ if len(entries) != 1 || entries[0].Previewable {
+ t.Fatalf("svg entry = %#v, want download-only", entries)
+ }
+ preview, err := service.Preview(context.Background(), workspaceTestScope(root), "icon.svg")
+ if err != nil {
+ t.Fatalf("Preview returned error: %v", err)
+ }
+ if preview.Kind != "binary" || preview.PreviewURL != "" {
+ t.Fatalf("svg preview = %#v, want binary without preview url", preview)
+ }
+}
+
+func TestWorkspaceFileServiceRejectsLargeTextPreview(t *testing.T) {
+ root := t.TempDir()
+ if err := os.WriteFile(filepath.Join(root, "large.log"), []byte(strings.Repeat("a", 1024*1024+1)), 0640); err != nil {
+ t.Fatal(err)
+ }
+
+ service := NewWorkspaceFileService(&recordingWorkspaceFileAuditRepo{})
+ _, err := service.Preview(context.Background(), workspaceTestScope(root), "large.log")
+ if !errors.Is(err, ErrWorkspacePreviewTooLarge) {
+ t.Fatalf("Preview error = %v, want ErrWorkspacePreviewTooLarge", err)
+ }
+}
+
+func TestWorkspaceFileServiceDownloadAuditsSuccessfulFileOpen(t *testing.T) {
+ root := t.TempDir()
+ if err := os.WriteFile(filepath.Join(root, "report.txt"), []byte("report"), 0640); err != nil {
+ t.Fatal(err)
+ }
+ auditRepo := &recordingWorkspaceFileAuditRepo{}
+
+ service := NewWorkspaceFileService(auditRepo)
+ file, filename, size, err := service.OpenDownload(context.Background(), workspaceTestScope(root), "report.txt")
+ if err != nil {
+ t.Fatalf("OpenDownload returned error: %v", err)
+ }
+ file.Close()
+
+ if filename != "report.txt" || size != int64(len("report")) {
+ t.Fatalf("download metadata filename=%q size=%d", filename, size)
+ }
+ if len(auditRepo.items) != 1 || auditRepo.items[0].Action != "download" || auditRepo.items[0].RelativePath != "report.txt" {
+ t.Fatalf("audit items = %#v, want download audit", auditRepo.items)
+ }
+}
+
+func TestWorkspaceFileServiceUploadRejectsEscapingSymlinkParent(t *testing.T) {
+ root := t.TempDir()
+ outside := t.TempDir()
+ if err := os.Symlink(outside, filepath.Join(root, "linked")); err != nil {
+ t.Skipf("symlink creation is not available: %v", err)
+ }
+
+ service := NewWorkspaceFileService(&recordingWorkspaceFileAuditRepo{})
+ _, err := service.Upload(context.Background(), workspaceTestScope(root), "linked", "owned.txt", strings.NewReader("owned"), int64(len("owned")))
+ if !errors.Is(err, ErrWorkspacePathEscape) {
+ t.Fatalf("Upload error = %v, want ErrWorkspacePathEscape", err)
+ }
+ if _, statErr := os.Stat(filepath.Join(outside, "owned.txt")); !errors.Is(statErr, os.ErrNotExist) {
+ t.Fatalf("outside file stat error = %v, want not exist", statErr)
+ }
+}
+
+func TestWorkspaceFileServiceUploadEnforcesMaxBytesBeforeWriting(t *testing.T) {
+ t.Setenv(WorkspaceUploadMaxBytesEnv, "3")
+ root := t.TempDir()
+
+ service := NewWorkspaceFileService(&recordingWorkspaceFileAuditRepo{})
+ _, err := service.Upload(context.Background(), workspaceTestScope(root), "", "big.txt", strings.NewReader("abcd"), int64(len("abcd")))
+ if !errors.Is(err, ErrWorkspaceUploadTooLarge) {
+ t.Fatalf("Upload error = %v, want ErrWorkspaceUploadTooLarge", err)
+ }
+ if _, statErr := os.Stat(filepath.Join(root, "big.txt")); !errors.Is(statErr, os.ErrNotExist) {
+ t.Fatalf("uploaded file stat error = %v, want not exist", statErr)
+ }
+}
+
+func TestWorkspaceFileServiceRuntimeWorkspaceOwner(t *testing.T) {
+ scope := WorkspaceFileScope{
+ InstanceID: 74,
+ UserID: 1,
+ WorkspacePath: "/workspaces/openclaw/user-1/instance-74",
+ }
+ uid, gid, ok := runtimeWorkspaceOwner(scope)
+ if !ok {
+ t.Fatal("runtimeWorkspaceOwner ok = false, want true")
+ }
+ if uid != RuntimeLinuxID(74) || gid != RuntimeLinuxID(74) {
+ t.Fatalf("runtime owner uid:gid = %d:%d, want %d:%d", uid, gid, RuntimeLinuxID(74), RuntimeLinuxID(74))
+ }
+
+ scope.WorkspacePath = "/tmp/workspaces/openclaw/user-1/instance-74"
+ if _, _, ok := runtimeWorkspaceOwner(scope); !ok {
+ t.Fatal("runtimeWorkspaceOwner should support configurable workspace roots")
+ }
+
+ scope.WorkspacePath = "/workspaces/openclaw/user-2/instance-74"
+ if _, _, ok := runtimeWorkspaceOwner(scope); ok {
+ t.Fatal("runtimeWorkspaceOwner ok = true for mismatched user")
+ }
+
+ scope.WorkspacePath = "/var/lib/clawmanager/user-1/instance-74"
+ if _, _, ok := runtimeWorkspaceOwner(scope); ok {
+ t.Fatal("runtimeWorkspaceOwner ok = true for non-runtime workspace")
+ }
+}
+
+func TestWorkspaceFileServiceUploadAuditFailureRemovesFile(t *testing.T) {
+ root := t.TempDir()
+ auditRepo := &recordingWorkspaceFileAuditRepo{err: errors.New("audit unavailable")}
+ service := NewWorkspaceFileService(auditRepo)
+
+ _, err := service.Upload(context.Background(), workspaceTestScope(root), "", "created.txt", strings.NewReader("body"), int64(len("body")))
+ if err == nil || !strings.Contains(err.Error(), "audit unavailable") {
+ t.Fatalf("Upload error = %v, want audit failure", err)
+ }
+ if _, statErr := os.Stat(filepath.Join(root, "created.txt")); !errors.Is(statErr, os.ErrNotExist) {
+ t.Fatalf("created file stat error = %v, want not exist", statErr)
+ }
+}
+
+func TestWorkspaceFileServiceMkdirAuditFailureRemovesDirectory(t *testing.T) {
+ root := t.TempDir()
+ auditRepo := &recordingWorkspaceFileAuditRepo{err: errors.New("audit unavailable")}
+ service := NewWorkspaceFileService(auditRepo)
+
+ _, err := service.Mkdir(context.Background(), workspaceTestScope(root), "docs")
+ if err == nil || !strings.Contains(err.Error(), "audit unavailable") {
+ t.Fatalf("Mkdir error = %v, want audit failure", err)
+ }
+ if _, statErr := os.Stat(filepath.Join(root, "docs")); !errors.Is(statErr, os.ErrNotExist) {
+ t.Fatalf("created directory stat error = %v, want not exist", statErr)
+ }
+}
+
+func TestWorkspaceFileServiceRenameAuditFailureRollsBack(t *testing.T) {
+ root := t.TempDir()
+ if err := os.WriteFile(filepath.Join(root, "old.txt"), []byte("body"), 0640); err != nil {
+ t.Fatal(err)
+ }
+ auditRepo := &recordingWorkspaceFileAuditRepo{err: errors.New("audit unavailable")}
+ service := NewWorkspaceFileService(auditRepo)
+
+ _, err := service.Rename(context.Background(), workspaceTestScope(root), "old.txt", "new.txt")
+ if err == nil || !strings.Contains(err.Error(), "audit unavailable") {
+ t.Fatalf("Rename error = %v, want audit failure", err)
+ }
+ if _, statErr := os.Stat(filepath.Join(root, "old.txt")); statErr != nil {
+ t.Fatalf("old file stat error = %v, want rollback", statErr)
+ }
+ if _, statErr := os.Stat(filepath.Join(root, "new.txt")); !errors.Is(statErr, os.ErrNotExist) {
+ t.Fatalf("new file stat error = %v, want not exist", statErr)
+ }
+}
+
+func TestWorkspaceFileServiceDeleteAuditFailureKeepsFile(t *testing.T) {
+ root := t.TempDir()
+ if err := os.WriteFile(filepath.Join(root, "keep.txt"), []byte("body"), 0640); err != nil {
+ t.Fatal(err)
+ }
+ auditRepo := &recordingWorkspaceFileAuditRepo{err: errors.New("audit unavailable")}
+ service := NewWorkspaceFileService(auditRepo)
+
+ err := service.Delete(context.Background(), workspaceTestScope(root), "keep.txt")
+ if err == nil || !strings.Contains(err.Error(), "audit unavailable") {
+ t.Fatalf("Delete error = %v, want audit failure", err)
+ }
+ if _, statErr := os.Stat(filepath.Join(root, "keep.txt")); statErr != nil {
+ t.Fatalf("kept file stat error = %v", statErr)
+ }
+}
+
+func TestWorkspaceFileServiceMutationsAuditAndProtectRoot(t *testing.T) {
+ root := t.TempDir()
+ auditRepo := &recordingWorkspaceFileAuditRepo{}
+ service := NewWorkspaceFileService(auditRepo)
+ scope := workspaceTestScope(root)
+
+ if err := service.Delete(context.Background(), scope, ""); !errors.Is(err, ErrWorkspaceRootOperation) {
+ t.Fatalf("Delete root error = %v, want ErrWorkspaceRootOperation", err)
+ }
+ if _, err := service.Rename(context.Background(), scope, "", "renamed"); !errors.Is(err, ErrWorkspaceRootOperation) {
+ t.Fatalf("Rename root error = %v, want ErrWorkspaceRootOperation", err)
+ }
+ if _, err := service.Mkdir(context.Background(), scope, "docs"); err != nil {
+ t.Fatalf("Mkdir returned error: %v", err)
+ }
+ if _, err := service.Rename(context.Background(), scope, "docs", "notes"); err != nil {
+ t.Fatalf("Rename returned error: %v", err)
+ }
+ if err := service.Delete(context.Background(), scope, "notes"); err != nil {
+ t.Fatalf("Delete returned error: %v", err)
+ }
+
+ actions := make([]string, 0, len(auditRepo.items))
+ for _, item := range auditRepo.items {
+ actions = append(actions, item.Action)
+ }
+ if strings.Join(actions, ",") != "mkdir,rename,delete" {
+ t.Fatalf("audit actions = %v, want mkdir,rename,delete", actions)
+ }
+}
diff --git a/backend/internal/services/workspace_path_guard.go b/backend/internal/services/workspace_path_guard.go
new file mode 100644
index 0000000..c8b1e31
--- /dev/null
+++ b/backend/internal/services/workspace_path_guard.go
@@ -0,0 +1,202 @@
+package services
+
+import (
+ "errors"
+ "fmt"
+ "os"
+ "path"
+ "path/filepath"
+ "runtime"
+ "strings"
+)
+
+var (
+ ErrWorkspacePathInvalid = errors.New("workspace path is invalid")
+ ErrWorkspacePathEscape = errors.New("workspace path escapes workspace")
+ ErrWorkspacePathNotFound = errors.New("workspace entry not found")
+)
+
+type ResolvedWorkspacePath struct {
+ Root string
+ Path string
+ RealPath string
+ RelativePath string
+}
+
+func ResolveWorkspacePath(workspaceRoot, relativePath string, allowMissingLeaf bool) (*ResolvedWorkspacePath, error) {
+ root := strings.TrimSpace(workspaceRoot)
+ if root == "" {
+ return nil, fmt.Errorf("%w: workspace root is required", ErrWorkspacePathInvalid)
+ }
+ if strings.Contains(root, "\x00") {
+ return nil, fmt.Errorf("%w: workspace root contains null byte", ErrWorkspacePathInvalid)
+ }
+
+ rootAbs, err := filepath.Abs(root)
+ if err != nil {
+ return nil, fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, err)
+ }
+ rootReal, err := filepath.EvalSymlinks(rootAbs)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, fmt.Errorf("%w: workspace root", ErrWorkspacePathNotFound)
+ }
+ return nil, fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, err)
+ }
+ rootReal = filepath.Clean(rootReal)
+ rootInfo, err := os.Stat(rootReal)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, fmt.Errorf("%w: workspace root", ErrWorkspacePathNotFound)
+ }
+ return nil, fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, err)
+ }
+ if !rootInfo.IsDir() {
+ return nil, fmt.Errorf("%w: workspace root is not a directory", ErrWorkspacePathInvalid)
+ }
+
+ relative, err := cleanWorkspaceRelativePath(relativePath)
+ if err != nil {
+ return nil, err
+ }
+
+ targetPath := rootReal
+ if relative != "" {
+ targetPath = filepath.Join(rootReal, filepath.FromSlash(relative))
+ }
+ targetPath = filepath.Clean(targetPath)
+ if !isWorkspaceSubpath(rootReal, targetPath) {
+ return nil, fmt.Errorf("%w: %s", ErrWorkspacePathEscape, relativePath)
+ }
+
+ realPath, err := resolveWorkspaceRealPath(targetPath, allowMissingLeaf)
+ if err != nil {
+ return nil, err
+ }
+ realPath = filepath.Clean(realPath)
+ if !isWorkspaceSubpath(rootReal, realPath) {
+ return nil, fmt.Errorf("%w: %s", ErrWorkspacePathEscape, relativePath)
+ }
+
+ return &ResolvedWorkspacePath{
+ Root: rootReal,
+ Path: targetPath,
+ RealPath: realPath,
+ RelativePath: relative,
+ }, nil
+}
+
+func cleanWorkspaceRelativePath(relativePath string) (string, error) {
+ raw := strings.TrimSpace(relativePath)
+ if raw == "" || raw == "." {
+ return "", nil
+ }
+ if strings.Contains(raw, "\x00") {
+ return "", fmt.Errorf("%w: path contains null byte", ErrWorkspacePathInvalid)
+ }
+ if filepath.IsAbs(raw) || filepath.VolumeName(raw) != "" {
+ return "", fmt.Errorf("%w: absolute paths are not allowed", ErrWorkspacePathEscape)
+ }
+
+ normalized := strings.ReplaceAll(raw, "\\", "/")
+ if path.IsAbs(normalized) || isWindowsDrivePath(normalized) {
+ return "", fmt.Errorf("%w: absolute paths are not allowed", ErrWorkspacePathEscape)
+ }
+
+ parts := strings.Split(normalized, "/")
+ cleaned := make([]string, 0, len(parts))
+ for _, part := range parts {
+ switch part {
+ case "", ".":
+ continue
+ case "..":
+ return "", fmt.Errorf("%w: traversal is not allowed", ErrWorkspacePathEscape)
+ default:
+ cleaned = append(cleaned, part)
+ }
+ }
+ if len(cleaned) == 0 {
+ return "", nil
+ }
+ return path.Clean(strings.Join(cleaned, "/")), nil
+}
+
+func isWindowsDrivePath(value string) bool {
+ if len(value) < 2 || value[1] != ':' {
+ return false
+ }
+ first := value[0]
+ return (first >= 'a' && first <= 'z') || (first >= 'A' && first <= 'Z')
+}
+
+func resolveWorkspaceRealPath(targetPath string, allowMissingLeaf bool) (string, error) {
+ realPath, err := filepath.EvalSymlinks(targetPath)
+ if err == nil {
+ return realPath, nil
+ }
+ if !os.IsNotExist(err) {
+ return "", fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, err)
+ }
+ if !allowMissingLeaf {
+ return "", fmt.Errorf("%w: %s", ErrWorkspacePathNotFound, targetPath)
+ }
+
+ existingPath := targetPath
+ missingParts := []string{}
+ for {
+ info, statErr := os.Lstat(existingPath)
+ if statErr == nil {
+ if len(missingParts) > 0 {
+ info, statErr = os.Stat(existingPath)
+ if statErr != nil {
+ if os.IsNotExist(statErr) {
+ return "", fmt.Errorf("%w: %s", ErrWorkspacePathNotFound, existingPath)
+ }
+ return "", fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, statErr)
+ }
+ if !info.IsDir() {
+ return "", fmt.Errorf("%w: parent is not a directory", ErrWorkspacePathInvalid)
+ }
+ }
+ parentReal, evalErr := filepath.EvalSymlinks(existingPath)
+ if evalErr != nil {
+ if os.IsNotExist(evalErr) {
+ return "", fmt.Errorf("%w: %s", ErrWorkspacePathNotFound, existingPath)
+ }
+ return "", fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, evalErr)
+ }
+ parts := append([]string{parentReal}, missingParts...)
+ return filepath.Join(parts...), nil
+ }
+ if !os.IsNotExist(statErr) {
+ return "", fmt.Errorf("%w: %v", ErrWorkspacePathInvalid, statErr)
+ }
+
+ parent := filepath.Dir(existingPath)
+ if parent == existingPath {
+ return "", fmt.Errorf("%w: %s", ErrWorkspacePathNotFound, targetPath)
+ }
+ missingParts = append([]string{filepath.Base(existingPath)}, missingParts...)
+ existingPath = parent
+ }
+}
+
+func isWorkspaceSubpath(root, target string) bool {
+ root = filepath.Clean(root)
+ target = filepath.Clean(target)
+ if sameWorkspacePath(root, target) {
+ return true
+ }
+ relative, err := filepath.Rel(root, target)
+ if err != nil || relative == "." || filepath.IsAbs(relative) {
+ return false
+ }
+ return relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator))
+}
+
+func sameWorkspacePath(left, right string) bool {
+ if runtime.GOOS == "windows" {
+ return strings.EqualFold(left, right)
+ }
+ return left == right
+}
diff --git a/backend/internal/services/workspace_path_guard_test.go b/backend/internal/services/workspace_path_guard_test.go
new file mode 100644
index 0000000..1ca2f13
--- /dev/null
+++ b/backend/internal/services/workspace_path_guard_test.go
@@ -0,0 +1,76 @@
+package services
+
+import (
+ "errors"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+func TestResolveWorkspacePathRejectsTraversalAndAbsolutePaths(t *testing.T) {
+ root := t.TempDir()
+
+ for _, relativePath := range []string{
+ "../secret.txt",
+ "docs/../../secret.txt",
+ "/etc/passwd",
+ `C:\Windows\System32\drivers\etc\hosts`,
+ "C:/Windows/System32/drivers/etc/hosts",
+ } {
+ t.Run(relativePath, func(t *testing.T) {
+ _, err := ResolveWorkspacePath(root, relativePath, false)
+ if !errors.Is(err, ErrWorkspacePathEscape) && !errors.Is(err, ErrWorkspacePathInvalid) {
+ t.Fatalf("ResolveWorkspacePath(%q) error = %v, want path safety error", relativePath, err)
+ }
+ })
+ }
+}
+
+func TestResolveWorkspacePathAllowsMissingChildInsideWorkspace(t *testing.T) {
+ root := t.TempDir()
+ if err := os.Mkdir(filepath.Join(root, "docs"), 0750); err != nil {
+ t.Fatal(err)
+ }
+
+ resolved, err := ResolveWorkspacePath(root, "docs/new.txt", true)
+ if err != nil {
+ t.Fatalf("ResolveWorkspacePath returned error: %v", err)
+ }
+
+ if resolved.RelativePath != "docs/new.txt" {
+ t.Fatalf("relative path = %q, want docs/new.txt", resolved.RelativePath)
+ }
+ if !strings.HasSuffix(filepath.ToSlash(resolved.Path), "/docs/new.txt") {
+ t.Fatalf("target path = %q, want path under docs/new.txt", resolved.Path)
+ }
+}
+
+func TestResolveWorkspacePathRejectsExistingSymlinkEscape(t *testing.T) {
+ root := t.TempDir()
+ outside := t.TempDir()
+ if err := os.WriteFile(filepath.Join(outside, "secret.txt"), []byte("secret"), 0640); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(filepath.Join(outside, "secret.txt"), filepath.Join(root, "leak.txt")); err != nil {
+ t.Skipf("symlink creation is not available: %v", err)
+ }
+
+ _, err := ResolveWorkspacePath(root, "leak.txt", false)
+ if !errors.Is(err, ErrWorkspacePathEscape) {
+ t.Fatalf("ResolveWorkspacePath through symlink error = %v, want ErrWorkspacePathEscape", err)
+ }
+}
+
+func TestResolveWorkspacePathRejectsMissingTargetUnderEscapingSymlinkParent(t *testing.T) {
+ root := t.TempDir()
+ outside := t.TempDir()
+ if err := os.Symlink(outside, filepath.Join(root, "linked")); err != nil {
+ t.Skipf("symlink creation is not available: %v", err)
+ }
+
+ _, err := ResolveWorkspacePath(root, "linked/new.txt", true)
+ if !errors.Is(err, ErrWorkspacePathEscape) {
+ t.Fatalf("ResolveWorkspacePath through symlink parent error = %v, want ErrWorkspacePathEscape", err)
+ }
+}
diff --git a/deployments/container/start.sh b/deployments/container/start.sh
index 61174d1..a58a977 100644
--- a/deployments/container/start.sh
+++ b/deployments/container/start.sh
@@ -3,26 +3,84 @@ set -eu
TLS_DIR="/etc/nginx/tls"
TLS_SOURCE_DIR="${TLS_SOURCE_DIR:-/var/run/clawreef-tls}"
+TLS_SHARED_DIR="${TLS_SHARED_DIR:-/workspaces/.clawmanager/tls}"
+TLS_CN="${TLS_CN:-clawmanager.local}"
+TLS_SUBJECT_ALT_NAME="${TLS_SUBJECT_ALT_NAME:-DNS:clawmanager.local,DNS:localhost,IP:127.0.0.1}"
TLS_CERT="${TLS_DIR}/tls.crt"
TLS_KEY="${TLS_DIR}/tls.key"
SOURCE_CERT="${TLS_SOURCE_DIR}/tls.crt"
SOURCE_KEY="${TLS_SOURCE_DIR}/tls.key"
+SHARED_CERT="${TLS_SHARED_DIR}/tls.crt"
+SHARED_KEY="${TLS_SHARED_DIR}/tls.key"
+SHARED_LOCK_DIR="${TLS_SHARED_DIR}.lock"
-mkdir -p "${TLS_DIR}"
-
-if [ -f "${SOURCE_CERT}" ] && [ -f "${SOURCE_KEY}" ]; then
- cp "${SOURCE_CERT}" "${TLS_CERT}"
- cp "${SOURCE_KEY}" "${TLS_KEY}"
-elif [ ! -f "${TLS_CERT}" ] || [ ! -f "${TLS_KEY}" ]; then
- echo "TLS certificate not found, generating a self-signed certificate for bootstrap use."
+generate_self_signed_cert() {
+ cert_path="$1"
+ key_path="$2"
openssl req \
-x509 \
-nodes \
-days 365 \
-newkey rsa:2048 \
- -subj "/CN=clawreef.local" \
- -keyout "${TLS_KEY}" \
- -out "${TLS_CERT}"
+ -subj "/CN=${TLS_CN}" \
+ -addext "subjectAltName=${TLS_SUBJECT_ALT_NAME}" \
+ -keyout "${key_path}" \
+ -out "${cert_path}" || \
+ openssl req \
+ -x509 \
+ -nodes \
+ -days 365 \
+ -newkey rsa:2048 \
+ -subj "/CN=${TLS_CN}" \
+ -keyout "${key_path}" \
+ -out "${cert_path}"
+}
+
+copy_cert_pair() {
+ cp "$1" "${TLS_CERT}"
+ cp "$2" "${TLS_KEY}"
+ chmod 0644 "${TLS_CERT}"
+ chmod 0600 "${TLS_KEY}"
+}
+
+ensure_shared_cert() {
+ if [ -f "${SHARED_CERT}" ] && [ -f "${SHARED_KEY}" ]; then
+ return 0
+ fi
+
+ mkdir -p "${TLS_SHARED_DIR}" 2>/dev/null || return 1
+ if mkdir "${SHARED_LOCK_DIR}" 2>/dev/null; then
+ if [ ! -f "${SHARED_CERT}" ] || [ ! -f "${SHARED_KEY}" ]; then
+ tmp_cert="${SHARED_CERT}.$$"
+ tmp_key="${SHARED_KEY}.$$"
+ generate_self_signed_cert "${tmp_cert}" "${tmp_key}"
+ chmod 0644 "${tmp_cert}"
+ chmod 0600 "${tmp_key}"
+ mv "${tmp_cert}" "${SHARED_CERT}"
+ mv "${tmp_key}" "${SHARED_KEY}"
+ fi
+ rmdir "${SHARED_LOCK_DIR}" 2>/dev/null || true
+ return 0
+ fi
+
+ for _ in $(seq 1 60); do
+ if [ -f "${SHARED_CERT}" ] && [ -f "${SHARED_KEY}" ]; then
+ return 0
+ fi
+ sleep 1
+ done
+ return 1
+}
+
+mkdir -p "${TLS_DIR}"
+
+if [ -f "${SOURCE_CERT}" ] && [ -f "${SOURCE_KEY}" ]; then
+ copy_cert_pair "${SOURCE_CERT}" "${SOURCE_KEY}"
+elif ensure_shared_cert; then
+ copy_cert_pair "${SHARED_CERT}" "${SHARED_KEY}"
+elif [ ! -f "${TLS_CERT}" ] || [ ! -f "${TLS_KEY}" ]; then
+ echo "TLS certificate not found, generating a self-signed certificate for bootstrap use."
+ generate_self_signed_cert "${TLS_CERT}" "${TLS_KEY}"
fi
export SERVER_ADDRESS="${SERVER_ADDRESS:-:9001}"
diff --git a/deployments/hermes-runtime/Dockerfile.tui-dist b/deployments/hermes-runtime/Dockerfile.tui-dist
new file mode 100644
index 0000000..ae45802
--- /dev/null
+++ b/deployments/hermes-runtime/Dockerfile.tui-dist
@@ -0,0 +1,9 @@
+ARG HERMES_RUNTIME_BASE_IMAGE=172.16.1.12:5010/hermes:team-lite-envfix-20260610123402
+FROM ${HERMES_RUNTIME_BASE_IMAGE}
+
+USER root
+
+RUN cd /usr/local/lib/hermes-agent/ui-tui \
+ && npm run build \
+ && test -f dist/entry.js \
+ && chmod -R a+rX dist
diff --git a/deployments/k3s/clawmanager.yaml b/deployments/k3s/clawmanager.yaml
index 7c72dac..a264109 100644
--- a/deployments/k3s/clawmanager.yaml
+++ b/deployments/k3s/clawmanager.yaml
@@ -13,6 +13,9 @@ stringData:
mysql-root-password: root123
mysql-password: clawreef123
jwt-secret: change-me-in-production
+ runtime-agent-control-token: change-me-runtime-control-token
+ runtime-agent-report-token: change-me-runtime-report-token
+ openclaw-gateway-token: change-me-openclaw-gateway-token
---
apiVersion: v1
kind: ConfigMap
@@ -635,8 +638,8 @@ data:
SELECT
'openclaw',
'shell',
- 'OpenClaw Shell',
- 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-shell:latest',
+ 'OpenClaw Lite',
+ 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest',
TRUE
WHERE NOT EXISTS (
SELECT 1
@@ -671,6 +674,39 @@ data:
updated_at = CURRENT_TIMESTAMP
WHERE type IN ('openclaw', 'ubuntu', 'webtop', 'hermes')
AND mount_path IN ('/data', '/home/user/data', '/config/.hermes');
+ 031_add_hermes_lite_runtime_image.sql: |
+ USE clawmanager;
+ UPDATE system_image_settings
+ SET
+ image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest',
+ display_name = 'Hermes Lite',
+ is_enabled = TRUE
+ WHERE instance_type = 'hermes'
+ AND runtime_type = 'shell'
+ AND image IN (
+ 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-shell:latest',
+ 'registry.example.com/your-custom-shell-image:latest'
+ );
+
+ INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled)
+ SELECT
+ 'hermes',
+ 'shell',
+ 'Hermes Lite',
+ 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest',
+ TRUE
+ WHERE NOT EXISTS (
+ SELECT 1
+ FROM system_image_settings
+ WHERE instance_type = 'hermes'
+ AND runtime_type = 'shell'
+ )
+ AND NOT EXISTS (
+ SELECT 1
+ FROM system_image_settings
+ WHERE instance_type = 'hermes'
+ AND is_enabled = FALSE
+ );
---
apiVersion: v1
kind: PersistentVolumeClaim
@@ -792,6 +828,19 @@ spec:
---
apiVersion: v1
kind: Service
+metadata:
+ name: clawmanager-redis
+ namespace: clawmanager-system
+spec:
+ selector:
+ app: clawmanager-team-redis
+ ports:
+ - name: redis
+ port: 6379
+ targetPort: redis
+---
+apiVersion: v1
+kind: Service
metadata:
name: clawmanager-team-redis
namespace: clawmanager-system
@@ -804,12 +853,120 @@ spec:
targetPort: redis
---
apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: clawmanager-workspaces
+ namespace: clawmanager-system
+spec:
+ accessModes:
+ - ReadWriteMany
+ resources:
+ requests:
+ storage: 50Gi
+ storageClassName: local-path
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: workspace-store
+ namespace: clawmanager-system
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: workspace-store
+ template:
+ metadata:
+ labels:
+ app: workspace-store
+ spec:
+ containers:
+ - name: nfs-server
+ image: itsthenetwork/nfs-server-alpine:12
+ imagePullPolicy: IfNotPresent
+ securityContext:
+ privileged: true
+ env:
+ - name: SHARED_DIRECTORY
+ value: /exports/workspaces
+ ports:
+ - name: nfs
+ containerPort: 2049
+ volumeMounts:
+ - name: workspaces
+ mountPath: /exports/workspaces
+ volumes:
+ - name: workspaces
+ persistentVolumeClaim:
+ claimName: clawmanager-workspaces
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: workspace-store
+ namespace: clawmanager-system
+spec:
+ selector:
+ app: workspace-store
+ ports:
+ - name: nfs
+ port: 2049
+ targetPort: nfs
+---
+apiVersion: v1
kind: ServiceAccount
metadata:
name: clawmanager-app
namespace: clawmanager-system
---
apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: clawmanager-runtime-manager
+rules:
+ - apiGroups:
+ - coordination.k8s.io
+ resources:
+ - leases
+ verbs:
+ - get
+ - create
+ - update
+ - apiGroups:
+ - apps
+ resources:
+ - deployments
+ - deployments/scale
+ verbs:
+ - get
+ - list
+ - watch
+ - create
+ - update
+ - patch
+ - apiGroups:
+ - ""
+ resources:
+ - pods
+ verbs:
+ - get
+ - list
+ - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: clawmanager-runtime-manager
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: clawmanager-runtime-manager
+subjects:
+ - kind: ServiceAccount
+ name: clawmanager-app
+ namespace: clawmanager-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: clawmanager-app-cluster-admin
@@ -828,7 +985,7 @@ metadata:
name: clawmanager-app
namespace: clawmanager-system
spec:
- replicas: 1
+ replicas: 3
selector:
matchLabels:
app: clawmanager-app
@@ -878,10 +1035,47 @@ spec:
value: "local-path"
- name: CLAWMANAGER_WORKSPACE_ARCHIVE_MAX_MIB
value: "500"
+ - name: PLATFORM_REDIS_URL
+ value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0"
+ - name: TEAM_REDIS_URL
+ value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0"
- name: CLAWMANAGER_TEAM_REDIS_URL
- value: "redis://clawmanager-team-redis.clawmanager-system.svc.cluster.local:6379/0"
+ value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0"
- name: CLAWMANAGER_TEAM_MANAGER_BASE_URL
value: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001"
+ - name: RUNTIME_NAMESPACE
+ value: "clawmanager-system"
+ - name: RUNTIME_WORKSPACE_ROOT
+ value: "/workspaces"
+ - name: RUNTIME_WORKSPACE_NFS_SERVER
+ value: "workspace-store.clawmanager-system.svc.cluster.local"
+ - name: RUNTIME_WORKSPACE_NFS_PATH
+ value: "/"
+ - name: RUNTIME_MAX_GATEWAYS_PER_POD
+ value: "100"
+ - name: RUNTIME_GATEWAY_PORT_START
+ value: "20000"
+ - name: RUNTIME_GATEWAY_PORT_END
+ value: "20099"
+ - name: RUNTIME_SCHEDULER_ENABLED
+ value: "true"
+ - name: RUNTIME_HEARTBEAT_TIMEOUT
+ value: "10s"
+ - name: RUNTIME_AGENT_CONTROL_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: clawmanager-secrets
+ key: runtime-agent-control-token
+ - name: RUNTIME_AGENT_REPORT_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: clawmanager-secrets
+ key: runtime-agent-report-token
+ - name: OPENCLAW_GATEWAY_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: clawmanager-secrets
+ key: openclaw-gateway-token
- name: SKILL_SCANNER_ENABLED
value: "true"
- name: SKILL_SCANNER_BASE_URL
@@ -895,8 +1089,13 @@ spec:
- name: OBJECT_STORAGE_LOCAL_FALLBACK
value: "/data/object-storage"
volumeMounts:
+ - name: tls-cert
+ mountPath: /var/run/clawreef-tls
+ readOnly: true
- name: object-storage
mountPath: /data/object-storage
+ - name: workspaces
+ mountPath: /workspaces
readinessProbe:
httpGet:
path: /healthz
@@ -914,8 +1113,203 @@ spec:
periodSeconds: 20
timeoutSeconds: 5
volumes:
+ - name: tls-cert
+ secret:
+ secretName: clawmanager-tls
+ optional: true
- name: object-storage
emptyDir: {}
+ - name: workspaces
+ nfs:
+ server: workspace-store.clawmanager-system.svc.cluster.local
+ path: /exports/workspaces
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: openclaw-runtime
+ namespace: clawmanager-system
+ labels:
+ app: openclaw-runtime
+ clawmanager.io/runtime-type: openclaw
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: openclaw-runtime
+ clawmanager.io/runtime-type: openclaw
+ template:
+ metadata:
+ labels:
+ app: openclaw-runtime
+ clawmanager.io/runtime-type: openclaw
+ spec:
+ containers:
+ - name: runtime
+ image: ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest
+ imagePullPolicy: Always
+ env:
+ - name: CLAWMANAGER_RUNTIME_TYPE
+ value: openclaw
+ - name: CLAWMANAGER_BACKEND_URL
+ value: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001"
+ - name: CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME
+ value: openclaw-runtime
+ - name: CLAWMANAGER_RUNTIME_IMAGE_REF
+ value: ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest
+ - name: RUNTIME_WORKSPACE_ROOT
+ value: /workspaces
+ - name: RUNTIME_GATEWAY_PORT_START
+ value: "20000"
+ - name: RUNTIME_GATEWAY_PORT_END
+ value: "20099"
+ - name: RUNTIME_AGENT_LISTEN_ADDR
+ value: "0.0.0.0:19090"
+ - name: RUNTIME_AGENT_PUBLIC_PORT
+ value: "19090"
+ - name: RUNTIME_GATEWAY_COMMAND
+ value: "/usr/local/bin/openclaw gateway run --allow-unconfigured --auth token --bind auto --force"
+ - name: OPENCLAW_GATEWAY_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: clawmanager-secrets
+ key: openclaw-gateway-token
+ - name: RUNTIME_AGENT_CONTROL_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: clawmanager-secrets
+ key: runtime-agent-control-token
+ - name: RUNTIME_AGENT_REPORT_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: clawmanager-secrets
+ key: runtime-agent-report-token
+ - name: POD_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ - name: POD_IP
+ valueFrom:
+ fieldRef:
+ fieldPath: status.podIP
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
+ ports:
+ - name: agent
+ containerPort: 19090
+ volumeMounts:
+ - name: workspaces
+ mountPath: /workspaces
+ volumes:
+ - name: workspaces
+ nfs:
+ server: workspace-store.clawmanager-system.svc.cluster.local
+ path: /exports/workspaces
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: hermes-runtime
+ namespace: clawmanager-system
+ labels:
+ app: hermes-runtime
+ clawmanager.io/runtime-type: hermes
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: hermes-runtime
+ clawmanager.io/runtime-type: hermes
+ template:
+ metadata:
+ labels:
+ app: hermes-runtime
+ clawmanager.io/runtime-type: hermes
+ spec:
+ containers:
+ - name: runtime
+ image: ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest
+ imagePullPolicy: IfNotPresent
+ env:
+ - name: CLAWMANAGER_RUNTIME_TYPE
+ value: hermes
+ - name: CLAWMANAGER_BACKEND_URL
+ value: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001"
+ - name: CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME
+ value: hermes-runtime
+ - name: CLAWMANAGER_RUNTIME_IMAGE_REF
+ value: ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest
+ - name: CLAWMANAGER_AGENT_PORT
+ value: "19090"
+ - name: CLAWMANAGER_GATEWAY_PORT_START
+ value: "20000"
+ - name: CLAWMANAGER_GATEWAY_PORT_END
+ value: "20099"
+ - name: HERMES_TUI_DIR
+ value: /usr/local/lib/hermes-agent/ui-tui
+ - name: CLAWMANAGER_AGENT_CONTROL_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: clawmanager-secrets
+ key: runtime-agent-control-token
+ - name: CLAWMANAGER_AGENT_REPORT_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: clawmanager-secrets
+ key: runtime-agent-report-token
+ - name: RUNTIME_WORKSPACE_ROOT
+ value: /workspaces
+ - name: RUNTIME_AGENT_LISTEN_ADDR
+ value: "0.0.0.0:19090"
+ - name: RUNTIME_AGENT_PUBLIC_PORT
+ value: "19090"
+ - name: RUNTIME_GATEWAY_PORT_START
+ value: "20000"
+ - name: RUNTIME_GATEWAY_PORT_END
+ value: "20099"
+ - name: RUNTIME_AGENT_CONTROL_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: clawmanager-secrets
+ key: runtime-agent-control-token
+ - name: RUNTIME_AGENT_REPORT_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: clawmanager-secrets
+ key: runtime-agent-report-token
+ - name: POD_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ - name: POD_IP
+ valueFrom:
+ fieldRef:
+ fieldPath: status.podIP
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
+ ports:
+ - name: agent
+ containerPort: 19090
+ volumeMounts:
+ - name: workspaces
+ mountPath: /workspaces
+ volumes:
+ - name: workspaces
+ nfs:
+ server: workspace-store.clawmanager-system.svc.cluster.local
+ path: /exports/workspaces
---
apiVersion: v1
kind: Service
@@ -930,7 +1324,7 @@ spec:
- name: https
port: 443
targetPort: https
- nodePort: 30443
+ nodePort: 39443
---
apiVersion: v1
kind: Service
diff --git a/deployments/k8s/clawmanager.yaml b/deployments/k8s/clawmanager.yaml
index 27ac892..167fc14 100644
--- a/deployments/k8s/clawmanager.yaml
+++ b/deployments/k8s/clawmanager.yaml
@@ -13,6 +13,9 @@ stringData:
mysql-root-password: root123
mysql-password: clawreef123
jwt-secret: change-me-in-production
+ runtime-agent-control-token: change-me-runtime-control-token
+ runtime-agent-report-token: change-me-runtime-report-token
+ openclaw-gateway-token: change-me-openclaw-gateway-token
minio-root-user: minioadmin
minio-root-password: minioadmin123
minio-access-key: minioadmin
@@ -639,8 +642,8 @@ data:
SELECT
'openclaw',
'shell',
- 'OpenClaw Shell',
- 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-shell:latest',
+ 'OpenClaw Lite',
+ 'ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest',
TRUE
WHERE NOT EXISTS (
SELECT 1
@@ -675,6 +678,39 @@ data:
updated_at = CURRENT_TIMESTAMP
WHERE type IN ('openclaw', 'ubuntu', 'webtop', 'hermes')
AND mount_path IN ('/data', '/home/user/data', '/config/.hermes');
+ 031_add_hermes_lite_runtime_image.sql: |
+ USE clawmanager;
+ UPDATE system_image_settings
+ SET
+ image = 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest',
+ display_name = 'Hermes Lite',
+ is_enabled = TRUE
+ WHERE instance_type = 'hermes'
+ AND runtime_type = 'shell'
+ AND image IN (
+ 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-shell:latest',
+ 'registry.example.com/your-custom-shell-image:latest'
+ );
+
+ INSERT INTO system_image_settings (instance_type, runtime_type, display_name, image, is_enabled)
+ SELECT
+ 'hermes',
+ 'shell',
+ 'Hermes Lite',
+ 'ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest',
+ TRUE
+ WHERE NOT EXISTS (
+ SELECT 1
+ FROM system_image_settings
+ WHERE instance_type = 'hermes'
+ AND runtime_type = 'shell'
+ )
+ AND NOT EXISTS (
+ SELECT 1
+ FROM system_image_settings
+ WHERE instance_type = 'hermes'
+ AND is_enabled = FALSE
+ );
---
apiVersion: v1
kind: PersistentVolume
@@ -912,6 +948,19 @@ spec:
---
apiVersion: v1
kind: Service
+metadata:
+ name: clawmanager-redis
+ namespace: clawmanager-system
+spec:
+ selector:
+ app: clawmanager-team-redis
+ ports:
+ - name: redis
+ port: 6379
+ targetPort: redis
+---
+apiVersion: v1
+kind: Service
metadata:
name: clawmanager-team-redis
namespace: clawmanager-system
@@ -924,12 +973,137 @@ spec:
targetPort: redis
---
apiVersion: v1
+kind: PersistentVolume
+metadata:
+ name: clawmanager-workspaces-pv
+spec:
+ capacity:
+ storage: 50Gi
+ volumeMode: Filesystem
+ accessModes:
+ - ReadWriteMany
+ persistentVolumeReclaimPolicy: Retain
+ storageClassName: manual
+ hostPath:
+ path: /data/clawmanager/system/workspaces
+ type: DirectoryOrCreate
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: clawmanager-workspaces
+ namespace: clawmanager-system
+spec:
+ accessModes:
+ - ReadWriteMany
+ resources:
+ requests:
+ storage: 50Gi
+ storageClassName: manual
+ volumeName: clawmanager-workspaces-pv
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: workspace-store
+ namespace: clawmanager-system
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: workspace-store
+ template:
+ metadata:
+ labels:
+ app: workspace-store
+ spec:
+ containers:
+ - name: nfs-server
+ image: itsthenetwork/nfs-server-alpine:12
+ imagePullPolicy: IfNotPresent
+ securityContext:
+ privileged: true
+ env:
+ - name: SHARED_DIRECTORY
+ value: /exports/workspaces
+ ports:
+ - name: nfs
+ containerPort: 2049
+ volumeMounts:
+ - name: workspaces
+ mountPath: /exports/workspaces
+ volumes:
+ - name: workspaces
+ persistentVolumeClaim:
+ claimName: clawmanager-workspaces
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: workspace-store
+ namespace: clawmanager-system
+spec:
+ selector:
+ app: workspace-store
+ ports:
+ - name: nfs
+ port: 2049
+ targetPort: nfs
+---
+apiVersion: v1
kind: ServiceAccount
metadata:
name: clawmanager-app
namespace: clawmanager-system
---
apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: clawmanager-runtime-manager
+rules:
+ - apiGroups:
+ - coordination.k8s.io
+ resources:
+ - leases
+ verbs:
+ - get
+ - create
+ - update
+ - apiGroups:
+ - apps
+ resources:
+ - deployments
+ - deployments/scale
+ verbs:
+ - get
+ - list
+ - watch
+ - create
+ - update
+ - patch
+ - apiGroups:
+ - ""
+ resources:
+ - pods
+ verbs:
+ - get
+ - list
+ - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: clawmanager-runtime-manager
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: clawmanager-runtime-manager
+subjects:
+ - kind: ServiceAccount
+ name: clawmanager-app
+ namespace: clawmanager-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: clawmanager-app-cluster-admin
@@ -1003,7 +1177,7 @@ metadata:
name: clawmanager-app
namespace: clawmanager-system
spec:
- replicas: 1
+ replicas: 3
selector:
matchLabels:
app: clawmanager-app
@@ -1077,10 +1251,47 @@ spec:
value: "/data/clawreef"
- name: CLAWMANAGER_WORKSPACE_ARCHIVE_MAX_MIB
value: "500"
+ - name: PLATFORM_REDIS_URL
+ value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0"
+ - name: TEAM_REDIS_URL
+ value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0"
- name: CLAWMANAGER_TEAM_REDIS_URL
- value: "redis://clawmanager-team-redis.clawmanager-system.svc.cluster.local:6379/0"
+ value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0"
- name: CLAWMANAGER_TEAM_MANAGER_BASE_URL
value: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001"
+ - name: RUNTIME_NAMESPACE
+ value: "clawmanager-system"
+ - name: RUNTIME_WORKSPACE_ROOT
+ value: "/workspaces"
+ - name: RUNTIME_WORKSPACE_NFS_SERVER
+ value: "workspace-store.clawmanager-system.svc.cluster.local"
+ - name: RUNTIME_WORKSPACE_NFS_PATH
+ value: "/"
+ - name: RUNTIME_MAX_GATEWAYS_PER_POD
+ value: "100"
+ - name: RUNTIME_GATEWAY_PORT_START
+ value: "20000"
+ - name: RUNTIME_GATEWAY_PORT_END
+ value: "20099"
+ - name: RUNTIME_SCHEDULER_ENABLED
+ value: "true"
+ - name: RUNTIME_HEARTBEAT_TIMEOUT
+ value: "10s"
+ - name: RUNTIME_AGENT_CONTROL_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: clawmanager-secrets
+ key: runtime-agent-control-token
+ - name: RUNTIME_AGENT_REPORT_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: clawmanager-secrets
+ key: runtime-agent-report-token
+ - name: OPENCLAW_GATEWAY_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: clawmanager-secrets
+ key: openclaw-gateway-token
- name: SKILL_SCANNER_ENABLED
value: "true"
- name: SKILL_SCANNER_BASE_URL
@@ -1095,6 +1306,8 @@ spec:
- name: tls-cert
mountPath: /var/run/clawreef-tls
readOnly: true
+ - name: workspaces
+ mountPath: /workspaces
readinessProbe:
httpGet:
scheme: HTTPS
@@ -1114,6 +1327,197 @@ spec:
secret:
secretName: clawmanager-tls
optional: true
+ - name: workspaces
+ nfs:
+ server: workspace-store.clawmanager-system.svc.cluster.local
+ path: /exports/workspaces
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: openclaw-runtime
+ namespace: clawmanager-system
+ labels:
+ app: openclaw-runtime
+ clawmanager.io/runtime-type: openclaw
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: openclaw-runtime
+ clawmanager.io/runtime-type: openclaw
+ template:
+ metadata:
+ labels:
+ app: openclaw-runtime
+ clawmanager.io/runtime-type: openclaw
+ spec:
+ containers:
+ - name: runtime
+ image: ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest
+ imagePullPolicy: Always
+ env:
+ - name: CLAWMANAGER_RUNTIME_TYPE
+ value: openclaw
+ - name: CLAWMANAGER_BACKEND_URL
+ value: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001"
+ - name: CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME
+ value: openclaw-runtime
+ - name: CLAWMANAGER_RUNTIME_IMAGE_REF
+ value: ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest
+ - name: RUNTIME_WORKSPACE_ROOT
+ value: /workspaces
+ - name: RUNTIME_GATEWAY_PORT_START
+ value: "20000"
+ - name: RUNTIME_GATEWAY_PORT_END
+ value: "20099"
+ - name: RUNTIME_AGENT_LISTEN_ADDR
+ value: "0.0.0.0:19090"
+ - name: RUNTIME_AGENT_PUBLIC_PORT
+ value: "19090"
+ - name: RUNTIME_GATEWAY_COMMAND
+ value: "/usr/local/bin/openclaw gateway run --allow-unconfigured --auth token --bind auto --force"
+ - name: OPENCLAW_GATEWAY_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: clawmanager-secrets
+ key: openclaw-gateway-token
+ - name: RUNTIME_AGENT_CONTROL_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: clawmanager-secrets
+ key: runtime-agent-control-token
+ - name: RUNTIME_AGENT_REPORT_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: clawmanager-secrets
+ key: runtime-agent-report-token
+ - name: POD_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ - name: POD_IP
+ valueFrom:
+ fieldRef:
+ fieldPath: status.podIP
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
+ ports:
+ - name: agent
+ containerPort: 19090
+ volumeMounts:
+ - name: workspaces
+ mountPath: /workspaces
+ volumes:
+ - name: workspaces
+ nfs:
+ server: workspace-store.clawmanager-system.svc.cluster.local
+ path: /exports/workspaces
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: hermes-runtime
+ namespace: clawmanager-system
+ labels:
+ app: hermes-runtime
+ clawmanager.io/runtime-type: hermes
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: hermes-runtime
+ clawmanager.io/runtime-type: hermes
+ template:
+ metadata:
+ labels:
+ app: hermes-runtime
+ clawmanager.io/runtime-type: hermes
+ spec:
+ containers:
+ - name: runtime
+ image: ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest
+ imagePullPolicy: IfNotPresent
+ env:
+ - name: CLAWMANAGER_RUNTIME_TYPE
+ value: hermes
+ - name: CLAWMANAGER_BACKEND_URL
+ value: "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001"
+ - name: CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME
+ value: hermes-runtime
+ - name: CLAWMANAGER_RUNTIME_IMAGE_REF
+ value: ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest
+ - name: CLAWMANAGER_AGENT_PORT
+ value: "19090"
+ - name: CLAWMANAGER_GATEWAY_PORT_START
+ value: "20000"
+ - name: CLAWMANAGER_GATEWAY_PORT_END
+ value: "20099"
+ - name: HERMES_TUI_DIR
+ value: /usr/local/lib/hermes-agent/ui-tui
+ - name: CLAWMANAGER_AGENT_CONTROL_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: clawmanager-secrets
+ key: runtime-agent-control-token
+ - name: CLAWMANAGER_AGENT_REPORT_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: clawmanager-secrets
+ key: runtime-agent-report-token
+ - name: RUNTIME_WORKSPACE_ROOT
+ value: /workspaces
+ - name: RUNTIME_AGENT_LISTEN_ADDR
+ value: "0.0.0.0:19090"
+ - name: RUNTIME_AGENT_PUBLIC_PORT
+ value: "19090"
+ - name: RUNTIME_GATEWAY_PORT_START
+ value: "20000"
+ - name: RUNTIME_GATEWAY_PORT_END
+ value: "20099"
+ - name: RUNTIME_AGENT_CONTROL_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: clawmanager-secrets
+ key: runtime-agent-control-token
+ - name: RUNTIME_AGENT_REPORT_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: clawmanager-secrets
+ key: runtime-agent-report-token
+ - name: POD_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ - name: POD_IP
+ valueFrom:
+ fieldRef:
+ fieldPath: status.podIP
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
+ ports:
+ - name: agent
+ containerPort: 19090
+ volumeMounts:
+ - name: workspaces
+ mountPath: /workspaces
+ volumes:
+ - name: workspaces
+ nfs:
+ server: workspace-store.clawmanager-system.svc.cluster.local
+ path: /exports/workspaces
---
apiVersion: v1
kind: Service
@@ -1128,7 +1532,7 @@ spec:
- name: https
port: 443
targetPort: https
- nodePort: 30443
+ nodePort: 39443
---
apiVersion: v1
kind: Service
diff --git a/deployments/nginx/nginx.conf b/deployments/nginx/nginx.conf
index c0d51fb..488d82a 100644
--- a/deployments/nginx/nginx.conf
+++ b/deployments/nginx/nginx.conf
@@ -11,6 +11,20 @@ http {
sendfile on;
tcp_nopush on;
tcp_nodelay on;
+ gzip on;
+ gzip_comp_level 6;
+ gzip_min_length 1024;
+ gzip_proxied any;
+ gzip_vary on;
+ gzip_types
+ application/javascript
+ application/json
+ application/xml
+ image/svg+xml
+ text/css
+ text/javascript
+ text/plain
+ text/xml;
keepalive_timeout 75;
keepalive_requests 10000;
server_tokens off;
@@ -53,6 +67,43 @@ http {
return 200 'ok';
}
+ location ^~ /assets/ {
+ try_files $uri =404;
+ access_log off;
+ expires 1y;
+ add_header Cache-Control "public, max-age=31536000, immutable";
+ }
+
+ location ~* ^/(?:lobster_logo|lobster_transparent|openclaw|hermes)\.png$ {
+ try_files $uri =404;
+ access_log off;
+ expires 30d;
+ add_header Cache-Control "public, max-age=2592000";
+ }
+
+ location ~* ^/vendor-icons/.+\.(?:png|jpg|jpeg|gif|webp|ico|svg)$ {
+ try_files $uri =404;
+ access_log off;
+ expires 30d;
+ add_header Cache-Control "public, max-age=2592000";
+ }
+
+ location ~ ^/s/ {
+ proxy_pass http://clawreef_backend;
+ proxy_http_version 1.1;
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection $connection_upgrade;
+ proxy_connect_timeout 15s;
+ proxy_read_timeout 3600s;
+ proxy_send_timeout 3600s;
+ proxy_buffering off;
+ proxy_request_buffering off;
+ }
+
location ~ ^/api/v1/instances/[0-9]+/proxy/? {
proxy_pass http://clawreef_backend;
proxy_http_version 1.1;
diff --git a/docs/agent-control-plane.md b/docs/agent-control-plane.md
index 91dcc00..5c04bbe 100644
--- a/docs/agent-control-plane.md
+++ b/docs/agent-control-plane.md
@@ -37,7 +37,9 @@ Examples of platform-driven runtime actions include:
## Related Guides
+- [ClawManager Agent V2 Development Guide](./clawmanager-agent-v2-development-guide.md)
- [Runtime Agent Integration Guide](./runtime-agent-integration-guide.md)
+- [Hermes Lite / Pro Agent Development Guide](./hermes-lite-pro-agent-development.md)
- [Hermes Runtime Image and Agent Development Guide](./hermes-runtime-agent-development.md)
- [Admin and User Guide](./admin-user-guide.md)
- [Resource Management Guide](./resource-management.md)
diff --git a/docs/agent-runtime-development-spec.md b/docs/agent-runtime-development-spec.md
new file mode 100644
index 0000000..dcfaf86
--- /dev/null
+++ b/docs/agent-runtime-development-spec.md
@@ -0,0 +1,512 @@
+# Agent Runtime 开发规范
+
+本文是 ClawManager 托管 runtime 的 agent 侧开发规范,适用于 OpenClaw、Hermes,以及后续新增的任意 runtime。开发新 runtime 时,应先满足本文的通用要求,再补充 runtime 自己的启动命令、配置文件和健康检查细节。
+
+相关协议和背景请同时参考:
+
+- `docs/clawmanager-agent-v2-development-guide.md`
+- `docs/clawmanager-agent-v2-contract.md`
+- `docs/runtime-agent-integration-guide.md`
+- `docs/hermes-lite-pro-agent-development.md`
+- `docs/hermes-runtime-agent-development.md`
+
+## 1. 范围和目标
+
+一个托管 runtime 镜像不能只启动业务进程。它必须内置一个常驻 runtime agent,由 agent 在 Pod 内负责:
+
+- Runtime Pod 注册、heartbeat、metrics、gateway 状态上报。
+- 接收 ClawManager 下发的 create、delete、drain、restart 等控制指令。
+- 管理本 Pod 内多个 gateway 子进程。
+- 管理端口池、进程、workspace、UID/GID、资源限制和健康检查。
+- 把 ClawManager 注入的 AI Gateway、实例 token、代理信任和 runtime 配置写入每个实例自己的 workspace。
+
+一句话原则:agent 是 Pod 内 gateway 管理器,不是同步启动脚本。
+
+## 2. 架构约束
+
+每个 Runtime Pod 内只能有一个 runtime agent 控制进程。一个用户实例对应一个 gateway 子进程和一个 workspace。
+
+```mermaid
+flowchart LR
+ CM["ClawManager Backend"]
+ POD["Runtime Pod"]
+ AG["runtime-agent"]
+ GW1["gateway: instance 101"]
+ GW2["gateway: instance 102"]
+ WS["/workspaces shared volume"]
+
+ CM -- "POST /v1/gateways" --> AG
+ CM -- "DELETE /v1/gateways/{id}" --> AG
+ CM -- "POST /v1/drain" --> AG
+ AG -- "register / heartbeat / metrics / gateways report" --> CM
+ AG --> GW1
+ AG --> GW2
+ GW1 --> WS
+ GW2 --> WS
+```
+
+浏览器只访问 ClawManager 的 HTTPS 入口。ClawManager 再反代到 Pod 内部 gateway。runtime agent 不应把每个 gateway 的自签 HTTPS 地址直接暴露给前端。
+
+## 3. 必须遵守的原则
+
+| 原则 | 要求 |
+| --- | --- |
+| 异步创建 | `POST /v1/gateways` 必须快速返回,通常 1 到 3 秒内返回 `status=starting`,不能阻塞等待业务 gateway 完全启动。 |
+| 幂等创建 | 以 `instance_id + generation` 为主要幂等键,重复请求必须返回同一个 gateway 元数据,不能重复启动进程。 |
+| 状态以真实进程为准 | `running` 只能在进程存在且健康检查通过时上报,不能只读旧缓存。 |
+| 端口统一管理 | 端口由 agent 本地分配器加锁分配,启动前检查监听占用,停止后释放。 |
+| 配置按 workspace 隔离 | 每个实例的 runtime 配置写入自己的 workspace,不能写全局配置。 |
+| 不覆盖用户配置 | 修改 runtime 配置必须做 JSON merge,保留已有字段,数组 append 后去重。 |
+| 不写死环境地址 | 不写死外部 IP、域名、CIDR,必须从 env 或 ClawManager 请求中读取。 |
+| 内部服务优先 | runtime 内部访问 ClawManager 使用 Kubernetes Service DNS,不使用浏览器访问的外部地址。 |
+| 时间必须可信 | token、设备签名、JWT 和 WebSocket 校验依赖时间,agent 必须检测时钟偏移,不能在明显偏移时报告健康。 |
+| 安全默认开启 | 控制接口校验 token,路径校验不能越权,命令执行不能拼 shell 字符串。 |
+
+## 4. Runtime Pod 环境变量
+
+agent 至少支持以下环境变量:
+
+| 变量 | 说明 |
+| --- | --- |
+| `CLAWMANAGER_RUNTIME_TYPE` | runtime 类型,例如 `openclaw`、`hermes`。 |
+| `RUNTIME_WORKSPACE_ROOT` | workspace 根目录,默认 `/workspaces`。 |
+| `RUNTIME_GATEWAY_PORT_START` | gateway 端口起始值,默认 `20000`。 |
+| `RUNTIME_GATEWAY_PORT_END` | gateway 端口结束值,默认 `20099`。 |
+| `RUNTIME_AGENT_CONTROL_TOKEN` | ClawManager 调 agent 控制接口使用。 |
+| `RUNTIME_AGENT_REPORT_TOKEN` | agent 上报 ClawManager 使用。 |
+| `CLAWMANAGER_BACKEND_URL` | ClawManager backend 内部地址,推荐 `http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001`。 |
+| `RUNTIME_AGENT_LISTEN_ADDR` | agent 控制接口监听地址,默认 `0.0.0.0:19090`。 |
+| `RUNTIME_AGENT_PUBLIC_PORT` | 注册给 ClawManager 的 agent 端口,默认 `19090`。 |
+| `CLAWMANAGER_RUNTIME_IMAGE_REF` | 当前 runtime 镜像标识。 |
+| `POD_NAME` / `POD_NAMESPACE` / `POD_IP` / `NODE_NAME` | 通过 Downward API 注入的 Pod 身份。 |
+| `CLAWMANAGER_TRUSTED_PROXY_CIDRS` | ClawManager app 或 gateway Pod 的来源网段,供 runtime 的 trusted proxy 配置使用。 |
+| `CLAWMANAGER_CONTROL_UI_ORIGIN` | ClawManager 内部 Control UI origin,默认可由 `CLAWMANAGER_BACKEND_URL` 推导。 |
+
+要求:
+
+- `CLAWMANAGER_BACKEND_URL` 必须是内部服务地址,不能使用 `https://172.16.1.12:39443` 这类浏览器外部入口。
+- 只有对浏览器生成页面 URL 时才使用外部入口。runtime gateway 的信任来源和 LLM base URL 都应使用内部服务 DNS。
+- agent 启动时要校验必填 env,缺失时上报 `unhealthy`,并在 `/v1/health` 返回 503。
+
+## 5. Gateway 创建协议
+
+ClawManager 调用:
+
+```http
+POST /v1/gateways
+X-ClawManager-Control-Token: ${RUNTIME_AGENT_CONTROL_TOKEN}
+Content-Type: application/json
+```
+
+请求字段以 `docs/clawmanager-agent-v2-contract.md` 为准,当前应支持:
+
+```json
+{
+ "instance_id": 123,
+ "user_id": 45,
+ "agent_type": "hermes",
+ "workspace_path": "/workspaces/hermes/user-45/instance-123",
+ "port_range": {
+ "start": 20000,
+ "end": 20099
+ },
+ "uid": 200123,
+ "gid": 200123,
+ "cpu_cores": 2,
+ "memory_mb": 4096,
+ "disk_quota_mb": 20480,
+ "generation": 7,
+ "environment": {
+ "CLAWMANAGER_LLM_BASE_URL": "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001/api/v1/gateway/llm",
+ "CLAWMANAGER_LLM_API_KEY": "instance-token",
+ "CLAWMANAGER_LLM_MODEL": "[\"auto\"]",
+ "CLAWMANAGER_LLM_PROVIDER": "openai-compatible",
+ "CLAWMANAGER_INSTANCE_TOKEN": "instance-token",
+ "OPENAI_API_KEY": "instance-token"
+ }
+}
+```
+
+成功响应必须快速返回:
+
+```json
+{
+ "gateway_id": "gw-123-7",
+ "instance_id": 123,
+ "port": 20017,
+ "pid": null,
+ "status": "starting",
+ "workspace_path": "/workspaces/hermes/user-45/instance-123"
+}
+```
+
+创建流程:
+
+1. 校验控制 token、`agent_type`、`instance_id`、`user_id`、`generation`、`workspace_path`、端口范围。
+2. 检查当前 Pod 是否 draining。draining 时返回 409 或 503。
+3. 按 `instance_id + generation` 查询本地状态。已存在 `starting` 或 `running` 时直接返回已有元数据。
+4. 若收到更高 generation,停止并释放同实例旧 generation gateway。
+5. 加锁分配端口,先标记 reserved。
+6. 创建 workspace 和 home 目录,设置 owner 为请求中的 `uid/gid`。
+7. 写入或合并 runtime 配置。
+8. 记录 gateway 元数据为 `starting`。
+9. 后台启动进程,立即返回。
+10. 后台健康检查通过后,上报 `/api/v1/runtime-agent/gateways/report`,状态为 `running`。
+11. 启动失败或健康检查失败时,上报 `error`,释放端口,并保留错误信息。
+
+禁止事项:
+
+- 禁止在 HTTP handler 里直接运行前台命令并等待它退出。
+- 禁止同一个 create 请求触发多个 gateway 进程。
+- 禁止把请求里的任意字符串拼接成 shell 命令。
+- 禁止创建不在端口范围内的监听端口。
+
+## 6. 端口和资源管理
+
+agent 必须实现并发安全的端口分配器:
+
+```text
+reserve(instance_id, generation, count) -> port_set
+commit(instance_id, generation, port_set)
+release(instance_id, generation)
+list_used()
+```
+
+要求:
+
+- 分配时持有互斥锁。
+- 在锁内检查 agent 内存状态和系统监听端口。
+- runtime 如果需要主端口、control 端口、browser 端口或 sidecar 端口,必须一次分配一组端口。
+- 启动失败、健康检查失败、DELETE 成功后必须释放端口。
+- `used_slots` 来自真实 `starting` 和健康 `running` gateway 数量,不能来自旧缓存。
+
+资源隔离要求:
+
+- 每个 gateway 使用独立进程组。
+- 业务进程最终以请求里的 `uid/gid` 运行。
+- CPU、Memory 使用 cgroup 限制。
+- Disk 使用 filesystem quota 或周期扫描加错误上报。
+- 删除 gateway 不删除 workspace。
+
+## 7. Workspace 和配置写入
+
+workspace 固定格式:
+
+```text
+/workspaces/{runtime}/user-{user_id}/instance-{instance_id}
+```
+
+agent 必须:
+
+- 对 workspace root 和请求路径执行 realpath。
+- 拒绝 `..`、符号链接逃逸、跨用户或跨实例路径。
+- 创建 `${workspace}/home`,并设置 `HOME=${workspace}/home`。
+- 密钥、token、provider 配置文件权限建议 `0600`。
+- owner 设置为该 gateway 的 `uid/gid`。
+
+runtime 配置必须写在 workspace 内。例如:
+
+| Runtime | 推荐配置路径 |
+| --- | --- |
+| OpenClaw | `{workspace}/home/.openclaw/openclaw.json` |
+| Hermes | `{workspace}/home/.hermes/hermes.json` 或 Hermes 原生配置路径 |
+
+JSON merge 规则:
+
+- 对象字段递归 merge。
+- 已存在字段默认保留。
+- 平台必须控制的字段可以覆盖,但要在代码中显式列出。
+- 数组字段 append 后去重,例如 `allowedOrigins`。
+- 不删除用户已有的未知字段。
+- 写入前先生成临时文件,再原子 rename。
+
+## 8. AI Gateway 和模型凭证注入
+
+ClawManager 会在 `POST /v1/gateways` 的 `environment` 字段中下发实例级 LLM 环境变量。agent 必须把这些变量传给 gateway 子进程,并按 runtime 需要写入配置文件。
+
+允许转发给子进程的变量应使用白名单:
+
+```text
+CLAWMANAGER_LLM_BASE_URL
+CLAWMANAGER_LLM_API_KEY
+CLAWMANAGER_LLM_MODEL
+CLAWMANAGER_LLM_PROVIDER
+CLAWMANAGER_INSTANCE_TOKEN
+OPENAI_BASE_URL
+OPENAI_API_BASE
+OPENAI_API_KEY
+OPENAI_MODEL
+```
+
+要求:
+
+- `CLAWMANAGER_LLM_BASE_URL` 使用 ClawManager 内部服务 DNS。
+- `CLAWMANAGER_LLM_API_KEY` 和 `OPENAI_API_KEY` 使用当前实例 token。
+- 不能要求用户在 runtime 页面手工填写 OpenAI key。
+- 不能把 API key、实例 token、bootstrap token 打到日志。
+- 如果 `environment` 里缺少 API key,agent 不应报告 gateway `running`,应上报 `error`,错误消息明确说明缺少实例 LLM token。
+
+Hermes 落地时应把上述 env 转换成 Hermes 的 provider 配置,推荐语义:
+
+```json
+{
+ "models": {
+ "providers": {
+ "auto": {
+ "type": "openai-compatible",
+ "baseUrl": "${CLAWMANAGER_LLM_BASE_URL}",
+ "apiKey": "${CLAWMANAGER_LLM_API_KEY}"
+ }
+ },
+ "primary": "auto/auto"
+ }
+}
+```
+
+如果 Hermes 使用不同配置 schema,也必须满足同样的安全和可用性要求:通过 ClawManager AI Gateway 访问模型,不在镜像内写死厂商 key。
+
+## 9. 代理信任和浏览器访问
+
+runtime gateway 在 Pod 内部推荐使用 HTTP。浏览器访问链路应为:
+
+```text
+Browser HTTPS -> ClawManager HTTPS -> ClawManager backend proxy -> Runtime Pod HTTP gateway
+```
+
+agent 写入 runtime 配置时必须配置:
+
+- Control UI 的 `basePath` 为 `/api/v1/instances/{instance_id}/proxy`。
+- 允许的 origin 使用 ClawManager 内部 origin,例如 `http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001`。
+- trusted proxies 使用 `CLAWMANAGER_TRUSTED_PROXY_CIDRS` 或等价 env,不写死 `100.68.x.x`。
+- 不使用外部地址 `https://172.16.1.12:39443` 作为 runtime 内部信任来源。
+
+OpenClaw 参考配置:
+
+```json
+{
+ "gateway": {
+ "auth": {
+ "mode": "trusted-proxy"
+ },
+ "controlUi": {
+ "allowedOrigins": [
+ "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001"
+ ],
+ "basePath": "/api/v1/instances/123/proxy",
+ "dangerouslyDisableDeviceAuth": true
+ },
+ "trustedProxies": [
+ "100.68.0.0/16"
+ ]
+ }
+}
+```
+
+说明:
+
+- `dangerouslyDisableDeviceAuth` 只能在 ClawManager trusted proxy 模式下使用。
+- 如果 runtime 不支持 trusted proxy 模式,必须明确实现一种等价机制,保证浏览器通过 ClawManager 代理即可自动进入,不需要用户手工粘贴 gateway token。
+- 如果出现 `origin not allowed`,优先检查 `allowedOrigins` 是否使用内部 origin,以及 WebSocket 代理是否正确传递 Origin。
+- 如果出现 `Proxy headers detected from untrusted address`,优先检查 trusted proxies CIDR。
+
+## 10. 时间同步规范
+
+runtime 侧不能忽略时间问题。以下问题通常与时钟偏移有关:
+
+- `device signature expired`
+- token 未过期但被拒绝。
+- JWT、签名 URL、WebSocket 鉴权异常。
+- ClawManager 显示状态和 gateway 实际状态不一致。
+
+agent 必须实现:
+
+- 注册和 heartbeat 响应如果带 `server_time`,记录本地时间与服务端时间偏移。
+- 若偏移超过 30 秒,在日志和 health 中上报 `clock_skew_warning`。
+- 若偏移超过 120 秒,不应报告 `ready` 或 gateway `running`,应上报 `unhealthy` 或 `error`。
+- 生成或校验任何带时间的签名时使用 UTC。
+- 不在容器里自行修改系统时间。时间同步由 Kubernetes 节点的 NTP、chrony 或宿主机配置负责。
+
+运维要求:
+
+- 所有 Kubernetes 节点必须启用 NTP 或 chrony。
+- 发现签名过期类错误时,先比较浏览器机器、ClawManager Pod、runtime Pod、Kubernetes Node 的 UTC 时间。
+- agent 健康检查输出应包含当前本地时间、最近服务端时间和估算偏移,便于排查。
+
+## 11. 状态上报和恢复语义
+
+Runtime Pod 注册 payload 必须包含容量:
+
+```json
+{
+ "runtime_type": "hermes",
+ "capacity": 100,
+ "used_slots": 12,
+ "available_slots": 88,
+ "state": "ready",
+ "draining": false
+}
+```
+
+Gateway report 每个 gateway 建议包含:
+
+```json
+{
+ "instance_id": 66,
+ "gateway_id": "gw-66-7",
+ "gateway_port": 20000,
+ "gateway_pid": 1234,
+ "state": "running",
+ "generation": 7,
+ "workspace_path": "/workspaces/hermes/user-1/instance-66",
+ "started_at": "2026-06-03T06:00:00Z",
+ "updated_at": "2026-06-03T06:00:10Z",
+ "error_message": ""
+}
+```
+
+状态规则:
+
+- `starting`:进程准备中或健康检查尚未通过。
+- `running`:进程存在、端口监听、HTTP 健康检查通过、必要的代理和 LLM 检查通过。
+- `error`:启动失败、配置失败、健康检查失败、凭证缺失或端口冲突。
+- `stopped`:进程已停止,端口已释放。
+- `unhealthy`:进程存在但健康检查失败。
+
+Pod 重启或升级时:
+
+- agent 启动后先初始化 workspace root、端口池、控制服务和报告客户端。
+- 控制服务可用后注册 Runtime Pod。
+- 只有初始化完成后才能上报 `ready`。
+- 不要把旧 Pod 的 gateway 当成运行中,除非进程确实存在且健康。
+- 收到 SIGTERM 时进入 draining,拒绝新 create,尽量上报最后状态。
+- 支持 ClawManager 下发 drain、stop、restart gateway 指令。
+
+## 12. 健康检查
+
+`/v1/health` 只有在 agent 能接受控制指令时才返回 2xx。以下情况必须返回 503:
+
+- 必填环境变量缺失。
+- workspace root 不可访问。
+- 端口池不可用。
+- report token 或 control token 缺失。
+- 时钟偏移超过健康阈值。
+- agent 正在关闭且不再接受请求。
+
+gateway 从 `starting` 转为 `running` 前至少校验:
+
+- 端口已监听。
+- HTTP `/` 或 runtime 约定健康端点可访问。
+- 代理 base path 配置已写入。
+- 使用 ClawManager 内部 Origin 的 Control UI 或 WebSocket 握手不再返回 `origin not allowed`。
+- runtime 如果需要 LLM,使用实例 token 调用 ClawManager AI Gateway 可以返回模型列表或完成一次轻量探测。
+
+健康检查失败时不要上报 `running`。应上报 `error`,并带短错误文本。
+
+## 13. 安全规范
+
+agent 控制接口必须校验:
+
+- `X-ClawManager-Control-Token`。
+- `instance_id`、`user_id`、`generation` 类型和值范围。
+- `agent_type` 必须等于当前 Pod runtime type。
+- `workspace_path` 不越权。
+- 端口只能来自允许范围。
+- `environment` 只允许白名单 key。
+
+进程启动必须使用 argv 数组,不使用 shell 拼接:
+
+```text
+execve(binary, ["hermes", "gateway", "run", "--port", "20017"], env)
+```
+
+禁止:
+
+- 把请求参数拼成 `sh -c`。
+- 日志打印 token、API key、channel secret。
+- 允许用户覆盖 agent 自己的控制 token 和上报 token。
+- 删除 gateway 时顺手删除 workspace。
+- 为了调通临时写死 IP、CIDR 或 `allowedOrigins=["*"]`,除非明确是本地开发环境。
+
+## 14. Hermes Runtime 开发落地清单
+
+开发 Hermes runtime 时,至少完成以下事项:
+
+| 项目 | 要求 |
+| --- | --- |
+| Runtime 类型 | `CLAWMANAGER_RUNTIME_TYPE=hermes`,注册和上报 payload 中都使用 `hermes`。 |
+| Agent 控制服务 | 监听 `0.0.0.0:19090`,实现 `/v1/health`、`/v1/gateways`、`DELETE /v1/gateways/{id}`、`/v1/drain`。 |
+| Gateway 启动 | 后台启动 Hermes gateway,使用实例 workspace、端口、UID/GID、HOME。 |
+| 配置文件 | 写入 `{workspace}/home/.hermes/hermes.json` 或 Hermes 原生配置,必须 JSON merge。 |
+| AI Gateway | 将 `CLAWMANAGER_LLM_*` 和 `OPENAI_*` 注入 Hermes provider 配置。 |
+| 代理路径 | Hermes UI 或 WebSocket 必须支持 `/api/v1/instances/{instance_id}/proxy` base path。 |
+| Trusted proxy | 信任 ClawManager 内部 proxy 来源,不信任任意外部来源。 |
+| 自动登录 | 浏览器通过 ClawManager proxy 进入时不要求手工粘贴 token 或重复确认自签证书。 |
+| 端口组 | 如果 Hermes 除主端口外还有 control/browser/sidecar 端口,按端口组分配。 |
+| Skills | skill inventory 和状态上报按 `instance_id`、`workspace_path` 分组。 |
+| 时间 | 实现 clock skew 检测;偏移过大时不报告 ready/running。 |
+| 验收 | 100 个实例并发创建无端口冲突,Pod drain 和重启后状态正确。 |
+
+Hermes 启动命令由 Hermes 项目确定,但 agent 必须保证:
+
+- 命令参数固定且可审计。
+- 端口和 workspace 只能来自 agent 校验后的值。
+- LLM、proxy、auth 配置在进程启动前已经写入。
+- `POST /v1/gateways` 返回时进程可以仍在启动,但 metadata 必须已经持久化。
+
+## 15. 测试和验收
+
+单元测试至少覆盖:
+
+- create gateway 幂等。
+- 高 generation 替换低 generation。
+- 端口分配并发无冲突。
+- 端口被系统占用时跳过。
+- workspace path 越权被拒绝。
+- JSON merge 保留用户字段,数组去重。
+- LLM env 白名单转发。
+- 缺少 LLM token 时 gateway 不报告 running。
+- trusted proxy 和 allowed origin 配置生成。
+- clock skew 阈值判断。
+
+集成测试至少覆盖:
+
+- Runtime Pod 注册为 `ready`,capacity、used_slots、available_slots 正确。
+- 创建 100 个 gateway,端口唯一,状态最终为 running。
+- 重复创建同一个 `instance_id + generation` 不重复启动进程。
+- ClawManager 反代 WebSocket 不出现 `origin not allowed`。
+- 通过 ClawManager proxy 进入 runtime 不要求手工粘贴 gateway token。
+- runtime 内模型调用经过 ClawManager AI Gateway,不出现 `No API key found`。
+- Pod drain 后拒绝新 create,已有 gateway 持续上报,DELETE 后释放端口。
+- Pod 重启后不误报旧 gateway running。
+
+上线前 Kubernetes 验证建议:
+
+```bash
+kubectl -n clawmanager-system get pods -l app.kubernetes.io/component=runtime-agent -o wide
+kubectl -n clawmanager-system logs deploy/hermes-runtime --tail=200
+kubectl -n clawmanager-system exec deploy/hermes-runtime -- curl -fsS http://127.0.0.1:19090/v1/health
+kubectl -n clawmanager-system exec deploy/hermes-runtime -- date -u
+kubectl -n clawmanager-system exec deploy/clawmanager-app -- date -u
+```
+
+如果出现认证或签名问题,排查顺序:
+
+1. 时间偏移。
+2. 实例 token 是否下发到 `environment`。
+3. runtime 配置文件是否包含 provider 和 API key。
+4. ClawManager 内部 LLM base URL 是否可达。
+5. allowed origins 和 trusted proxies 是否使用内部服务地址和 env CIDR。
+6. gateway 是否真正健康,而不是缓存状态误报。
+
+## 16. Do / Don't
+
+| Do | Don't |
+| --- | --- |
+| 用内部 Kubernetes Service DNS 配置 runtime 到 ClawManager 的访问。 | 写死 `172.16.1.12`、NodePort、外部 HTTPS 入口。 |
+| `POST /v1/gateways` 快速返回 `starting`,后台拉起进程。 | 在 HTTP 请求里阻塞等待 `gateway run` 前台进程。 |
+| 按 `instance_id + generation` 幂等。 | 重试时重复启动多个 gateway。 |
+| 上报状态前检查真实进程和健康。 | 只读旧状态文件就上报 `running`。 |
+| JSON merge runtime 配置。 | 覆盖用户的整个配置文件。 |
+| 白名单转发 env。 | 把用户请求里的所有 env 原样传给进程。 |
+| 使用 trusted proxy 让浏览器通过 ClawManager 自动进入。 | 要求用户手工粘贴 gateway token。 |
+| 检测并上报 clock skew。 | 忽略时间,导致 `device signature expired` 反复出现。 |
diff --git a/docs/clawmanager-agent-v2-contract.md b/docs/clawmanager-agent-v2-contract.md
new file mode 100644
index 0000000..8879cd2
--- /dev/null
+++ b/docs/clawmanager-agent-v2-contract.md
@@ -0,0 +1,178 @@
+# clawmanager-agent V2 Contract
+
+## Purpose
+
+The agent runs inside each shared OpenClaw or Hermes runtime Pod. It manages gateway subprocesses, ports, cgroups, Linux users, workspace quota, health checks, skill/status reporting, and metrics.
+
+## Agent-To-Backend Reports
+
+All report requests use header `X-ClawManager-Agent-Token`.
+
+### POST /api/v1/runtime-agent/register
+
+Request body:
+
+```json
+{
+ "runtime_type": "openclaw",
+ "namespace": "clawmanager-system",
+ "pod_name": "openclaw-runtime-6f77f8b8c7-abcde",
+ "pod_uid": "pod-uid",
+ "pod_ip": "10.42.0.31",
+ "node_name": "node-a",
+ "deployment_name": "openclaw-runtime",
+ "image_ref": "ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest",
+ "agent_endpoint": "http://10.42.0.31:19090",
+ "capacity": 100
+}
+```
+
+### POST /api/v1/runtime-agent/heartbeat
+
+Request body:
+
+```json
+{
+ "pod_name": "openclaw-runtime-6f77f8b8c7-abcde",
+ "runtime_type": "openclaw",
+ "state": "ready",
+ "used_slots": 37,
+ "draining": false
+}
+```
+
+### POST /api/v1/runtime-agent/metrics/report
+
+Request body:
+
+```json
+{
+ "pod_name": "openclaw-runtime-6f77f8b8c7-abcde",
+ "cpu_millis_used": 13600,
+ "memory_bytes_used": 42949672960,
+ "disk_bytes_used": 214748364800,
+ "network_rx_bytes": 9223372,
+ "network_tx_bytes": 19223372,
+ "gateways": [
+ {
+ "instance_id": 123,
+ "gateway_id": "gw-123-7",
+ "port": 20017,
+ "state": "running",
+ "last_health_at": "2026-06-01T10:00:00Z"
+ }
+ ]
+}
+```
+
+## Backend-To-Agent Commands
+
+All command requests use header `X-ClawManager-Control-Token`.
+
+ClawManager treats redirects from agent command endpoints as errors. Agents should return direct 2xx, 4xx, or 5xx responses instead of 3xx redirects.
+
+### GET /v1/health
+
+The agent reports whether it can accept control-plane commands.
+
+Success status: HTTP 200.
+
+Success response body:
+
+```json
+{
+ "status": "ready"
+}
+```
+
+ClawManager currently treats any HTTP 2xx response as success and does not require a response body for this endpoint. Any non-2xx status is treated as an error.
+
+### POST /v1/gateways
+
+The agent creates a gateway subprocess and returns the bound port. If no port is free in the requested range, return HTTP 409.
+
+Request body:
+
+```json
+{
+ "instance_id": 123,
+ "user_id": 45,
+ "agent_type": "openclaw",
+ "workspace_path": "/workspaces/openclaw/user-45/instance-123",
+ "port_range": {
+ "start": 20000,
+ "end": 20099
+ },
+ "uid": 200123,
+ "gid": 200123,
+ "cpu_cores": 2,
+ "memory_mb": 4096,
+ "disk_quota_mb": 20480,
+ "generation": 7
+}
+```
+
+Success status: HTTP 200 or HTTP 201.
+
+Success response body:
+
+```json
+{
+ "gateway_id": "gw-123-7",
+ "port": 20017,
+ "pid": 8842,
+ "status": "running"
+}
+```
+
+### DELETE /v1/gateways/{gateway_id}
+
+The agent stops the gateway and releases cgroup, process, and port resources.
+
+`gateway_id` is URL path-escaped by ClawManager. Agents must decode the path segment before lookup.
+
+Success status: HTTP 200, HTTP 202, or HTTP 204. ClawManager does not require a response body.
+
+### POST /v1/drain
+
+The agent marks the Pod as draining and rejects new gateway creation. Existing gateways continue until ClawManager migrates them.
+
+Request body:
+
+```json
+{
+ "draining": true
+}
+```
+
+Success status: HTTP 200 or HTTP 202. ClawManager does not require a response body.
+
+## Backend-To-Agent Error Responses
+
+For now, agents should return a plain text response body for command errors.
+
+When no port is free for `POST /v1/gateways`, return:
+
+```http
+HTTP/1.1 409 Conflict
+Content-Type: text/plain
+
+no free port
+```
+
+ClawManager maps this to `runtime agent conflict: no free port`.
+
+For general failures, return an appropriate non-2xx status with a short plain text body:
+
+```http
+HTTP/1.1 500 Internal Server Error
+Content-Type: text/plain
+
+failed to start gateway process
+```
+
+ClawManager includes both the status code and response body in the returned error.
+
+## Isolation Requirements
+
+Every gateway runs as UID/GID `200000 + instance_id`. The agent rejects workspace paths outside `/workspaces/{runtime}/user-{user_id}/instance-{instance_id}` after resolving symlinks. CPU and memory limits are enforced with cgroups. Workspace quota is enforced before writes.
diff --git a/docs/clawmanager-agent-v2-development-guide.md b/docs/clawmanager-agent-v2-development-guide.md
new file mode 100644
index 0000000..ec62088
--- /dev/null
+++ b/docs/clawmanager-agent-v2-development-guide.md
@@ -0,0 +1,888 @@
+# ClawManager Agent V2 开发规范
+
+本文整理当前 ClawManager 代码中的最新 agent 约定,面向 OpenClaw、Hermes 以及后续新增的托管 runtime。开发或改造 runtime 镜像时,应优先遵守本文;字段级契约以 `docs/clawmanager-agent-v2-contract.md` 和后端代码为准。
+
+## 1. 架构定位
+
+最新架构中有两类 agent 通信面:
+
+1. **Runtime Pod Agent,也就是本文主线的 V2 agent**
+ 运行在共享 runtime Pod 内,负责本 Pod 内多个 gateway 子进程的创建、删除、端口、workspace、资源隔离、健康检查和状态上报。ClawManager 通过它把一个实例绑定到 `pod_ip + gateway_port`。
+
+2. **Instance Agent Control Plane,旧的 `/api/v1/agent/*` 协议**
+ 面向单个实例的状态、命令、skill inventory、skill package 上传和配置 revision 下载。当前后端仍保留这套协议,runtime 可以继续复用它完成实例级状态与 skill 管理;但 gateway 生命周期管理必须走 Runtime Pod Agent。
+
+一句话原则:**Runtime Pod Agent 是 Pod 内 gateway 管理器,不是一个阻塞式启动脚本。**
+
+## 2. 支持范围
+
+当前 V2 托管 runtime 类型:
+
+| 类型 | 说明 |
+| --- | --- |
+| `openclaw` | OpenClaw runtime |
+| `hermes` | Hermes runtime |
+
+当前平台默认值:
+
+| 项 | 默认值 |
+| --- | --- |
+| workspace root | `/workspaces` |
+| workspace path | `/workspaces/{runtime}/user-{user_id}/instance-{instance_id}` |
+| gateway 端口范围 | `20000-20099` |
+| 单 Runtime Pod 容量 | `100` |
+| 实例 UID/GID | `200000 + instance_id` |
+| agent 控制端口 | `19090` |
+
+## 3. Runtime Pod Agent 职责
+
+Runtime Pod Agent 必须实现:
+
+- 启动本地控制服务:`GET /v1/health`、`POST /v1/gateways`、`DELETE /v1/gateways/{gateway_id}`、`POST /v1/drain`。
+- 向 ClawManager 上报 runtime pod 注册、heartbeat、metrics、gateway 状态和 skill 状态。
+- 管理本 Pod 内 gateway 子进程生命周期。
+- 管理端口池,避免同一 Pod 内端口冲突。
+- 创建并校验 workspace,设置正确 UID/GID。
+- 将 ClawManager 下发的 LLM、代理、实例 token、runtime 配置写入实例自己的 workspace。
+- 实现 drain,拒绝新 gateway,保持已有 gateway 运行,等待 ClawManager 迁移和删除。
+- 避免任何敏感 token、API key、Authorization header 出现在日志中。
+
+不应该由 Runtime Pod Agent 实现:
+
+- 直接对浏览器暴露 gateway 自签地址。
+- 管理 ClawManager 用户权限。
+- 删除用户 workspace。
+- 把外部 NodePort、外部 HTTPS 入口或固定内网 IP 写死进 runtime 配置。
+
+## 4. 运行环境变量
+
+当前 Runtime Deployment 构建器注入以下变量,agent 实现应优先支持:
+
+| 变量 | 说明 |
+| --- | --- |
+| `CLAWMANAGER_RUNTIME_TYPE` | `openclaw` 或 `hermes` |
+| `CLAWMANAGER_AGENT_PORT` | agent 控制端口,当前为 `19090` |
+| `CLAWMANAGER_GATEWAY_PORT_START` | gateway 起始端口 |
+| `CLAWMANAGER_GATEWAY_PORT_END` | gateway 结束端口 |
+| `CLAWMANAGER_AGENT_CONTROL_TOKEN` | ClawManager 调 agent 控制接口使用 |
+| `CLAWMANAGER_AGENT_REPORT_TOKEN` | agent 上报 ClawManager 使用 |
+
+为了兼容早期文档和旧镜像,agent 建议同时识别以下别名:
+
+| 新变量 | 兼容别名 |
+| --- | --- |
+| `CLAWMANAGER_AGENT_CONTROL_TOKEN` | `RUNTIME_AGENT_CONTROL_TOKEN` |
+| `CLAWMANAGER_AGENT_REPORT_TOKEN` | `RUNTIME_AGENT_REPORT_TOKEN` |
+| `CLAWMANAGER_GATEWAY_PORT_START` | `RUNTIME_GATEWAY_PORT_START` |
+| `CLAWMANAGER_GATEWAY_PORT_END` | `RUNTIME_GATEWAY_PORT_END` |
+| `CLAWMANAGER_AGENT_PORT` | `RUNTIME_AGENT_PUBLIC_PORT` |
+
+建议支持的附加变量:
+
+| 变量 | 默认值/要求 |
+| --- | --- |
+| `RUNTIME_WORKSPACE_ROOT` | 默认 `/workspaces` |
+| `RUNTIME_AGENT_LISTEN_ADDR` | 默认 `0.0.0.0:19090` |
+| `CLAWMANAGER_BACKEND_URL` | ClawManager backend 内部地址,推荐 Kubernetes Service DNS |
+| `CLAWMANAGER_RUNTIME_IMAGE_REF` | 当前镜像标识,用于注册上报 |
+| `POD_NAME` / `POD_NAMESPACE` / `POD_IP` / `NODE_NAME` | 建议通过 Downward API 注入 |
+| `CLAWMANAGER_TRUSTED_PROXY_CIDRS` | ClawManager app/gateway 的可信代理网段 |
+
+`CLAWMANAGER_BACKEND_URL` 必须使用集群内部地址,例如:
+
+```text
+http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001
+```
+
+不要在 runtime 内部配置中写入浏览器外部入口,例如 `https://172.16.1.12:39443`。
+
+## 5. Agent 到 ClawManager 的上报接口
+
+所有 Runtime Pod Agent 上报请求必须带:
+
+```http
+X-ClawManager-Agent-Token: ${CLAWMANAGER_AGENT_REPORT_TOKEN}
+Content-Type: application/json
+```
+
+后端统一返回:
+
+```json
+{
+ "success": true,
+ "message": "...",
+ "data": {}
+}
+```
+
+### 5.1 注册 Runtime Pod
+
+```http
+POST {CLAWMANAGER_BACKEND_URL}/api/v1/runtime-agent/register
+```
+
+必填字段:
+
+- `runtime_type`
+- `namespace`
+- `pod_name`
+- `deployment_name`
+- `image_ref`
+
+推荐完整 payload:
+
+```json
+{
+ "runtime_type": "openclaw",
+ "namespace": "clawmanager-system",
+ "pod_name": "openclaw-runtime-6f77f8b8c7-abcde",
+ "pod_uid": "pod-uid",
+ "pod_ip": "10.42.0.31",
+ "node_name": "node-a",
+ "deployment_name": "openclaw-runtime",
+ "image_ref": "ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest",
+ "agent_endpoint": "http://10.42.0.31:19090",
+ "state": "ready",
+ "capacity": 100,
+ "used_slots": 0,
+ "draining": false,
+ "metrics": {
+ "agent_version": "0.1.0"
+ },
+ "reported_at": "2026-06-08T09:00:00Z"
+}
+```
+
+要求:
+
+- `agent_endpoint` 必须是 ClawManager Pod 可以访问的地址,通常是 `http://{POD_IP}:19090`,不能是 `127.0.0.1`。
+- `capacity <= 0` 时后端会按 `100` 处理,但 agent 应主动上报真实容量。
+- `state` 为空时后端按 `ready` 处理;agent 不应在初始化未完成时上报 ready。
+- `metrics` 必须是合法 JSON。
+
+### 5.2 Heartbeat
+
+```http
+POST {CLAWMANAGER_BACKEND_URL}/api/v1/runtime-agent/heartbeat
+```
+
+payload:
+
+```json
+{
+ "pod_id": 17,
+ "namespace": "clawmanager-system",
+ "pod_name": "openclaw-runtime-6f77f8b8c7-abcde",
+ "state": "ready",
+ "used_slots": 37,
+ "draining": false,
+ "reported_at": "2026-06-08T09:00:02Z"
+}
+```
+
+要求:
+
+- 优先使用注册返回的 `data.pod.id` 作为 `pod_id`。
+- 没有 `pod_id` 时必须带 `namespace + pod_name`。
+- `state` 必填。
+- `used_slots` 必须来自当前真实 `starting/running` gateway 数量,不要使用过期缓存。
+- 建议 heartbeat 间隔 2 秒;如果环境压力较大,可放宽,但必须小于 ClawManager 的 heartbeat 超时窗口。
+
+### 5.3 Metrics
+
+```http
+POST {CLAWMANAGER_BACKEND_URL}/api/v1/runtime-agent/metrics/report
+```
+
+payload:
+
+```json
+{
+ "pod_id": 17,
+ "cpu_millis_used": 13600,
+ "memory_bytes_used": 42949672960,
+ "disk_bytes_used": 214748364800,
+ "network_rx_bytes": 9223372,
+ "network_tx_bytes": 19223372,
+ "metrics": {
+ "gateway_count": 37,
+ "load_1m": 2.4
+ },
+ "reported_at": "2026-06-08T09:00:05Z"
+}
+```
+
+要求:
+
+- CPU 单位是 millicore。
+- memory/disk/network 单位是 bytes。
+- network 字段必须是单调递增 counter,不要填瞬时速率。
+- `metrics` 是扩展 JSON,必须合法。
+
+### 5.4 Gateway Report
+
+```http
+POST {CLAWMANAGER_BACKEND_URL}/api/v1/runtime-agent/gateways/report
+```
+
+payload:
+
+```json
+{
+ "pod_id": 17,
+ "gateways": [
+ {
+ "instance_id": 123,
+ "gateway_id": "gw-123-7",
+ "gateway_port": 20017,
+ "gateway_pid": 8842,
+ "state": "running",
+ "generation": 7,
+ "health_at": "2026-06-08T09:00:05Z"
+ }
+ ]
+}
+```
+
+要求:
+
+- `gateways` 必填。
+- 每个 gateway 的 `instance_id`、`state`、`generation` 必填。
+- 后端只接受“当前绑定 Pod + 当前 generation”的上报;旧 Pod 或旧 generation 上报会被忽略。
+- `state=running` 或 `state=healthy` 会把绑定更新为 running。
+- `starting`、`stopped`、`error`、`unhealthy` 等状态会更新绑定状态。
+- 错误状态必须带 `error_message`,不要只上报空状态。
+
+### 5.5 Skills Report
+
+```http
+POST {CLAWMANAGER_BACKEND_URL}/api/v1/runtime-agent/skills/report
+```
+
+当前后端接收并发布任意 JSON payload。推荐按实例分组:
+
+```json
+{
+ "pod_id": 17,
+ "runtime_type": "openclaw",
+ "mode": "full",
+ "reported_at": "2026-06-08T09:00:05Z",
+ "instances": [
+ {
+ "instance_id": 123,
+ "workspace_path": "/workspaces/openclaw/user-45/instance-123",
+ "skills": [
+ {
+ "skill_id": "weather",
+ "skill_version": "1.0.0",
+ "identifier": "weather",
+ "install_path": "/workspaces/openclaw/user-45/instance-123/.openclaw/skills/weather",
+ "content_md5": "0123456789abcdef0123456789abcdef",
+ "source": "runtime",
+ "type": "agent-skill"
+ }
+ ]
+ }
+ ]
+}
+```
+
+skill inventory 必须按 `instance_id` 和 `workspace_path` 隔离,不要把一个实例的 skill 混到另一个实例。
+
+## 6. ClawManager 到 Agent 的控制接口
+
+所有控制请求必须校验:
+
+```http
+X-ClawManager-Control-Token: ${CLAWMANAGER_AGENT_CONTROL_TOKEN}
+```
+
+token 错误返回 `401`。不要返回 `3xx` redirect;后端会把 redirect 当成失败。
+
+### 6.1 Health
+
+```http
+GET /v1/health
+```
+
+任意 HTTP 2xx 都会被后端视为成功。建议响应:
+
+```json
+{
+ "status": "ready"
+}
+```
+
+以下情况必须返回 `503`:
+
+- 必填环境变量缺失。
+- workspace root 不可访问。
+- 端口池不可用。
+- control/report token 缺失。
+- clock skew 超过健康阈值。
+- agent 正在关闭,不再接受新请求。
+
+### 6.2 Create Gateway
+
+```http
+POST /v1/gateways
+```
+
+请求:
+
+```json
+{
+ "instance_id": 123,
+ "user_id": 45,
+ "agent_type": "openclaw",
+ "workspace_path": "/workspaces/openclaw/user-45/instance-123",
+ "port_range": {
+ "start": 20000,
+ "end": 20099
+ },
+ "uid": 200123,
+ "gid": 200123,
+ "cpu_cores": 2,
+ "memory_mb": 4096,
+ "disk_quota_mb": 20480,
+ "generation": 7,
+ "environment": {
+ "CLAWMANAGER_LLM_BASE_URL": "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001/api/v1/gateway/llm",
+ "CLAWMANAGER_LLM_API_KEY": "instance-token",
+ "CLAWMANAGER_LLM_MODEL": "[\"auto\"]",
+ "CLAWMANAGER_LLM_PROVIDER": "openai-compatible",
+ "CLAWMANAGER_INSTANCE_TOKEN": "instance-token",
+ "OPENAI_API_KEY": "instance-token"
+ }
+}
+```
+
+响应:
+
+```json
+{
+ "gateway_id": "gw-123-7",
+ "port": 20017,
+ "pid": 8842,
+ "status": "starting"
+}
+```
+
+实现要求:
+
+- handler 必须快速返回,通常 1 到 3 秒内返回 `starting` 或已有 gateway 元数据。
+- 不能在 HTTP handler 内阻塞等待前台 gateway 进程完整运行。
+- `agent_type` 必须等于当前 Pod runtime type。
+- Pod draining 时返回 `409` 或 `503`,拒绝新建。
+- 以 `instance_id + generation` 为幂等键;重复请求不能重复启动进程。
+- 收到更高 generation 时,停止同实例旧 generation gateway 并释放端口。
+- 端口必须来自请求的 `port_range`。
+- 没有可用端口时返回:
+
+```http
+HTTP/1.1 409 Conflict
+Content-Type: text/plain
+
+no free port
+```
+
+- 启动失败、健康检查失败、配置写入失败时,上报 gateway `error` 并释放端口。
+
+### 6.3 Delete Gateway
+
+```http
+DELETE /v1/gateways/{gateway_id}
+```
+
+要求:
+
+- `gateway_id` 是 URL path escaped,agent 必须 decode 后再查找。
+- 优雅停止进程组,超时后强制 kill。
+- 释放端口、cgroup、进程表和本地状态。
+- 不删除 workspace。
+- gateway 不存在时建议返回 `204`,保持删除幂等。
+- 成功状态可以是 `200`、`202` 或 `204`。
+
+### 6.4 Drain
+
+```http
+POST /v1/drain
+```
+
+请求:
+
+```json
+{
+ "draining": true
+}
+```
+
+要求:
+
+- 设置本地 draining 状态。
+- 立即拒绝新的 `POST /v1/gateways`。
+- 已有 gateway 不主动停止,继续健康检查和上报。
+- 立即补发 heartbeat,`state=draining, draining=true`。
+- 等 ClawManager 迁移实例后,逐个收到 DELETE 再清理。
+
+## 7. Gateway 生命周期
+
+推荐状态机:
+
+```text
+requested -> reserved_port -> starting -> running
+ -> error
+running -> stopping -> stopped
+running -> unhealthy -> running/error
+```
+
+创建流程:
+
+1. 校验 token、runtime type、instance/user/generation、workspace、端口范围。
+2. 如果 draining,拒绝创建。
+3. 查本地状态,处理幂等。
+4. 如果 generation 更高,清理旧 generation。
+5. 加锁分配端口,标记为 reserved。
+6. 创建 workspace 和 home,设置 owner 为请求中的 `uid/gid`。
+7. 写入 runtime 配置和 LLM/proxy 配置。
+8. 持久化 gateway 元数据为 `starting`。
+9. 后台启动 gateway 子进程,立即返回。
+10. 健康检查通过后上报 `running`。
+11. 失败时上报 `error`,记录短错误信息并释放端口。
+
+`running` 必须以真实进程和真实健康检查为准,不能只读旧状态文件。
+
+## 8. 端口管理
+
+agent 必须实现并发安全的端口分配器:
+
+```text
+reserve(instance_id, generation, count) -> port_set
+commit(instance_id, generation, port_set)
+release(instance_id, generation)
+list_used()
+```
+
+要求:
+
+- 分配时持有互斥锁。
+- 在锁内检查 agent 内存状态和系统监听端口。
+- 启动失败、健康检查失败、DELETE 成功后必须释放端口。
+- 如果 runtime 需要多个端口,必须一次性分配端口组,避免部分成功。
+- 同一 Pod 内端口不能冲突;不同 Pod 可以复用相同端口,因为路由使用 `pod_ip + port`。
+
+## 9. Workspace 与资源隔离
+
+workspace 固定格式:
+
+```text
+/workspaces/{runtime}/user-{user_id}/instance-{instance_id}
+```
+
+agent 必须:
+
+- 对 workspace root 和请求路径执行 realpath。
+- 拒绝 `..`、绝对路径逃逸、符号链接逃逸、跨用户、跨实例路径。
+- 创建 `${workspace}/home`,并设置 `HOME=${workspace}/home`。
+- owner 设置为请求中的 `uid/gid`,当前默认 `200000 + instance_id`。
+- token、provider、配置文件权限建议 `0600`。
+- 删除 gateway 时不删除 workspace。
+
+资源限制要求:
+
+- 每个 gateway 使用独立进程组。
+- 业务进程最终以实例 UID/GID 运行。
+- CPU/Memory 使用 cgroup 限制。
+- Disk 优先使用 filesystem quota;不支持时至少周期扫描并在超限时上报错误。
+- 资源限制无法启用时必须在 report 或错误消息中体现,不要静默忽略。
+
+## 10. Runtime 配置和 LLM 注入
+
+ClawManager 会通过 `POST /v1/gateways` 的 `environment` 字段下发实例级 LLM 环境变量。agent 只能转发白名单变量:
+
+```text
+CLAWMANAGER_LLM_BASE_URL
+CLAWMANAGER_LLM_API_KEY
+CLAWMANAGER_LLM_MODEL
+CLAWMANAGER_LLM_PROVIDER
+CLAWMANAGER_INSTANCE_TOKEN
+OPENAI_BASE_URL
+OPENAI_API_BASE
+OPENAI_API_KEY
+OPENAI_MODEL
+```
+
+要求:
+
+- LLM base URL 使用 ClawManager 内部 Service DNS。
+- API key 使用当前实例 token。
+- 不要求用户在 runtime 页面手工填写 OpenAI key。
+- 不打印 token/API key。
+- 如果缺少 LLM key,不能上报 gateway `running`;应上报 `error` 并说明缺少实例 LLM token。
+
+runtime 配置必须写在实例 workspace 内,例如:
+
+| Runtime | 推荐路径 |
+| --- | --- |
+| OpenClaw | `{workspace}/home/.openclaw/openclaw.json` |
+| Hermes | `{workspace}/home/.hermes/hermes.json` 或 Hermes 原生配置路径 |
+
+JSON merge 规则:
+
+- 对象字段递归 merge。
+- 保留用户已有未知字段。
+- 平台必须控制的字段可以覆盖,但要在代码中显式列出。
+- 数组 append 后去重,例如 `allowedOrigins`。
+- 写入前先生成临时文件,再原子 rename。
+
+## 11. 代理和浏览器访问
+
+浏览器访问链路必须是:
+
+```text
+Browser HTTPS -> ClawManager HTTPS -> ClawManager backend proxy -> Runtime Pod HTTP gateway
+```
+
+agent 写 runtime 配置时必须保证:
+
+- gateway 支持 `/api/v1/instances/{instance_id}/proxy` base path。
+- allowed origin 使用 ClawManager 内部 origin,例如 `http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001`。
+- trusted proxies 来自 `CLAWMANAGER_TRUSTED_PROXY_CIDRS` 或等价 env。
+- 不使用外部地址作为 runtime 内部 trusted origin。
+- 通过 ClawManager proxy 访问时,不应要求用户手工粘贴 gateway token。
+
+## 12. 时间同步
+
+agent 必须处理 clock skew:
+
+- 注册和 heartbeat 响应如果带 `server_time`,记录本地时间和服务端时间偏移。
+- 偏移超过 30 秒时记录 warning,并在 health 中暴露 `clock_skew_warning`。
+- 偏移超过 120 秒时不应上报 `ready` 或 gateway `running`。
+- 所有签名、token 时间和上报时间使用 UTC。
+- 不在容器里修改系统时间,时间同步由 Kubernetes 节点 NTP/chrony 负责。
+
+常见症状包括 `device signature expired`、JWT/WebSocket 鉴权异常、签名 URL 过期和 gateway 实际状态与 ClawManager 展示不一致。
+
+## 13. Instance Agent v1 兼容协议
+
+当前后端仍提供 `/api/v1/agent/*`,用于实例级状态、命令和 skill 管理。托管 runtime 如果需要复用这套能力,应由具体实例的 gateway/agent 逻辑使用实例级环境变量。
+
+ClawManager 会为支持托管集成的实例注入:
+
+| 变量 | 说明 |
+| --- | --- |
+| `CLAWMANAGER_AGENT_ENABLED` | `true` 时启用实例 agent |
+| `CLAWMANAGER_AGENT_BASE_URL` | ClawManager API base URL,不带 `/api/v1/agent` 后缀 |
+| `CLAWMANAGER_AGENT_BOOTSTRAP_TOKEN` | 首次注册使用 |
+| `CLAWMANAGER_AGENT_INSTANCE_ID` | 当前实例 ID |
+| `CLAWMANAGER_AGENT_PROTOCOL_VERSION` | 当前为 `v1` |
+| `CLAWMANAGER_AGENT_PERSISTENT_DIR` | 当前实例持久化目录 |
+| `CLAWMANAGER_AGENT_DISK_LIMIT_BYTES` | 实例磁盘配额 |
+
+### 13.1 Register
+
+```http
+POST {base}/api/v1/agent/register
+Authorization: Bearer ${CLAWMANAGER_AGENT_BOOTSTRAP_TOKEN}
+```
+
+payload:
+
+```json
+{
+ "instance_id": 123,
+ "agent_id": "openclaw-123-main",
+ "agent_version": "0.1.0",
+ "protocol_version": "v1",
+ "capabilities": [
+ "runtime.status",
+ "runtime.health",
+ "metrics.report",
+ "skills.inventory",
+ "skills.upload",
+ "commands.poll",
+ "llm.gateway"
+ ],
+ "host_info": {
+ "runtime_type": "openclaw",
+ "runtime_name": "OpenClaw",
+ "image": "ghcr.io/yuan-lab-llm/agentsruntime/openclaw:latest",
+ "persistent_dir": "/config"
+ }
+}
+```
+
+响应 `data` 包含:
+
+- `session_token`
+- `session_expires_at`
+- `heartbeat_interval_seconds`,当前默认 `15`
+- `command_poll_interval_seconds`,当前默认 `5`
+- `server_time`
+
+session token 每次成功 heartbeat 会续期 24 小时。收到 `401` 时删除本地 session,并用 bootstrap token 重新注册。
+
+### 13.2 Heartbeat
+
+```http
+POST {base}/api/v1/agent/heartbeat
+Authorization: Bearer ${session_token}
+```
+
+payload:
+
+```json
+{
+ "agent_id": "openclaw-123-main",
+ "timestamp": "2026-06-08T09:00:15Z",
+ "openclaw_status": "running",
+ "current_config_revision_id": null,
+ "summary": {
+ "runtime_type": "openclaw",
+ "runtime_status": "running",
+ "runtime_pid": 245,
+ "openclaw_pid": 245,
+ "skill_count": 8,
+ "disk_used_bytes": 2147483648,
+ "disk_limit_bytes": 10737418240
+ }
+}
+```
+
+兼容要求:
+
+- `openclaw_status`、`openclaw_pid`、`openclaw_version` 是历史字段名;非 OpenClaw runtime 也需要填自己的主进程状态、PID 和版本,直到协议升级。
+- 后端 45 秒内收到 heartbeat 展示 online,45 到 120 秒展示 stale,超过 120 秒展示 offline。
+- heartbeat 响应里的 `has_pending_command=true` 时,应立即拉取命令。
+
+### 13.3 State Report
+
+```http
+POST {base}/api/v1/agent/state/report
+Authorization: Bearer ${session_token}
+```
+
+payload:
+
+```json
+{
+ "agent_id": "openclaw-123-main",
+ "reported_at": "2026-06-08T09:00:20Z",
+ "runtime": {
+ "openclaw_status": "running",
+ "openclaw_pid": 245,
+ "openclaw_version": "openclaw-2026.5.4",
+ "current_config_revision_id": null
+ },
+ "system_info": {
+ "runtime_type": "openclaw",
+ "cpu": {
+ "cores": 2,
+ "load": {
+ "1m": 0.64,
+ "5m": 0.52,
+ "15m": 0.4
+ }
+ },
+ "memory": {
+ "mem_total_bytes": 4294967296,
+ "mem_available_bytes": 2147483648
+ },
+ "disk": {
+ "root_total_bytes": 10737418240,
+ "root_free_bytes": 8589934592
+ },
+ "network": {
+ "interfaces": [
+ {
+ "name": "eth0",
+ "status": "up",
+ "addresses": ["10.42.0.12"],
+ "rx_bytes": 123456789,
+ "tx_bytes": 98765432
+ }
+ ]
+ }
+ },
+ "health": {
+ "runtime_process": "ok",
+ "agent": "ok",
+ "metrics_collector": "ok"
+ }
+}
+```
+
+network 的 `rx_bytes` / `tx_bytes` 必须是单调递增 counter。
+
+### 13.4 Commands
+
+拉取:
+
+```http
+GET {base}/api/v1/agent/commands/next
+Authorization: Bearer ${session_token}
+```
+
+开始:
+
+```http
+POST {base}/api/v1/agent/commands/{id}/start
+Authorization: Bearer ${session_token}
+```
+
+完成:
+
+```http
+POST {base}/api/v1/agent/commands/{id}/finish
+Authorization: Bearer ${session_token}
+```
+
+finish 只允许:
+
+- `succeeded`
+- `failed`
+
+当前支持的命令类型:
+
+```text
+start_openclaw
+stop_openclaw
+restart_openclaw
+collect_system_info
+apply_config_revision
+reload_config
+health_check
+install_skill
+update_skill
+uninstall_skill
+remove_skill
+disable_skill
+quarantine_skill
+handle_skill_risk
+sync_skill_inventory
+refresh_skill_inventory
+collect_skill_package
+```
+
+未知命令必须 finish 为 `failed`,`error_message` 写明 `unsupported command type: `。
+
+### 13.5 Skills
+
+inventory:
+
+```http
+POST {base}/api/v1/agent/skills/inventory
+Authorization: Bearer ${session_token}
+```
+
+upload:
+
+```http
+POST {base}/api/v1/agent/skills/upload
+Authorization: Bearer ${session_token}
+Content-Type: multipart/form-data
+```
+
+上传表单字段:
+
+- `file`
+- `agent_id`
+- `skill_id`
+- `skill_version`
+- `identifier`
+- `content_md5`
+- `source`
+
+要求:
+
+- `identifier` 和 `content_md5` 必须稳定。
+- `mode=full` 的 inventory 代表全量结果,平台会用它对齐实例 skill 状态。
+- `content_md5` 按目录内容指纹计算,不是 zip 文件 MD5;算法见 `docs/skill-content-md5-spec.md`。
+- 打包和解压必须防止路径逃逸。
+
+## 14. 安全规范
+
+- 控制接口只接受正确 `X-ClawManager-Control-Token`。
+- 上报接口只使用 `X-ClawManager-Agent-Token`。
+- 实例级接口使用 Bearer bootstrap/session token。
+- 日志不得打印 token、API key、完整 Authorization header、channel secret。
+- 启动进程使用 argv 数组,不使用 shell 拼接:
+
+```text
+execve(binary, ["openclaw", "gateway", "run", "--port", "20017"], env)
+```
+
+- 不允许用户环境变量覆盖 agent 自己的 control/report token。
+- 不允许 `allowedOrigins=["*"]` 作为生产配置。
+- 所有用户输入路径都要边界校验。
+- 命令执行必须有超时,超过 `timeout_seconds` 主动终止并 finish failed。
+
+## 15. 测试要求
+
+单元测试至少覆盖:
+
+- Runtime type 只接受 `openclaw` / `hermes`。
+- create gateway 幂等。
+- 更高 generation 替换旧 generation。
+- 端口并发分配无冲突。
+- 端口被系统占用时跳过。
+- 端口耗尽返回 `409 no free port`。
+- workspace path 越权被拒绝。
+- JSON merge 保留用户字段,数组去重。
+- LLM env 白名单转发。
+- 缺少 LLM token 时 gateway 不报 `running`。
+- trusted proxy 和 allowed origin 配置生成。
+- clock skew 阈值判断。
+- redirect 响应不被当成成功。
+- DELETE gateway id path escape/decode。
+
+集成测试至少覆盖:
+
+- Runtime Pod 注册为 `ready`,capacity/used_slots/draining 正确。
+- heartbeat 更新 Pod 状态。
+- metrics 上报后管理端 Runtime Pods 页面能看到指标。
+- 创建 100 个 gateway 端口唯一,第 101 个触发扩容或 no capacity。
+- 重复 create 同一 `instance_id + generation` 不重复启动。
+- gateway report 旧 generation 不覆盖新 generation。
+- ClawManager 通过 proxy 能访问 gateway 页面和 WebSocket。
+- 通过 ClawManager proxy 访问时不要求用户手工粘贴 gateway token。
+- Pod drain 后拒绝新 create,已有 gateway 持续上报,DELETE 后释放端口。
+- Pod 重启后不误报旧 gateway running。
+- instance agent register/heartbeat/state/commands/skills 全链路可用。
+- 日志中没有 token 和 API key。
+
+## 16. Do / Don't
+
+| Do | Don't |
+| --- | --- |
+| 使用 Kubernetes Service DNS 访问 ClawManager backend | 写死外部 NodePort、`172.16.1.12` 或浏览器 HTTPS 入口 |
+| `POST /v1/gateways` 快速返回 `starting` | 在 HTTP handler 中阻塞等待前台进程退出 |
+| 以 `instance_id + generation` 幂等 | 重试时重复启动多个 gateway |
+| 以真实进程和健康检查决定 `running` | 只读旧状态文件就上报 `running` |
+| JSON merge runtime 配置 | 覆盖用户整个配置文件 |
+| 只转发白名单 env | 把用户请求里的全部 env 原样传给进程 |
+| trusted proxy 模式下通过 ClawManager 自动进入 | 要求用户手工粘贴 gateway token |
+| 检测并上报 clock skew | 忽略时间导致签名和 WebSocket 鉴权问题反复出现 |
+| DELETE gateway 不删除 workspace | 清理进程时顺手删除用户数据 |
+
+## 17. 代码位置
+
+后端关键实现:
+
+- Runtime Pod 上报接口:`backend/internal/handlers/runtime_agent_handler.go`
+- ClawManager 调 agent 客户端:`backend/internal/services/runtime_agent_client.go`
+- Runtime 调度器:`backend/internal/services/runtime_scheduler.go`
+- Runtime Deployment 构建:`backend/internal/services/k8s/runtime_deployment_service.go`
+- Runtime 常量与路径:`backend/internal/services/runtime_capacity.go`
+- 实例 agent handler:`backend/internal/handlers/agent_handler.go`
+- 实例 agent service:`backend/internal/services/instance_agent_service.go`
+- 实例命令 service:`backend/internal/services/instance_command_service.go`
+- 实例 runtime 状态 service:`backend/internal/services/instance_runtime_status_service.go`
+
+相关文档:
+
+- `docs/clawmanager-agent-v2-contract.md`
+- `docs/runtime-agent-integration-guide.md`
+- `docs/hermes-lite-pro-agent-development.md`
+- `docs/hermes-runtime-agent-development.md`
+- `docs/skill-content-md5-spec.md`
diff --git a/docs/hermes-lite-pro-agent-development.md b/docs/hermes-lite-pro-agent-development.md
new file mode 100644
index 0000000..82ef047
--- /dev/null
+++ b/docs/hermes-lite-pro-agent-development.md
@@ -0,0 +1,227 @@
+# Hermes Lite / Pro Agent 开发说明
+
+本文写给 Hermes agent 侧开发者,用来区分 ClawManager 中 Hermes Lite 和 Hermes Pro 两种形态。先读本文,再分别参考:
+
+- Lite: `docs/agent-runtime-development-spec.md` 和 `docs/clawmanager-agent-v2-contract.md`
+- Pro: `docs/hermes-runtime-agent-development.md` 和 `docs/agent-control-plane.md`
+
+一句话原则:
+
+- **Hermes Lite 是 Runtime Pod Agent V2**:一个共享 `hermes-runtime` Pod 内只有一个 pod-level agent,它负责创建、删除和上报多个 Hermes gateway 子进程。
+- **Hermes Pro 是 Instance Agent Control Plane**:每个 Hermes Pro 实例有自己的桌面容器和 instance-level agent,它只负责这个实例自己的状态、命令、skill 和 metrics。
+
+不要只根据镜像名或 `hermes` 这个 runtime 类型判断开发目标。Hermes Lite 和 Hermes Pro 都叫 Hermes,但 agent 协议、进程模型、目录和端口完全不同。
+
+## 1. 模式对照
+
+| 项目 | Hermes Lite | Hermes Pro |
+| --- | --- | --- |
+| ClawManager mode | `lite` | `pro` |
+| Runtime backend | `gateway` | `desktop` |
+| Kubernetes 资源 | 共享 Hermes runtime Deployment/Pod | 每个实例一个专属 Deployment + Service |
+| agent 定位 | Pod 级 gateway 管理器 | 实例级状态和命令 agent |
+| agent 入站端口 | 必须监听 `0.0.0.0:19090` | 不需要实现 runtime-pod 控制端口 |
+| ClawManager 调 agent | `GET /v1/health`、`POST /v1/gateways`、`DELETE /v1/gateways/{id}`、`POST /v1/drain` | 不适用 |
+| agent 上报 ClawManager | `/api/v1/runtime-agent/*` | `/api/v1/agent/*` |
+| 用户实例进程 | 每个 Lite 实例是一个 gateway 子进程 | 整个桌面容器就是实例 |
+| 用户访问端口 | agent 分配 `20000-20099` 中的 gateway 端口 | Webtop/KasmVNC `3001` |
+| workspace | `/workspaces/hermes/user-{user_id}/instance-{instance_id}` | `/config/.hermes` |
+| 典型实现参考 | 按 OpenClaw Lite 的 Runtime Pod Agent V2 做 | 按当前 Hermes Webtop guide 做 |
+
+## 2. 如何判断当前该跑哪种 agent
+
+建议 Hermes 镜像里可以放同一个 `hermes-agent` 二进制,但必须拆成两个清晰入口:
+
+```bash
+hermes-agent runtime-pod # Lite
+hermes-agent instance # Pro
+```
+
+也可以自动检测环境变量:
+
+| 检测条件 | 应进入的模式 |
+| --- | --- |
+| `CLAWMANAGER_RUNTIME_TYPE=hermes` 且存在 `CLAWMANAGER_AGENT_PORT` 或 `RUNTIME_AGENT_PUBLIC_PORT` | Lite runtime-pod agent |
+| `CLAWMANAGER_AGENT_ENABLED=true` 且存在 `CLAWMANAGER_AGENT_INSTANCE_ID` 和 `CLAWMANAGER_AGENT_BOOTSTRAP_TOKEN` | Pro instance agent |
+
+如果两组变量同时存在,优先按明确的启动参数决定;没有启动参数时建议拒绝启动并打印清晰错误,避免一个进程同时承担两套协议。
+
+## 3. Hermes Lite 应该怎么开发
+
+Hermes Lite 要仿照当前 OpenClaw Lite 做。差异只有 runtime 名称、Hermes gateway 启动命令、Hermes 配置文件位置和健康检查细节。
+
+Lite agent 是共享 Pod 内的控制进程,不是某个用户实例内部的 agent。它启动后必须:
+
+- 监听 `0.0.0.0:19090`。
+- 校验 `X-ClawManager-Control-Token: ${CLAWMANAGER_AGENT_CONTROL_TOKEN}`。
+- 实现 `GET /v1/health`、`POST /v1/gateways`、`DELETE /v1/gateways/{gateway_id}`、`POST /v1/drain`。
+- 向 `${CLAWMANAGER_BACKEND_URL}/api/v1/runtime-agent/*` 注册、heartbeat、metrics、gateway 状态和 skill inventory。
+- 管理本 Pod 内多个 Hermes gateway 子进程,端口默认从 `20000-20099` 分配。
+- 按 `instance_id + generation` 做幂等创建,重复请求不能启动多个进程。
+- `POST /v1/gateways` 必须快速返回 `starting`,后台继续启动和健康检查。
+- gateway 健康后再上报 `running`;缺少 LLM token、workspace 越权、端口冲突或 Hermes 配置失败时上报 `error`。
+- 每个实例使用独立 workspace,不能共用一个全局 `/config/.hermes`。
+
+Lite runtime Pod 会收到这些环境变量:
+
+| 变量 | 用途 |
+| --- | --- |
+| `CLAWMANAGER_RUNTIME_TYPE=hermes` | runtime 类型,注册和上报都用 `hermes` |
+| `CLAWMANAGER_BACKEND_URL` | ClawManager backend 内部地址 |
+| `CLAWMANAGER_RUNTIME_DEPLOYMENT_NAME` | 当前 runtime Deployment 名 |
+| `CLAWMANAGER_RUNTIME_IMAGE_REF` | 当前 runtime 镜像 |
+| `CLAWMANAGER_AGENT_PORT=19090` | agent 控制端口 |
+| `CLAWMANAGER_AGENT_CONTROL_TOKEN` | ClawManager 调 agent 控制接口使用 |
+| `CLAWMANAGER_AGENT_REPORT_TOKEN` | agent 上报 ClawManager 使用 |
+| `RUNTIME_WORKSPACE_ROOT=/workspaces` | workspace 根目录 |
+| `RUNTIME_AGENT_LISTEN_ADDR=0.0.0.0:19090` | agent 监听地址 |
+| `RUNTIME_AGENT_PUBLIC_PORT=19090` | agent 对 ClawManager 暴露的端口 |
+| `RUNTIME_GATEWAY_PORT_START` / `RUNTIME_GATEWAY_PORT_END` | gateway 端口范围 |
+| `POD_NAME` / `POD_NAMESPACE` / `POD_IP` / `NODE_NAME` | Pod 身份和调度信息 |
+| `CLAWMANAGER_TRUSTED_PROXY_CIDRS` | ClawManager 反代来源网段,用于 Hermes trusted proxy |
+
+ClawManager 创建 Lite 实例时会调用 runtime-pod agent:
+
+```http
+POST /v1/gateways
+X-ClawManager-Control-Token: ${CLAWMANAGER_AGENT_CONTROL_TOKEN}
+Content-Type: application/json
+```
+
+```json
+{
+ "instance_id": 123,
+ "user_id": 45,
+ "generation": 7,
+ "agent_type": "hermes",
+ "workspace_path": "/workspaces/hermes/user-45/instance-123",
+ "gateway_port_start": 20000,
+ "gateway_port_end": 20099,
+ "uid": 200123,
+ "gid": 200123,
+ "environment": {
+ "CLAWMANAGER_LLM_BASE_URL": "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001/api/v1/gateway/llm",
+ "CLAWMANAGER_LLM_API_KEY": "instance-token",
+ "CLAWMANAGER_LLM_MODEL": "[{\"id\":\"gpt-4.1\"}]",
+ "CLAWMANAGER_INSTANCE_TOKEN": "instance-token",
+ "OPENAI_BASE_URL": "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001/api/v1/gateway/llm",
+ "OPENAI_API_KEY": "instance-token"
+ }
+}
+```
+
+Lite agent 的建议处理流程:
+
+1. 校验控制 token、`agent_type=hermes`、`workspace_path` 必须在 `/workspaces/hermes/` 下。
+2. 用 `instance_id + generation` 查本地状态;如果已经创建过,返回同一个 gateway 元数据。
+3. 创建 workspace,并把 owner 设置为请求里的 `uid/gid`。
+4. 分配未占用 gateway 端口,记录端口、pid、workspace、generation。
+5. 把请求里的 `environment` 白名单变量传给 Hermes gateway 子进程。
+6. 写入 Hermes gateway 需要的 LLM、workspace、trusted proxy 和 instance token 配置。
+7. 后台启动 Hermes gateway,例如 `hermes gateway run --port ${port}`,实际命令由 Hermes 项目确定。
+8. 立即返回 `status=starting`。
+9. 后台轮询健康检查,成功后上报 `/api/v1/runtime-agent/gateways/report` 为 `running`。
+10. 删除时停止对应进程组并释放端口,但不要删除 workspace。
+
+Lite 下浏览器访问链路是:
+
+```text
+Browser -> ClawManager -> Runtime Pod HTTP gateway
+```
+
+ClawManager 会向 Lite runtime gateway 代理请求注入实例 token。Hermes gateway 不应该要求用户手工粘贴 gateway token;它需要信任来自 ClawManager 的代理来源,并正确处理 `Authorization`、`X-Forwarded-*` 和内部 base URL。
+
+## 4. Hermes Pro 应该怎么开发
+
+Hermes Pro 继续按现有 Hermes Webtop 文档实现。Pro 是专属桌面实例,不需要实现 Lite 的 `/v1/gateways` 协议,也不需要在一个 Pod 里管理多个用户实例。
+
+Pro image 的固定约定:
+
+- 基于 Webtop/KasmVNC。
+- 对外服务端口是 `3001`。
+- 持久化目录是 `/config/.hermes`。
+- ClawManager 会把 `SUBFOLDER` 设置为 `/api/v1/instances/{instance_id}/proxy/`。
+- agent 作为 s6 longrun 或等价守护进程运行在这个实例容器里。
+
+Pro instance agent 会收到这些环境变量:
+
+| 变量 | 用途 |
+| --- | --- |
+| `CLAWMANAGER_AGENT_ENABLED=true` | 启用实例 agent |
+| `CLAWMANAGER_AGENT_BASE_URL` | ClawManager API base URL,不带 `/api/v1/agent` 后缀 |
+| `CLAWMANAGER_AGENT_BOOTSTRAP_TOKEN` | 首次注册使用的一次性 token |
+| `CLAWMANAGER_AGENT_INSTANCE_ID` | 当前实例 ID |
+| `CLAWMANAGER_AGENT_PROTOCOL_VERSION=v1` | instance agent 协议版本 |
+| `CLAWMANAGER_AGENT_PERSISTENT_DIR=/config/.hermes` | Hermes 持久化目录 |
+| `CLAWMANAGER_AGENT_DISK_LIMIT_BYTES` | 实例磁盘配额 |
+
+Pro agent 的职责:
+
+- 读取 `CLAWMANAGER_AGENT_*` 变量。
+- 向 `${CLAWMANAGER_AGENT_BASE_URL}/api/v1/agent/register` 注册。
+- 使用 bootstrap 后返回的 session token 持续 heartbeat。
+- 上报 runtime state、health、system metrics、config revision 和 skill inventory。
+- 轮询并执行 ClawManager 下发的 instance command。
+- 管理 `/config/.hermes/skills` 里的 skill 下载、安装、上传和 inventory。
+- 所有状态都只代表这个 Pro 实例自身。
+
+Pro agent 不应该实现:
+
+- `POST /v1/gateways`
+- gateway 端口池
+- 多实例 workspace 隔离
+- `/api/v1/runtime-agent/*` runtime pod 上报
+- Lite 的 drain 和 gateway 迁移逻辑
+
+## 5. OpenClaw 对照开发法
+
+如果你已经看过 OpenClaw 的实现,可以这样映射:
+
+| OpenClaw 概念 | Hermes Lite 应替换为 |
+| --- | --- |
+| `CLAWMANAGER_RUNTIME_TYPE=openclaw` | `CLAWMANAGER_RUNTIME_TYPE=hermes` |
+| `agent_type=openclaw` | `agent_type=hermes` |
+| `/workspaces/openclaw/user-{uid}/instance-{id}` | `/workspaces/hermes/user-{uid}/instance-{id}` |
+| OpenClaw gateway 启动命令 | Hermes gateway 启动命令 |
+| OpenClaw workspace 配置文件 | Hermes workspace 内的实例级配置文件 |
+| OpenClaw 健康检查 | Hermes gateway 健康检查 |
+
+不要把 OpenClaw Pro/Desktop 的 agent 逻辑直接套到 Hermes Lite。Lite 的关键不是“启动一个桌面”,而是“在共享 Pod 中稳定管理多个 gateway 子进程”。
+
+## 6. 验收清单
+
+Hermes Lite 验收:
+
+- runtime pod agent 可以注册为 `runtime_type=hermes`。
+- `GET /v1/health` 返回 ready 状态和真实容量。
+- 创建 100 个 Hermes gateway 时端口唯一,状态最终变为 `running`。
+- 重复 `POST /v1/gateways` 不会启动重复进程。
+- 删除 gateway 会停止进程并释放端口,不会删除 workspace。
+- `POST /v1/drain` 后拒绝新 create,已有 gateway 持续上报。
+- Pod 重启后不会把不存在的旧 gateway 误报为 `running`。
+- 缺少 LLM token 时 gateway 不报告 `running`,错误信息明确。
+- 通过 ClawManager proxy 访问 Hermes Lite 不需要用户手工粘贴 token。
+- 每个 Lite 实例的 skill inventory、配置和 workspace 互相隔离。
+
+Hermes Pro 验收:
+
+- Pro 实例的 Webtop 端口 `3001` 可通过 ClawManager proxy 访问。
+- `/config/.hermes` 持久化,实例重启后数据仍在。
+- instance agent 能注册、heartbeat、上报 state 和 metrics。
+- instance command 能 poll、start、finish 并返回失败原因。
+- `/config/.hermes/skills` 的 skill 下载、安装、上传和 inventory 可用。
+- Pro agent 不监听或实现 `/v1/gateways`。
+
+## 7. 常见错误
+
+| 错误 | 正确做法 |
+| --- | --- |
+| 只实现 Pro instance agent,然后期望 Lite 可用 | Lite 必须实现 Runtime Pod Agent V2 |
+| Lite agent 监听 `3001` | Lite agent 监听 `19090`,gateway 子进程监听 `20000-20099` |
+| Lite 所有实例共用 `/config/.hermes` | Lite 每个实例使用自己的 `/workspaces/hermes/user-{uid}/instance-{id}` |
+| `POST /v1/gateways` 阻塞到 Hermes gateway 完全启动 | 快速返回 `starting`,后台健康检查后上报 `running` |
+| 重试 create 时启动多个 gateway | 按 `instance_id + generation` 幂等 |
+| 把外部 HTTPS/NodePort 写进 runtime 内部配置 | runtime 内部用 ClawManager backend service DNS |
+| 要求用户粘贴 gateway token | 使用 ClawManager 注入的 instance token 和 trusted proxy |
+| Lite 上报 `/api/v1/agent/*` 当作生命周期依据 | Lite 生命周期必须上报 `/api/v1/runtime-agent/*` |
+| Pro 实现 `/v1/gateways` 和端口池 | Pro 只实现 instance-level `/api/v1/agent/*` |
diff --git a/docs/hermes-runtime-agent-development.md b/docs/hermes-runtime-agent-development.md
index 1fc396a..25ce32c 100644
--- a/docs/hermes-runtime-agent-development.md
+++ b/docs/hermes-runtime-agent-development.md
@@ -2,6 +2,8 @@
This guide is for Hermes image and agent developers. It explains how to build a ClawManager-managed Hermes runtime image on top of Webtop, and how the embedded Hermes agent should integrate with ClawManager Agent Control Plane so Hermes can behave like OpenClaw: report live runtime status, report health and system metrics, sync skill inventory, upload skill packages, and poll runtime commands.
+> Mode note: this document describes the Hermes Pro Webtop/instance-agent path. For Hermes Lite, implement Runtime Pod Agent V2 instead. See [Hermes Lite / Pro Agent 开发说明](./hermes-lite-pro-agent-development.md).
+
## Goals
A Hermes image must satisfy two layers of requirements:
diff --git a/docs/lite-mode-development-progress-2026-06-11.md b/docs/lite-mode-development-progress-2026-06-11.md
new file mode 100644
index 0000000..859e8eb
--- /dev/null
+++ b/docs/lite-mode-development-progress-2026-06-11.md
@@ -0,0 +1,98 @@
+# Lite 模式与 Share Link 开发进度汇报
+
+> 汇报时间:2026-06-11 下午
+
+## 技术实现概览
+
+本次开发的核心是把实例形态从原来的运行时类型,抽象成用户可理解的 `Lite / Pro` 两种模式。
+
+- `Lite`:使用共享 Runtime Pod,在 Pod 内为每个实例创建独立 gateway 进程,适合轻量、低资源成本的使用场景。
+- `Pro`:使用独立的 Kubernetes Deployment / Service / PVC,适合需要完整桌面运行环境和独立资源的场景。
+
+后端以 `instance_mode` 作为产品层模式,内部再映射到不同 runtime backend:
+
+- `lite` -> `gateway`
+- `pro` -> `desktop`
+
+这样前端、普通实例创建、Team 成员创建、管理端展示和外部访问都统一使用 Lite / Pro 语义;底层调度、代理、资源控制仍然按 gateway / desktop 执行。
+
+Share Link 能力基于实例外部访问表实现,生成 `/s//` 形式的短链接。短链 code 只在创建时返回,数据库只保存 hash;访问时由 ClawManager 解析短链,再复用现有实例代理链路转发到对应 Lite gateway 或 Pro service。当前支持普通分享链接和密码访问,并支持固定时间、自定义时间和永久有效几类过期策略。
+
+## Lite / Pro 架构图
+
+```mermaid
+flowchart LR
+ user["用户 / 外部访问者"] --> web["ClawManager Web"]
+ user --> shortLink["Share Link
/s/<code>/"]
+
+ web --> api["ClawManager Backend API"]
+ shortLink --> publicAccess["短链解析与鉴权
Share Link / Password"]
+ publicAccess --> proxy["统一实例代理"]
+ api --> instanceService["Instance Service
instance_mode: lite / pro"]
+ instanceService --> mode{"实例模式"}
+
+ mode -->|Lite| liteScheduler["Lite Scheduler
runtime_type = gateway"]
+ mode -->|Pro| proLifecycle["Pro Lifecycle
runtime_type = desktop"]
+
+ liteScheduler --> runtimePod["共享 Runtime Pod"]
+ runtimePod --> runtimeAgent["Runtime Pod Agent V2"]
+ runtimeAgent --> gatewayA["Gateway 进程
Lite 实例 A"]
+ runtimeAgent --> gatewayB["Gateway 进程
Lite 实例 B"]
+ runtimeAgent --> gatewayN["Gateway 进程
Lite 实例 N"]
+
+ proLifecycle --> deployment["专属 Deployment"]
+ deployment --> service["专属 Service"]
+ deployment --> pvc["专属 PVC"]
+ service --> desktop["Desktop Runtime
Pro 实例"]
+
+ proxy -->|Lite| gatewayA
+ proxy -->|Lite| gatewayB
+ proxy -->|Pro| service
+
+ team["Team 创建"] --> teamService["Team Service"]
+ teamService --> teamMemberMode{"成员实例模式"}
+ teamMemberMode -->|Lite 成员| liteScheduler
+ teamMemberMode -->|Pro 成员| proLifecycle
+
+ aiGateway["AI Gateway / 审计 / 费用 / 风险"] --> attribution["实例归因
instance_id / instance_mode / runtime_type"]
+ gatewayA --> attribution
+ gatewayB --> attribution
+ desktop --> attribution
+```
+
+## 目标
+
+- 新增 Lite 实例模式,让用户在创建普通实例时可以选择 Lite / Pro。
+- 保留 Pro 模式的完整桌面实例能力,同时让 Lite 模式复用共享运行时,降低资源占用和启动成本。
+- 将 Lite / Pro 模式贯通到 Team 场景,让 Team 成员实例也可以按角色选择 Lite 或 Pro。
+- 增加 Share Link 能力,支持将实例访问入口分享给外部用户。
+- 建立自动化测试覆盖,保证普通实例、Team 实例和分享访问能力可以持续回归。
+
+## 进展
+
+- 普通实例创建已经支持 Lite / Pro 模式选择。
+- Lite 实例创建时会进入共享 gateway runtime 链路,Pro 实例创建时会进入独立 desktop runtime 链路。
+- 前端创建页、实例详情页、实例列表和管理端已完成 Lite / Pro 的展示和交互区分。
+- Lite 模式下已简化资源配置流程,不再要求用户配置专属 CPU、内存、存储和 GPU;Pro 模式继续保留专属资源配置能力。
+- Team 创建流程已经支持为成员选择 Lite / Pro 实例模式。
+- Team 成员实例已能按选择的模式创建,并保留后续接入 Team 协作链路所需的基础配置。
+- Share Link 已完成基础能力,包括短链接分享、密码访问、过期时间选择和复制入口。
+- Share Link 访问已统一走 ClawManager 代理,能够兼容 Lite gateway 和 Pro service 两种后端形态。
+- 相关改动已部署到 172.16.1.12 环境,并完成一轮自动化回归。
+- 当前自动化测试已覆盖普通实例 Lite / Pro 模式、Share Link 基础行为、管理端模式展示以及 Team Lite 创建链路。
+- 联调中发现,部分 Lite runtime / agent 能力仍需补齐,主要集中在 Team 任务消费和 Hermes Lite 会话稳定性上。
+
+## 计划
+
+- 继续推进 agent / runtime 侧补齐 Lite 模式能力,重点包括 Team 任务接收、执行状态回写和会话稳定性。
+- 在 agent / runtime 修复后,补充完整的 Team Lite 任务执行 e2e,覆盖从建队、成员实例创建、任务下发到状态回写的闭环。
+- 继续完善 Hermes Lite 的联调验证,确保 Lite 模式下聊天、访问和实例生命周期稳定。
+- 对 Share Link 增加更完整的回归覆盖,包括短链访问、密码访问、过期策略、禁用/重新生成等场景。
+- 在 172.16.1.12 环境继续做端到端回归,确认普通实例、Team 实例和 Share Link 在 Lite / Pro 两种模式下都能稳定运行。
+- 最后整理验收说明,明确 Lite / Pro 的支持范围、已验证场景和仍需 agent 侧配合的事项。
+
+## 简短汇报口径
+
+这次开发不是单独做 Team Lite,而是把实例模式整体升级为 Lite / Pro 两种形态。Lite 走共享 runtime gateway,Pro 走独立 desktop runtime,对用户层统一暴露 Lite / Pro,对后端仍按不同 runtime backend 调度。
+
+目前普通实例创建、实例展示、管理端识别、Team 成员创建以及 Share Link 分享能力都已经完成基础支持,并已在 172.16.1.12 做了一轮自动化回归。后续重点是继续和 agent / runtime 侧联调,补齐 Lite 模式下 Team 任务执行闭环和 Hermes Lite 会话稳定性,然后补充更完整的端到端验收测试。
diff --git a/docs/openclaw-windows-node-exec.md b/docs/openclaw-windows-node-exec.md
new file mode 100644
index 0000000..5cee925
--- /dev/null
+++ b/docs/openclaw-windows-node-exec.md
@@ -0,0 +1,233 @@
+# OpenClaw Gateway + Windows Node Exec Configuration
+
+本文说明一个常见 ClawManager 使用场景:
+
+```text
+用户浏览器
+ -> ClawManager Web / OpenClaw WebChat
+ -> 服务端 OpenClaw 实例里的 Gateway
+ -> 本机 Windows 上运行的 OpenClaw node
+ -> 在 Windows 本机执行 system.run / system.which
+```
+
+目标是:用户访问 ClawManager 创建出的 OpenClaw 实例 Web UI,在聊天中让 Agent 操作个人 Windows 电脑上的命令或自动化任务。
+
+## 1. 基本判断
+
+这个场景可行,但要区分两类授权:
+
+| 授权类型 | 作用 | 是否每次变化 |
+| --- | --- | --- |
+| Device pairing | 允许 Windows node 接入服务端 Gateway | 通常只需要一次 |
+| Exec approval | 允许某条 `system.run` 在 Windows node 上执行 | ask 模式下每条命令都会生成新的 requestId |
+
+如果错误是:
+
+```json
+{
+ "status": "error",
+ "tool": "exec",
+ "error": "UNAVAILABLE: SYSTEM_RUN_DENIED: approval required"
+}
+```
+
+说明链路已经打到 Windows node,问题不是网络,而是 `system.run` 被 Windows node 的 exec approval 策略拦住。
+
+## 2. 前置条件
+
+服务端 OpenClaw 实例里的 Gateway 必须满足:
+
+- Gateway WebSocket 端口能被 Windows 电脑访问,例如 `192.168.30.133:31889`。
+- Gateway 不能只监听容器内 `127.0.0.1`。
+- 如果使用明文 `ws://`,建议只在内网、VPN、Tailscale 或 SSH tunnel 内使用。
+- Windows node 需要拿到 Gateway 的认证 token 或 password。
+
+Windows 本机需要满足:
+
+- 已安装 OpenClaw CLI。
+- 能访问服务端 Gateway 地址和端口。
+- Node 进程能持久运行,建议调通后改为 service 模式。
+
+## 3. 启动 Windows Node
+
+在服务端 OpenClaw 实例里查看 Gateway token:
+
+```bash
+openclaw config get gateway.auth.token
+```
+
+在 Windows PowerShell 中设置 token 并启动 node:
+
+```powershell
+$env:OPENCLAW_GATEWAY_TOKEN=""
+openclaw node run --host 192.168.30.133 --port 31889 --display-name "windows-node"
+```
+
+如果要后台常驻:
+
+```powershell
+$env:OPENCLAW_GATEWAY_TOKEN=""
+openclaw node install --host 192.168.30.133 --port 31889 --display-name "windows-node" --force
+openclaw node restart
+openclaw node status
+```
+
+如果 Gateway 使用 TLS,再加:
+
+```powershell
+openclaw node run --host 192.168.30.133 --port 31889 --tls --display-name "windows-node"
+```
+
+## 4. 在 Gateway 侧完成 Node Pairing
+
+进入服务端 OpenClaw 实例终端,查看待审批设备:
+
+```bash
+openclaw devices list
+```
+
+找到 role 为 `node` 的请求,批准最新的 `requestId`:
+
+```bash
+openclaw devices approve
+```
+
+确认 node 已连接并 paired:
+
+```bash
+openclaw nodes status
+openclaw nodes describe --node "windows-node"
+```
+
+如果 `devices list` 每次看到新的 pairing `requestId`,通常说明 node 身份或认证信息变化了。优先检查:
+
+- Windows 上是否反复使用了不同的 `--node-id`。
+- Windows 上 `~/.openclaw/node.json` 是否被删除或重建。
+- Gateway token/password 是否变更。
+- Node 是否确实使用同一个 `OPENCLAW_GATEWAY_TOKEN` 启动。
+
+## 5. 将 Exec 指向 Windows Node
+
+在服务端 OpenClaw 实例里配置默认 exec host:
+
+```bash
+openclaw config set tools.exec.host node
+openclaw config set tools.exec.node "windows-node"
+```
+
+如果有多个 Agent,也可以按 Agent 配置:
+
+```bash
+openclaw config get agents.list
+openclaw config set agents.list[0].tools.exec.node "windows-node"
+```
+
+## 6. Exec Approval 配置方案
+
+### 方案 A:稳定自动化,免审批执行
+
+仅建议在完全可信的内网或 VPN 环境中使用。
+
+服务端 Gateway 请求策略:
+
+```bash
+openclaw config set tools.exec.security full
+openclaw config set tools.exec.ask off
+openclaw gateway restart
+```
+
+同时把 Windows node 本地 approvals 策略也打开:
+
+```bash
+openclaw approvals set --node "windows-node" --stdin <<'EOF'
+{
+ "version": 1,
+ "defaults": {
+ "security": "full",
+ "ask": "off",
+ "askFallback": "full"
+ }
+}
+EOF
+```
+
+注意:只改 `tools.exec.*` 不够。OpenClaw 会取 Gateway 请求策略和 node 本地 `~/.openclaw/exec-approvals.json` 中更严格的一方。如果 node 本地仍是 `ask: "always"` 或 `askFallback: "deny"`,还是会继续提示授权。
+
+### 方案 B:更安全,使用 allowlist
+
+服务端 Gateway 请求策略:
+
+```bash
+openclaw config set tools.exec.security allowlist
+openclaw config set tools.exec.ask off
+openclaw gateway restart
+```
+
+给 Windows node 添加允许执行的命令:
+
+```bash
+openclaw approvals allowlist add --agent "*" --node "windows-node" "powershell.exe"
+openclaw approvals allowlist add --agent "*" --node "windows-node" "pwsh.exe"
+openclaw approvals allowlist add --agent "*" --node "windows-node" "cmd.exe"
+```
+
+更推荐只放行具体脚本或具体工具,例如:
+
+```bash
+openclaw approvals allowlist add --agent "*" --node "windows-node" "C:\\Tools\\automation\\daily-task.ps1"
+openclaw approvals allowlist add --agent "*" --node "windows-node" "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe"
+```
+
+Windows 上如果命令被包装成 `cmd.exe /c ...`,allowlist 模式下仍可能需要审批。这是 OpenClaw 的安全行为。若需要完全无提示自动化,使用方案 A。
+
+## 7. 验证
+
+在服务端 OpenClaw 实例里执行:
+
+```bash
+openclaw nodes status
+openclaw nodes describe --node "windows-node"
+openclaw approvals get --node "windows-node"
+```
+
+然后在 WebChat 里让 Agent 执行一个简单命令:
+
+```text
+请在 node 上执行 hostname,并告诉我输出。
+```
+
+如果仍然报 `SYSTEM_RUN_DENIED`:
+
+1. 确认 `openclaw nodes status` 中 node 是 connected/paired。
+2. 确认 `openclaw nodes describe --node "windows-node"` 中包含 `system.run`。
+3. 确认 `openclaw approvals get --node "windows-node"` 的 effective policy 是预期值。
+4. 如果 effective policy 仍然是 ask 或 deny,说明 node 本地 approvals 文件没有被更新。
+5. 如果是 allowlist miss,说明命令没有命中 allowlist,改成 full 或补 allowlist。
+
+## 8. requestId 每次不同怎么处理
+
+如果 requestId 来自 `devices list`:
+
+- 这是 node pairing 请求。
+- 应该只需要批准一次。
+- 如果每次都变,说明 node 身份被重置或认证信息变化。
+
+如果 requestId 来自 exec approval:
+
+- 这是命令执行审批。
+- 每条新命令都有自己的 requestId,变化是正常现象。
+- 要避免反复授权,需要配置 `tools.exec.ask off`,并同步配置 node 本地 approvals。
+
+## 9. 安全建议
+
+- 不要把 Gateway WebSocket 端口直接暴露公网。
+- 生产环境优先使用 VPN、Tailscale、WireGuard 或 SSH tunnel。
+- 如果只是做固定自动化,优先使用 allowlist 放行具体脚本。
+- `security full + ask off` 相当于允许 WebChat 控制 Windows 本机命令,只有在完全可信环境使用。
+- 定期检查 Windows node 的 `~/.openclaw/exec-approvals.json`。
+- 不再使用时,及时停掉 node:
+
+```powershell
+openclaw node stop
+openclaw node uninstall
+```
diff --git a/docs/superpowers/plans/2026-06-01-clawmanager-v2-runtime-pool.md b/docs/superpowers/plans/2026-06-01-clawmanager-v2-runtime-pool.md
new file mode 100644
index 0000000..2ff15ef
--- /dev/null
+++ b/docs/superpowers/plans/2026-06-01-clawmanager-v2-runtime-pool.md
@@ -0,0 +1,3080 @@
+# ClawManager V2 Runtime Pool Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Build ClawManager V2 so OpenClaw and Hermes instances run as gateway processes inside shared runtime Pods, with workspace isolation, fast failover, admin runtime observability, and a simpler user UI.
+
+**Architecture:** The backend remains the source of truth in MySQL, uses Redis for hot locks/events/cache, and runs an embedded leader-elected runtime scheduler across multiple backend replicas. Runtime Pods are managed by Kubernetes Deployments and `clawmanager-agent`; each runtime Pod hosts up to 100 gateways and exposes gateway traffic by Pod IP plus assigned port. Workspaces live under a shared `/workspaces` mount and are accessed through a dedicated backend file manager API, not through the frame.
+
+**Tech Stack:** Go 1.26, Gin, MySQL via `upper/db`, Redis RESP client, Kubernetes client-go, React 19, Vite, TypeScript, Tailwind CSS, Kubernetes YAML.
+
+---
+
+## Scope Check
+
+This release touches several subsystems, but they are tightly ordered: schema first, scheduler second, lifecycle/proxy/files third, UI and manifests last. Keep the work in this single master plan, but commit after each task so any subsystem can be reviewed independently.
+
+## File Structure
+
+Backend data and migrations:
+- Modify: `backend/internal/db/migrations.go` - existing embedded migration runner remains unchanged unless ordering assumptions fail in tests.
+- Create: `backend/internal/db/migrations/023_add_runtime_pool_v2.sql` - runtime Pod, gateway binding, rollout, and workspace audit schema.
+- Modify: `backend/internal/db/migrations_test.go` - verify migration file ordering and SQL splitting still handles migration 023.
+- Modify: `backend/internal/models/instance.go` - add workspace fields and V2 runtime metadata.
+- Create: `backend/internal/models/runtime_pod.go` - runtime Pod state and metrics model.
+- Create: `backend/internal/models/instance_runtime_binding.go` - active gateway binding model.
+- Create: `backend/internal/models/runtime_rollout.go` - rollout state model for gray upgrades.
+- Create: `backend/internal/models/workspace_file_audit.go` - upload/download/write audit model.
+- Modify: `backend/internal/repository/instance_repository.go` - V2 lifecycle selectors and status update helpers.
+- Create: `backend/internal/repository/runtime_pod_repository.go` - runtime Pod CRUD and capacity claims.
+- Create: `backend/internal/repository/instance_runtime_binding_repository.go` - gateway binding CRUD and generation-safe updates.
+- Create: `backend/internal/repository/runtime_rollout_repository.go` - rollout CRUD and selectors.
+- Create: `backend/internal/repository/workspace_file_audit_repository.go` - append-only file audit writes.
+
+Backend runtime scheduling and agent integration:
+- Create: `docs/clawmanager-agent-v2-contract.md` - exact HTTP dual-channel contract required from `clawmanager-agent`.
+- Create: `backend/internal/services/redis_client.go` - platform Redis client reusing the existing RESP approach.
+- Create: `backend/internal/services/runtime_agent_client.go` - backend-to-agent command client.
+- Create: `backend/internal/services/runtime_scheduler.go` - leader loop, assignment, failover, and gray-drain orchestration.
+- Create: `backend/internal/services/runtime_capacity.go` - capacity calculations and runtime type normalization.
+- Create: `backend/internal/services/runtime_events.go` - Redis-backed runtime event fanout for all backend replicas.
+- Create: `backend/internal/services/runtime_leader.go` - Kubernetes Lease leader election wrapper.
+- Create: `backend/internal/services/k8s/runtime_deployment_service.go` - create, scale, and inspect OpenClaw/Hermes runtime Deployments.
+- Modify: `backend/internal/services/instance_service.go` - route V2 create/start/stop/delete through scheduler state; keep legacy behavior for old instance types.
+- Modify: `backend/internal/services/instance_proxy_service.go` - proxy V2 instances to binding Pod IP plus gateway port; keep legacy service proxy fallback.
+- Modify: `backend/internal/services/sync_service.go` - stop assuming every running instance owns a Pod.
+- Modify: `backend/cmd/server/main.go` - wire repositories, services, scheduler start, routes, and websocket runtime event bridge.
+
+Backend workspace file APIs:
+- Create: `backend/internal/services/workspace_path_guard.go` - path clean, realpath, and symlink escape prevention.
+- Create: `backend/internal/services/workspace_file_service.go` - list, preview, download, upload, mkdir, rename, delete.
+- Create: `backend/internal/handlers/workspace_file_handler.go` - user-facing workspace file endpoints.
+- Modify: `backend/internal/handlers/instance_handler.go` - attach workspace routes and simplify status response for V2.
+- Modify: `backend/internal/utils/response.go` - add V2 file validation errors to client-safe bad request mapping.
+
+Backend admin APIs:
+- Create: `backend/internal/handlers/runtime_pool_handler.go` - admin runtime Pod status, metrics, drain, and rollout endpoints.
+- Modify: `backend/internal/services/websocket_service.go` - add admin runtime event subscriptions with user-role checks.
+- Modify: `backend/internal/handlers/websocket_handler.go` if it exists in this repository; otherwise wire websocket changes in the existing handler file discovered during implementation.
+
+Frontend user and admin experience:
+- Modify: `frontend/package.json` and lock file - add `lucide-react` if icons are not already available.
+- Modify: `frontend/src/index.css` - neutral, simple design tokens; remove heavy gradients and oversized radii.
+- Modify: `frontend/src/components/UserLayout.tsx` - simpler user shell.
+- Modify: `frontend/src/components/AdminLayout.tsx` - simpler admin shell and add Runtime Pods navigation.
+- Modify: `frontend/src/types/instance.ts` - V2 instance availability and workspace types.
+- Create: `frontend/src/types/runtimePool.ts` - admin runtime Pod, gateway, metrics, rollout types.
+- Create: `frontend/src/services/workspaceService.ts` - workspace API client.
+- Create: `frontend/src/services/runtimePoolService.ts` - admin runtime API client.
+- Modify: `frontend/src/hooks/useWebSocket.ts` - add admin runtime event hook.
+- Modify: `frontend/src/services/instanceService.ts` - V2 status fields and workspace entry points.
+- Modify: `frontend/src/pages/instances/CreateInstancePage.tsx` - only allow creating OpenClaw and Hermes for V2.
+- Modify: `frontend/src/pages/instances/InstanceListPage.tsx` - hide Pod/service/resource details from ordinary users.
+- Modify: `frontend/src/pages/instances/InstanceDetailPage.tsx` - show availability, service frame, workspace file manager, and simple controls.
+- Create: `frontend/src/components/WorkspaceFileManager.tsx` - independent file list, preview, upload, download, rename, delete.
+- Create: `frontend/src/components/InstanceServiceFrame.tsx` - OpenClaw/Hermes iframe wrapper with availability state.
+- Create: `frontend/src/pages/admin/RuntimePodsPage.tsx` - admin-only runtime Pod metrics and gateway usage.
+- Modify: `frontend/src/pages/admin/SystemSettingsPage.tsx` - add OpenClaw/Hermes runtime image and gray rollout controls.
+- Modify: `frontend/src/router/index.tsx` - route `/admin/runtime-pods`.
+
+Deployment:
+- Modify: `deployments/k8s/clawmanager.yaml` - one-YAML install for Redis, workspace store, backend replicas, runtime Deployments, RBAC, env.
+- Modify: `deployments/k3s/clawmanager.yaml` - same defaults adapted to k3s/local path storage.
+- Modify: `backend/deployments/k8s/clawreef-incluster.yaml` if this file is still shipped as an install path; otherwise leave it unchanged and document that `deployments/k8s/clawmanager.yaml` is canonical.
+
+## Agent Contract Summary
+
+The `clawmanager-agent` changes are documented but implemented outside this repository. The backend implementation must assume these endpoints and payloads:
+
+Agent-to-backend channel:
+
+```http
+POST /api/v1/runtime-agent/register
+POST /api/v1/runtime-agent/heartbeat
+POST /api/v1/runtime-agent/gateways/report
+POST /api/v1/runtime-agent/skills/report
+POST /api/v1/runtime-agent/metrics/report
+```
+
+Backend-to-agent channel:
+
+```http
+GET http://{pod_ip}:19090/v1/health
+GET http://{pod_ip}:19090/v1/metrics
+POST http://{pod_ip}:19090/v1/gateways
+DELETE http://{pod_ip}:19090/v1/gateways/{gateway_id}
+POST http://{pod_ip}:19090/v1/gateways/{gateway_id}/health
+POST http://{pod_ip}:19090/v1/drain
+```
+
+Gateway create request:
+
+```json
+{
+ "instance_id": 123,
+ "user_id": 45,
+ "agent_type": "openclaw",
+ "workspace_path": "/workspaces/openclaw/user-45/instance-123",
+ "port_range": { "start": 20000, "end": 20099 },
+ "uid": 200123,
+ "gid": 200123,
+ "cpu_cores": 2,
+ "memory_mb": 4096,
+ "disk_quota_mb": 20480,
+ "generation": 7
+}
+```
+
+Gateway create response:
+
+```json
+{
+ "gateway_id": "gw-123-7",
+ "port": 20017,
+ "pid": 8842,
+ "status": "running"
+}
+```
+
+The agent must allocate a free port inside `20000-20099`, enforce one Linux UID/GID per instance using `200000 + instance_id`, apply cgroup CPU/memory limits, enforce workspace quota, reject symlink escapes inside workspaces, report gateway health, and survive backend replica changes. During gray upgrade, `POST /v1/drain` must reject new gateways and continue reporting existing gateway health until ClawManager migrates them away.
+
+## Task 1: Schema, Models, And Repository Interfaces
+
+**Files:**
+- Create: `backend/internal/db/migrations/023_add_runtime_pool_v2.sql`
+- Modify: `backend/internal/db/migrations_test.go`
+- Modify: `backend/internal/models/instance.go`
+- Create: `backend/internal/models/runtime_pod.go`
+- Create: `backend/internal/models/instance_runtime_binding.go`
+- Create: `backend/internal/models/runtime_rollout.go`
+- Create: `backend/internal/models/workspace_file_audit.go`
+- Modify: `backend/internal/repository/instance_repository.go`
+- Create: `backend/internal/repository/runtime_pod_repository.go`
+- Create: `backend/internal/repository/instance_runtime_binding_repository.go`
+- Create: `backend/internal/repository/runtime_rollout_repository.go`
+- Create: `backend/internal/repository/workspace_file_audit_repository.go`
+
+- [ ] **Step 1: Add failing migration order test**
+
+Append this test to `backend/internal/db/migrations_test.go`:
+
+```go
+func TestMigration023IsEmbedded(t *testing.T) {
+ files, err := migrationFiles.ReadDir("migrations")
+ if err != nil {
+ t.Fatalf("read embedded migrations: %v", err)
+ }
+
+ found := false
+ for _, file := range files {
+ if file.Name() == "023_add_runtime_pool_v2.sql" {
+ found = true
+ break
+ }
+ }
+ if !found {
+ t.Fatalf("migration 023_add_runtime_pool_v2.sql is not embedded")
+ }
+}
+```
+
+- [ ] **Step 2: Run migration test and verify it fails**
+
+Run:
+
+```powershell
+go test ./backend/internal/db -run TestMigration023IsEmbedded -count=1
+```
+
+Expected: fail with `migration 023_add_runtime_pool_v2.sql is not embedded`.
+
+- [ ] **Step 3: Create migration 023**
+
+Create `backend/internal/db/migrations/023_add_runtime_pool_v2.sql` with:
+
+```sql
+ALTER TABLE instances
+ ADD COLUMN workspace_path VARCHAR(1024) NULL AFTER mount_path,
+ ADD COLUMN workspace_usage_bytes BIGINT NOT NULL DEFAULT 0 AFTER workspace_path,
+ ADD COLUMN runtime_generation INT NOT NULL DEFAULT 1 AFTER workspace_usage_bytes,
+ ADD COLUMN runtime_error_message TEXT NULL AFTER runtime_generation;
+
+CREATE TABLE runtime_pods (
+ id BIGINT PRIMARY KEY AUTO_INCREMENT,
+ runtime_type VARCHAR(32) NOT NULL,
+ namespace VARCHAR(128) NOT NULL,
+ pod_name VARCHAR(255) NOT NULL,
+ pod_uid VARCHAR(128) NULL,
+ pod_ip VARCHAR(64) NULL,
+ node_name VARCHAR(255) NULL,
+ deployment_name VARCHAR(255) NOT NULL,
+ image_ref VARCHAR(512) NOT NULL,
+ agent_endpoint VARCHAR(255) NULL,
+ state VARCHAR(32) NOT NULL DEFAULT 'pending',
+ capacity INT NOT NULL DEFAULT 100,
+ used_slots INT NOT NULL DEFAULT 0,
+ draining TINYINT(1) NOT NULL DEFAULT 0,
+ cpu_millis_used BIGINT NOT NULL DEFAULT 0,
+ memory_bytes_used BIGINT NOT NULL DEFAULT 0,
+ disk_bytes_used BIGINT NOT NULL DEFAULT 0,
+ network_rx_bytes BIGINT NOT NULL DEFAULT 0,
+ network_tx_bytes BIGINT NOT NULL DEFAULT 0,
+ metrics_json JSON NULL,
+ last_seen_at DATETIME NULL,
+ created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+ UNIQUE KEY uk_runtime_pod_namespace_name (namespace, pod_name),
+ KEY idx_runtime_pods_schedulable (runtime_type, state, draining, used_slots, capacity),
+ KEY idx_runtime_pods_last_seen (last_seen_at)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+CREATE TABLE instance_runtime_bindings (
+ id BIGINT PRIMARY KEY AUTO_INCREMENT,
+ instance_id INT NOT NULL,
+ runtime_pod_id BIGINT NOT NULL,
+ runtime_type VARCHAR(32) NOT NULL,
+ gateway_id VARCHAR(128) NOT NULL,
+ gateway_port INT NOT NULL,
+ gateway_pid INT NULL,
+ workspace_path VARCHAR(1024) NOT NULL,
+ state VARCHAR(32) NOT NULL DEFAULT 'creating',
+ generation INT NOT NULL DEFAULT 1,
+ last_health_at DATETIME NULL,
+ error_message TEXT NULL,
+ created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+ UNIQUE KEY uk_instance_runtime_binding_instance (instance_id),
+ UNIQUE KEY uk_instance_runtime_binding_gateway (runtime_pod_id, gateway_port),
+ KEY idx_instance_runtime_binding_pod_state (runtime_pod_id, state),
+ CONSTRAINT fk_instance_runtime_binding_instance
+ FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE,
+ CONSTRAINT fk_instance_runtime_binding_pod
+ FOREIGN KEY (runtime_pod_id) REFERENCES runtime_pods(id) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+CREATE TABLE runtime_rollouts (
+ id BIGINT PRIMARY KEY AUTO_INCREMENT,
+ runtime_type VARCHAR(32) NOT NULL,
+ target_image_ref VARCHAR(512) NOT NULL,
+ status VARCHAR(32) NOT NULL DEFAULT 'pending',
+ batch_size INT NOT NULL DEFAULT 1,
+ max_unavailable INT NOT NULL DEFAULT 1,
+ started_by INT NULL,
+ started_at DATETIME NULL,
+ finished_at DATETIME NULL,
+ error_message TEXT NULL,
+ created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+ KEY idx_runtime_rollouts_type_status (runtime_type, status)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+CREATE TABLE workspace_file_audits (
+ id BIGINT PRIMARY KEY AUTO_INCREMENT,
+ instance_id INT NOT NULL,
+ user_id INT NOT NULL,
+ action VARCHAR(32) NOT NULL,
+ relative_path VARCHAR(1024) NOT NULL,
+ bytes BIGINT NOT NULL DEFAULT 0,
+ created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ KEY idx_workspace_file_audits_instance_time (instance_id, created_at),
+ KEY idx_workspace_file_audits_user_time (user_id, created_at),
+ CONSTRAINT fk_workspace_file_audits_instance
+ FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+```
+
+- [ ] **Step 4: Extend instance model**
+
+Add fields to `backend/internal/models/instance.go` after `MountPath` and after `StoppedAt`:
+
+```go
+ WorkspacePath *string `db:"workspace_path" json:"workspace_path,omitempty"`
+ WorkspaceUsageBytes int64 `db:"workspace_usage_bytes" json:"workspace_usage_bytes"`
+ RuntimeGeneration int `db:"runtime_generation" json:"runtime_generation"`
+ RuntimeErrorMessage *string `db:"runtime_error_message" json:"runtime_error_message,omitempty"`
+```
+
+- [ ] **Step 5: Add runtime models**
+
+Create `backend/internal/models/runtime_pod.go`:
+
+```go
+package models
+
+import "time"
+
+type RuntimePod struct {
+ ID int64 `db:"id,primarykey,autoincrement" json:"id"`
+ RuntimeType string `db:"runtime_type" json:"runtime_type"`
+ Namespace string `db:"namespace" json:"namespace"`
+ PodName string `db:"pod_name" json:"pod_name"`
+ PodUID *string `db:"pod_uid" json:"pod_uid,omitempty"`
+ PodIP *string `db:"pod_ip" json:"pod_ip,omitempty"`
+ NodeName *string `db:"node_name" json:"node_name,omitempty"`
+ DeploymentName string `db:"deployment_name" json:"deployment_name"`
+ ImageRef string `db:"image_ref" json:"image_ref"`
+ AgentEndpoint *string `db:"agent_endpoint" json:"agent_endpoint,omitempty"`
+ State string `db:"state" json:"state"`
+ Capacity int `db:"capacity" json:"capacity"`
+ UsedSlots int `db:"used_slots" json:"used_slots"`
+ Draining bool `db:"draining" json:"draining"`
+ CPUMillisUsed int64 `db:"cpu_millis_used" json:"cpu_millis_used"`
+ MemoryBytesUsed int64 `db:"memory_bytes_used" json:"memory_bytes_used"`
+ DiskBytesUsed int64 `db:"disk_bytes_used" json:"disk_bytes_used"`
+ NetworkRXBytes int64 `db:"network_rx_bytes" json:"network_rx_bytes"`
+ NetworkTXBytes int64 `db:"network_tx_bytes" json:"network_tx_bytes"`
+ MetricsJSON *string `db:"metrics_json" json:"metrics_json,omitempty"`
+ LastSeenAt *time.Time `db:"last_seen_at" json:"last_seen_at,omitempty"`
+ CreatedAt time.Time `db:"created_at" json:"created_at"`
+ UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
+}
+
+func (RuntimePod) TableName() string {
+ return "runtime_pods"
+}
+```
+
+Create `backend/internal/models/instance_runtime_binding.go`:
+
+```go
+package models
+
+import "time"
+
+type InstanceRuntimeBinding struct {
+ ID int64 `db:"id,primarykey,autoincrement" json:"id"`
+ InstanceID int `db:"instance_id" json:"instance_id"`
+ RuntimePodID int64 `db:"runtime_pod_id" json:"runtime_pod_id"`
+ RuntimeType string `db:"runtime_type" json:"runtime_type"`
+ GatewayID string `db:"gateway_id" json:"gateway_id"`
+ GatewayPort int `db:"gateway_port" json:"gateway_port"`
+ GatewayPID *int `db:"gateway_pid" json:"gateway_pid,omitempty"`
+ WorkspacePath string `db:"workspace_path" json:"workspace_path"`
+ State string `db:"state" json:"state"`
+ Generation int `db:"generation" json:"generation"`
+ LastHealthAt *time.Time `db:"last_health_at" json:"last_health_at,omitempty"`
+ ErrorMessage *string `db:"error_message" json:"error_message,omitempty"`
+ CreatedAt time.Time `db:"created_at" json:"created_at"`
+ UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
+}
+
+func (InstanceRuntimeBinding) TableName() string {
+ return "instance_runtime_bindings"
+}
+```
+
+Create `backend/internal/models/runtime_rollout.go`:
+
+```go
+package models
+
+import "time"
+
+type RuntimeRollout struct {
+ ID int64 `db:"id,primarykey,autoincrement" json:"id"`
+ RuntimeType string `db:"runtime_type" json:"runtime_type"`
+ TargetImageRef string `db:"target_image_ref" json:"target_image_ref"`
+ Status string `db:"status" json:"status"`
+ BatchSize int `db:"batch_size" json:"batch_size"`
+ MaxUnavailable int `db:"max_unavailable" json:"max_unavailable"`
+ StartedBy *int `db:"started_by" json:"started_by,omitempty"`
+ StartedAt *time.Time `db:"started_at" json:"started_at,omitempty"`
+ FinishedAt *time.Time `db:"finished_at" json:"finished_at,omitempty"`
+ ErrorMessage *string `db:"error_message" json:"error_message,omitempty"`
+ CreatedAt time.Time `db:"created_at" json:"created_at"`
+ UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
+}
+
+func (RuntimeRollout) TableName() string {
+ return "runtime_rollouts"
+}
+```
+
+Create `backend/internal/models/workspace_file_audit.go`:
+
+```go
+package models
+
+import "time"
+
+type WorkspaceFileAudit struct {
+ ID int64 `db:"id,primarykey,autoincrement" json:"id"`
+ InstanceID int `db:"instance_id" json:"instance_id"`
+ UserID int `db:"user_id" json:"user_id"`
+ Action string `db:"action" json:"action"`
+ RelativePath string `db:"relative_path" json:"relative_path"`
+ Bytes int64 `db:"bytes" json:"bytes"`
+ CreatedAt time.Time `db:"created_at" json:"created_at"`
+}
+
+func (WorkspaceFileAudit) TableName() string {
+ return "workspace_file_audits"
+}
+```
+
+- [ ] **Step 6: Add repository interfaces and implementations**
+
+Implement interfaces with these method signatures. Use `upper/db` collections for normal CRUD and raw SQL for capacity claims requiring atomic increments.
+
+`backend/internal/repository/runtime_pod_repository.go`:
+
+```go
+package repository
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "clawreef/internal/models"
+ "github.com/upper/db/v4"
+)
+
+type RuntimePodRepository interface {
+ UpsertFromAgent(ctx context.Context, pod *models.RuntimePod) error
+ GetByID(ctx context.Context, id int64) (*models.RuntimePod, error)
+ GetByNamespaceName(ctx context.Context, namespace, podName string) (*models.RuntimePod, error)
+ List(ctx context.Context, runtimeType string) ([]models.RuntimePod, error)
+ ListSchedulable(ctx context.Context, runtimeType string) ([]models.RuntimePod, error)
+ TryClaimSlot(ctx context.Context, podID int64) (bool, error)
+ ReleaseSlot(ctx context.Context, podID int64) error
+ MarkState(ctx context.Context, podID int64, state string, draining bool) error
+ MarkUnseenUnhealthy(ctx context.Context, cutoff time.Time) error
+ UpdateMetrics(ctx context.Context, podID int64, metrics RuntimePodMetricsUpdate) error
+}
+
+type RuntimePodMetricsUpdate struct {
+ CPUMillisUsed int64
+ MemoryBytesUsed int64
+ DiskBytesUsed int64
+ NetworkRXBytes int64
+ NetworkTXBytes int64
+ MetricsJSON string
+ LastSeenAt time.Time
+}
+
+type runtimePodRepository struct {
+ sess db.Session
+}
+
+func NewRuntimePodRepository(sess db.Session) RuntimePodRepository {
+ return &runtimePodRepository{sess: sess}
+}
+
+func (r *runtimePodRepository) collection() db.Collection {
+ return r.sess.Collection("runtime_pods")
+}
+```
+
+Then add each method in the same file. The `TryClaimSlot` implementation must use this SQL shape:
+
+```go
+func (r *runtimePodRepository) TryClaimSlot(ctx context.Context, podID int64) (bool, error) {
+ res, err := r.sess.SQL().
+ ExecContext(ctx, `UPDATE runtime_pods
+ SET used_slots = used_slots + 1
+ WHERE id = ? AND state = 'ready' AND draining = 0 AND used_slots < capacity`, podID)
+ if err != nil {
+ return false, err
+ }
+ affected, err := res.RowsAffected()
+ if err != nil {
+ return false, err
+ }
+ return affected == 1, nil
+}
+```
+
+`backend/internal/repository/instance_runtime_binding_repository.go` must include:
+
+```go
+type InstanceRuntimeBindingRepository interface {
+ Create(ctx context.Context, binding *models.InstanceRuntimeBinding) error
+ GetByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error)
+ GetRunningByInstanceID(ctx context.Context, instanceID int) (*models.InstanceRuntimeBinding, error)
+ ListByRuntimePodID(ctx context.Context, runtimePodID int64) ([]models.InstanceRuntimeBinding, error)
+ ListByRuntimePodIDs(ctx context.Context, runtimePodIDs []int64) ([]models.InstanceRuntimeBinding, error)
+ UpdateRunning(ctx context.Context, instanceID int, generation int, gatewayID string, port int, pid *int) error
+ UpdateState(ctx context.Context, instanceID int, generation int, state string, message *string) error
+ DeleteByInstanceID(ctx context.Context, instanceID int) error
+}
+```
+
+Extend `backend/internal/repository/instance_repository.go` with:
+
+```go
+ GetV2DesiredRunning(ctx context.Context, limit int) ([]models.Instance, error)
+ GetV2Creating(ctx context.Context, limit int) ([]models.Instance, error)
+ UpdateRuntimeState(ctx context.Context, id int, status string, generation int, message *string) error
+ SetWorkspacePath(ctx context.Context, id int, workspacePath string) error
+ UpdateWorkspaceUsage(ctx context.Context, id int, usageBytes int64) error
+```
+
+- [ ] **Step 7: Run backend tests for schema compilation**
+
+Run:
+
+```powershell
+go test ./backend/internal/db ./backend/internal/models ./backend/internal/repository -count=1
+```
+
+Expected: pass, or report packages with no test files and no compile errors.
+
+- [ ] **Step 8: Commit schema and repository layer**
+
+Run:
+
+```powershell
+git add backend/internal/db backend/internal/models backend/internal/repository
+git commit -m "feat: add runtime pool schema"
+```
+
+## Task 2: Agent Contract Documentation And HTTP Client
+
+**Files:**
+- Create: `docs/clawmanager-agent-v2-contract.md`
+- Create: `backend/internal/services/runtime_agent_client.go`
+- Create: `backend/internal/services/runtime_agent_client_test.go`
+
+- [ ] **Step 1: Write the agent contract document**
+
+Create `docs/clawmanager-agent-v2-contract.md` with these sections:
+
+```markdown
+# clawmanager-agent V2 Contract
+
+## Purpose
+
+The agent runs inside each shared OpenClaw or Hermes runtime Pod. It manages gateway subprocesses, ports, cgroups, Linux users, workspace quota, health checks, skill/status reporting, and metrics.
+
+## Agent-To-Backend Reports
+
+All report requests use header `X-ClawManager-Agent-Token`.
+
+### POST /api/v1/runtime-agent/register
+
+Request body:
+
+```json
+{
+ "runtime_type": "openclaw",
+ "namespace": "clawmanager-system",
+ "pod_name": "openclaw-runtime-6f77f8b8c7-abcde",
+ "pod_uid": "pod-uid",
+ "pod_ip": "10.42.0.31",
+ "node_name": "node-a",
+ "deployment_name": "openclaw-runtime",
+ "image_ref": "ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest",
+ "agent_endpoint": "http://10.42.0.31:19090",
+ "capacity": 100
+}
+```
+
+### POST /api/v1/runtime-agent/heartbeat
+
+Request body:
+
+```json
+{
+ "pod_name": "openclaw-runtime-6f77f8b8c7-abcde",
+ "runtime_type": "openclaw",
+ "state": "ready",
+ "used_slots": 37,
+ "draining": false
+}
+```
+
+### POST /api/v1/runtime-agent/metrics/report
+
+Request body:
+
+```json
+{
+ "pod_name": "openclaw-runtime-6f77f8b8c7-abcde",
+ "cpu_millis_used": 13600,
+ "memory_bytes_used": 42949672960,
+ "disk_bytes_used": 214748364800,
+ "network_rx_bytes": 9223372,
+ "network_tx_bytes": 19223372,
+ "gateways": [
+ {
+ "instance_id": 123,
+ "gateway_id": "gw-123-7",
+ "port": 20017,
+ "state": "running",
+ "last_health_at": "2026-06-01T10:00:00Z"
+ }
+ ]
+}
+```
+
+## Backend-To-Agent Commands
+
+All command requests use header `X-ClawManager-Control-Token`.
+
+### POST /v1/gateways
+
+The agent creates a gateway subprocess and returns the bound port. If no port is free in the requested range, return HTTP 409.
+
+### DELETE /v1/gateways/{gateway_id}
+
+The agent stops the gateway and releases cgroup, process, and port resources.
+
+### POST /v1/drain
+
+The agent marks the Pod as draining and rejects new gateway creation. Existing gateways continue until ClawManager migrates them.
+
+## Isolation Requirements
+
+Every gateway runs as UID/GID `200000 + instance_id`. The agent rejects workspace paths outside `/workspaces/{runtime}/user-{user_id}/instance-{instance_id}` after resolving symlinks. CPU and memory limits are enforced with cgroups. Workspace quota is enforced before writes.
+```
+
+- [ ] **Step 2: Write failing client tests**
+
+Create `backend/internal/services/runtime_agent_client_test.go`:
+
+```go
+package services
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+)
+
+func TestRuntimeAgentClientCreateGateway(t *testing.T) {
+ var gotToken string
+ var gotReq RuntimeAgentCreateGatewayRequest
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/v1/gateways" {
+ t.Fatalf("unexpected path %s", r.URL.Path)
+ }
+ gotToken = r.Header.Get("X-ClawManager-Control-Token")
+ if err := json.NewDecoder(r.Body).Decode(&gotReq); err != nil {
+ t.Fatalf("decode request: %v", err)
+ }
+ _ = json.NewEncoder(w).Encode(RuntimeAgentCreateGatewayResponse{
+ GatewayID: "gw-7-3",
+ Port: 20017,
+ PID: intPtr(8842),
+ Status: "running",
+ })
+ }))
+ defer server.Close()
+
+ client := NewRuntimeAgentClient("secret")
+ resp, err := client.CreateGateway(context.Background(), server.URL, RuntimeAgentCreateGatewayRequest{
+ InstanceID: 7,
+ UserID: 8,
+ AgentType: "openclaw",
+ WorkspacePath: "/workspaces/openclaw/user-8/instance-7",
+ PortRange: RuntimeAgentPortRange{Start: 20000, End: 20099},
+ UID: 200007,
+ GID: 200007,
+ CPUCores: 2,
+ MemoryMB: 4096,
+ DiskQuotaMB: 20480,
+ Generation: 3,
+ })
+ if err != nil {
+ t.Fatalf("CreateGateway returned error: %v", err)
+ }
+ if gotToken != "secret" {
+ t.Fatalf("unexpected token %q", gotToken)
+ }
+ if gotReq.InstanceID != 7 || gotReq.PortRange.Start != 20000 {
+ t.Fatalf("unexpected request %#v", gotReq)
+ }
+ if resp.GatewayID != "gw-7-3" || resp.Port != 20017 {
+ t.Fatalf("unexpected response %#v", resp)
+ }
+}
+
+func TestRuntimeAgentClientConflict(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Error(w, "no free port", http.StatusConflict)
+ }))
+ defer server.Close()
+
+ client := NewRuntimeAgentClient("secret")
+ _, err := client.CreateGateway(context.Background(), server.URL, RuntimeAgentCreateGatewayRequest{})
+ if err == nil || err.Error() != "runtime agent conflict: no free port\n" {
+ t.Fatalf("unexpected error %v", err)
+ }
+}
+
+func intPtr(v int) *int {
+ return &v
+}
+```
+
+- [ ] **Step 3: Run client tests and verify they fail**
+
+Run:
+
+```powershell
+go test ./backend/internal/services -run RuntimeAgentClient -count=1
+```
+
+Expected: fail because `RuntimeAgentCreateGatewayRequest` and `NewRuntimeAgentClient` do not exist.
+
+- [ ] **Step 4: Implement runtime agent client**
+
+Create `backend/internal/services/runtime_agent_client.go`:
+
+```go
+package services
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+ "time"
+)
+
+type RuntimeAgentClient interface {
+ Health(ctx context.Context, endpoint string) error
+ CreateGateway(ctx context.Context, endpoint string, req RuntimeAgentCreateGatewayRequest) (*RuntimeAgentCreateGatewayResponse, error)
+ DeleteGateway(ctx context.Context, endpoint, gatewayID string) error
+ Drain(ctx context.Context, endpoint string) error
+}
+
+type RuntimeAgentPortRange struct {
+ Start int `json:"start"`
+ End int `json:"end"`
+}
+
+type RuntimeAgentCreateGatewayRequest struct {
+ InstanceID int `json:"instance_id"`
+ UserID int `json:"user_id"`
+ AgentType string `json:"agent_type"`
+ WorkspacePath string `json:"workspace_path"`
+ PortRange RuntimeAgentPortRange `json:"port_range"`
+ UID int `json:"uid"`
+ GID int `json:"gid"`
+ CPUCores float64 `json:"cpu_cores"`
+ MemoryMB int `json:"memory_mb"`
+ DiskQuotaMB int `json:"disk_quota_mb"`
+ Generation int `json:"generation"`
+}
+
+type RuntimeAgentCreateGatewayResponse struct {
+ GatewayID string `json:"gateway_id"`
+ Port int `json:"port"`
+ PID *int `json:"pid,omitempty"`
+ Status string `json:"status"`
+}
+
+type runtimeAgentHTTPClient struct {
+ controlToken string
+ httpClient *http.Client
+}
+
+func NewRuntimeAgentClient(controlToken string) RuntimeAgentClient {
+ return &runtimeAgentHTTPClient{
+ controlToken: controlToken,
+ httpClient: &http.Client{
+ Timeout: 5 * time.Second,
+ },
+ }
+}
+
+func (c *runtimeAgentHTTPClient) Health(ctx context.Context, endpoint string) error {
+ return c.do(ctx, http.MethodGet, endpoint, "/v1/health", nil, nil)
+}
+
+func (c *runtimeAgentHTTPClient) CreateGateway(ctx context.Context, endpoint string, req RuntimeAgentCreateGatewayRequest) (*RuntimeAgentCreateGatewayResponse, error) {
+ var resp RuntimeAgentCreateGatewayResponse
+ if err := c.do(ctx, http.MethodPost, endpoint, "/v1/gateways", req, &resp); err != nil {
+ return nil, err
+ }
+ return &resp, nil
+}
+
+func (c *runtimeAgentHTTPClient) DeleteGateway(ctx context.Context, endpoint, gatewayID string) error {
+ return c.do(ctx, http.MethodDelete, endpoint, "/v1/gateways/"+gatewayID, nil, nil)
+}
+
+func (c *runtimeAgentHTTPClient) Drain(ctx context.Context, endpoint string) error {
+ return c.do(ctx, http.MethodPost, endpoint, "/v1/drain", map[string]bool{"draining": true}, nil)
+}
+
+func (c *runtimeAgentHTTPClient) do(ctx context.Context, method, endpoint, path string, body any, out any) error {
+ endpoint = strings.TrimRight(endpoint, "/")
+ var reader io.Reader
+ if body != nil {
+ payload, err := json.Marshal(body)
+ if err != nil {
+ return err
+ }
+ reader = bytes.NewReader(payload)
+ }
+ req, err := http.NewRequestWithContext(ctx, method, endpoint+path, reader)
+ if err != nil {
+ return err
+ }
+ req.Header.Set("X-ClawManager-Control-Token", c.controlToken)
+ if body != nil {
+ req.Header.Set("Content-Type", "application/json")
+ }
+ resp, err := c.httpClient.Do(req)
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode >= 400 {
+ msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+ if resp.StatusCode == http.StatusConflict {
+ return fmt.Errorf("runtime agent conflict: %s", string(msg))
+ }
+ return fmt.Errorf("runtime agent status %d: %s", resp.StatusCode, string(msg))
+ }
+ if out == nil {
+ return nil
+ }
+ return json.NewDecoder(resp.Body).Decode(out)
+}
+```
+
+- [ ] **Step 5: Run client tests**
+
+Run:
+
+```powershell
+go test ./backend/internal/services -run RuntimeAgentClient -count=1
+```
+
+Expected: pass.
+
+- [ ] **Step 6: Commit agent contract and client**
+
+Run:
+
+```powershell
+git add docs/clawmanager-agent-v2-contract.md backend/internal/services/runtime_agent_client.go backend/internal/services/runtime_agent_client_test.go
+git commit -m "feat: add runtime agent contract"
+```
+
+## Task 3: Runtime Capacity, Kubernetes Runtime Deployments, And Leader Election
+
+**Files:**
+- Create: `backend/internal/services/runtime_capacity.go`
+- Create: `backend/internal/services/runtime_capacity_test.go`
+- Create: `backend/internal/services/runtime_leader.go`
+- Create: `backend/internal/services/k8s/runtime_deployment_service.go`
+- Create: `backend/internal/services/k8s/runtime_deployment_service_test.go`
+- Modify: `backend/internal/config/config.go`
+
+- [ ] **Step 1: Write runtime capacity tests**
+
+Create `backend/internal/services/runtime_capacity_test.go`:
+
+```go
+package services
+
+import "testing"
+
+func TestNormalizeV2RuntimeType(t *testing.T) {
+ tests := map[string]string{
+ "openclaw": "openclaw",
+ "hermes": "hermes",
+ "webtop": "",
+ "ubuntu": "",
+ }
+ for input, want := range tests {
+ got, ok := NormalizeV2RuntimeType(input)
+ if want == "" {
+ if ok {
+ t.Fatalf("expected %s to be rejected", input)
+ }
+ continue
+ }
+ if !ok || got != want {
+ t.Fatalf("NormalizeV2RuntimeType(%q) = %q, %v", input, got, ok)
+ }
+ }
+}
+
+func TestRuntimeWorkspacePath(t *testing.T) {
+ got := RuntimeWorkspacePath("openclaw", 45, 123)
+ want := "/workspaces/openclaw/user-45/instance-123"
+ if got != want {
+ t.Fatalf("workspace path mismatch: want %s got %s", want, got)
+ }
+}
+
+func TestRuntimeLinuxID(t *testing.T) {
+ got := RuntimeLinuxID(123)
+ if got != 200123 {
+ t.Fatalf("runtime linux id mismatch: %d", got)
+ }
+}
+```
+
+- [ ] **Step 2: Implement capacity helpers**
+
+Create `backend/internal/services/runtime_capacity.go`:
+
+```go
+package services
+
+import "fmt"
+
+const (
+ RuntimeTypeOpenClaw = "openclaw"
+ RuntimeTypeHermes = "hermes"
+
+ RuntimeGatewayPortStart = 20000
+ RuntimeGatewayPortEnd = 20099
+ RuntimePodCapacity = 100
+ RuntimeLinuxIDBase = 200000
+)
+
+func NormalizeV2RuntimeType(instanceType string) (string, bool) {
+ switch instanceType {
+ case RuntimeTypeOpenClaw:
+ return RuntimeTypeOpenClaw, true
+ case RuntimeTypeHermes:
+ return RuntimeTypeHermes, true
+ default:
+ return "", false
+ }
+}
+
+func RuntimeWorkspacePath(runtimeType string, userID int, instanceID int) string {
+ return fmt.Sprintf("/workspaces/%s/user-%d/instance-%d", runtimeType, userID, instanceID)
+}
+
+func RuntimeLinuxID(instanceID int) int {
+ return RuntimeLinuxIDBase + instanceID
+}
+```
+
+- [ ] **Step 3: Run capacity tests**
+
+Run:
+
+```powershell
+go test ./backend/internal/services -run 'NormalizeV2RuntimeType|RuntimeWorkspacePath|RuntimeLinuxID' -count=1
+```
+
+Expected: pass.
+
+- [ ] **Step 4: Add runtime config fields**
+
+Modify `backend/internal/config/config.go` by adding fields to the main config struct:
+
+```go
+ Runtime RuntimeConfig
+```
+
+Add this struct and env parsing:
+
+```go
+type RuntimeConfig struct {
+ Namespace string
+ WorkspaceRoot string
+ AgentControlToken string
+ AgentReportToken string
+ BackendReplicaID string
+ RedisURL string
+ SchedulerEnabled bool
+ HeartbeatTimeout time.Duration
+ SchedulerTick time.Duration
+ OpenClawImage string
+ HermesImage string
+ MaxGatewaysPerPod int
+ GatewayPortStart int
+ GatewayPortEnd int
+}
+```
+
+Parse with defaults:
+
+```go
+Runtime: RuntimeConfig{
+ Namespace: getEnv("RUNTIME_NAMESPACE", getEnv("K8S_NAMESPACE", "clawmanager-system")),
+ WorkspaceRoot: getEnv("RUNTIME_WORKSPACE_ROOT", "/workspaces"),
+ AgentControlToken: getEnv("RUNTIME_AGENT_CONTROL_TOKEN", ""),
+ AgentReportToken: getEnv("RUNTIME_AGENT_REPORT_TOKEN", ""),
+ BackendReplicaID: getEnv("HOSTNAME", "clawmanager-backend-local"),
+ RedisURL: getEnv("PLATFORM_REDIS_URL", getEnv("TEAM_REDIS_URL", "")),
+ SchedulerEnabled: getEnvBool("RUNTIME_SCHEDULER_ENABLED", true),
+ HeartbeatTimeout: getEnvDuration("RUNTIME_HEARTBEAT_TIMEOUT", 10*time.Second),
+ SchedulerTick: getEnvDuration("RUNTIME_SCHEDULER_TICK", 2*time.Second),
+ OpenClawImage: getEnv("OPENCLAW_RUNTIME_IMAGE", "ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest"),
+ HermesImage: getEnv("HERMES_RUNTIME_IMAGE", "ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest"),
+ MaxGatewaysPerPod: getEnvInt("RUNTIME_MAX_GATEWAYS_PER_POD", RuntimePodCapacity),
+ GatewayPortStart: getEnvInt("RUNTIME_GATEWAY_PORT_START", RuntimeGatewayPortStart),
+ GatewayPortEnd: getEnvInt("RUNTIME_GATEWAY_PORT_END", RuntimeGatewayPortEnd),
+},
+```
+
+If `getEnvDuration`, `getEnvBool`, or `getEnvInt` is missing, add them in `config.go` using `strconv` and `time.ParseDuration`.
+
+- [ ] **Step 5: Add Kubernetes runtime deployment service test**
+
+Create `backend/internal/services/k8s/runtime_deployment_service_test.go` with a unit test that constructs the Deployment and checks the shared workspace mount:
+
+```go
+package k8s
+
+import "testing"
+
+func TestBuildRuntimeDeploymentUsesSharedWorkspaceAndAgentPort(t *testing.T) {
+ dep := BuildRuntimeDeployment(RuntimeDeploymentSpec{
+ Name: "openclaw-runtime",
+ Namespace: "clawmanager-system",
+ RuntimeType: "openclaw",
+ Image: "example/openclaw:latest",
+ Replicas: 2,
+ WorkspaceNFSServer: "workspace-store.clawmanager-system.svc.cluster.local",
+ WorkspaceNFSPath: "/exports/workspaces",
+ AgentControlToken: "control",
+ AgentReportToken: "report",
+ })
+
+ if dep.Name != "openclaw-runtime" || *dep.Spec.Replicas != 2 {
+ t.Fatalf("unexpected deployment identity %#v", dep)
+ }
+ container := dep.Spec.Template.Spec.Containers[0]
+ if container.Ports[0].ContainerPort != 19090 {
+ t.Fatalf("agent port missing: %#v", container.Ports)
+ }
+ foundWorkspace := false
+ for _, mount := range container.VolumeMounts {
+ if mount.Name == "workspaces" && mount.MountPath == "/workspaces" {
+ foundWorkspace = true
+ }
+ }
+ if !foundWorkspace {
+ t.Fatalf("workspace mount missing: %#v", container.VolumeMounts)
+ }
+}
+```
+
+- [ ] **Step 6: Implement Kubernetes runtime deployment service**
+
+Create `backend/internal/services/k8s/runtime_deployment_service.go` with:
+
+```go
+package k8s
+
+import (
+ "context"
+
+ appsv1 "k8s.io/api/apps/v1"
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ "k8s.io/client-go/kubernetes"
+)
+
+type RuntimeDeploymentSpec struct {
+ Name string
+ Namespace string
+ RuntimeType string
+ Image string
+ Replicas int32
+ WorkspaceNFSServer string
+ WorkspaceNFSPath string
+ AgentControlToken string
+ AgentReportToken string
+}
+
+type RuntimeDeploymentService interface {
+ Ensure(ctx context.Context, spec RuntimeDeploymentSpec) error
+ Scale(ctx context.Context, namespace, name string, replicas int32) error
+}
+
+type runtimeDeploymentService struct {
+ client kubernetes.Interface
+}
+
+func NewRuntimeDeploymentService(client kubernetes.Interface) RuntimeDeploymentService {
+ return &runtimeDeploymentService{client: client}
+}
+
+func int32Ptr(v int32) *int32 {
+ return &v
+}
+
+func BuildRuntimeDeployment(spec RuntimeDeploymentSpec) *appsv1.Deployment {
+ labels := map[string]string{
+ "app": spec.Name,
+ "clawmanager.io/runtime-type": spec.RuntimeType,
+ }
+ return &appsv1.Deployment{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: spec.Name,
+ Namespace: spec.Namespace,
+ Labels: labels,
+ },
+ Spec: appsv1.DeploymentSpec{
+ Replicas: int32Ptr(spec.Replicas),
+ Selector: &metav1.LabelSelector{MatchLabels: labels},
+ Template: corev1.PodTemplateSpec{
+ ObjectMeta: metav1.ObjectMeta{Labels: labels},
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{{
+ Name: "runtime",
+ Image: spec.Image,
+ Ports: []corev1.ContainerPort{{
+ Name: "agent",
+ ContainerPort: 19090,
+ }},
+ Env: []corev1.EnvVar{
+ {Name: "CLAWMANAGER_RUNTIME_TYPE", Value: spec.RuntimeType},
+ {Name: "CLAWMANAGER_AGENT_PORT", Value: "19090"},
+ {Name: "CLAWMANAGER_GATEWAY_PORT_START", Value: "20000"},
+ {Name: "CLAWMANAGER_GATEWAY_PORT_END", Value: "20099"},
+ {Name: "CLAWMANAGER_AGENT_CONTROL_TOKEN", Value: spec.AgentControlToken},
+ {Name: "CLAWMANAGER_AGENT_REPORT_TOKEN", Value: spec.AgentReportToken},
+ },
+ Resources: corev1.ResourceRequirements{
+ Requests: corev1.ResourceList{
+ corev1.ResourceCPU: resource.MustParse("500m"),
+ corev1.ResourceMemory: resource.MustParse("1Gi"),
+ },
+ },
+ VolumeMounts: []corev1.VolumeMount{{
+ Name: "workspaces",
+ MountPath: "/workspaces",
+ }},
+ }},
+ Volumes: []corev1.Volume{{
+ Name: "workspaces",
+ VolumeSource: corev1.VolumeSource{
+ NFS: &corev1.NFSVolumeSource{
+ Server: spec.WorkspaceNFSServer,
+ Path: spec.WorkspaceNFSPath,
+ },
+ },
+ }},
+ },
+ },
+ },
+ }
+}
+```
+
+Add `Ensure` and `Scale` using AppsV1 Deployments `Get`, `Create`, `Update`, and `UpdateScale`.
+
+- [ ] **Step 7: Implement Lease leader service**
+
+Create `backend/internal/services/runtime_leader.go`:
+
+```go
+package services
+
+import (
+ "context"
+ "time"
+
+ coordinationv1 "k8s.io/api/coordination/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/kubernetes"
+)
+
+type RuntimeLeaderService interface {
+ IsLeader(ctx context.Context) bool
+}
+
+type runtimeLeaderService struct {
+ client kubernetes.Interface
+ namespace string
+ name string
+ holder string
+ ttl time.Duration
+}
+
+func NewRuntimeLeaderService(client kubernetes.Interface, namespace, holder string) RuntimeLeaderService {
+ return &runtimeLeaderService{
+ client: client,
+ namespace: namespace,
+ name: "clawmanager-runtime-scheduler",
+ holder: holder,
+ ttl: 15 * time.Second,
+ }
+}
+
+func (s *runtimeLeaderService) IsLeader(ctx context.Context) bool {
+ now := metav1.Now()
+ leaseClient := s.client.CoordinationV1().Leases(s.namespace)
+ lease, err := leaseClient.Get(ctx, s.name, metav1.GetOptions{})
+ if err != nil {
+ lease = &coordinationv1.Lease{
+ ObjectMeta: metav1.ObjectMeta{Name: s.name, Namespace: s.namespace},
+ Spec: coordinationv1.LeaseSpec{
+ HolderIdentity: &s.holder,
+ RenewTime: &now,
+ LeaseDurationSeconds: int32Ptr32(int32(s.ttl.Seconds())),
+ },
+ }
+ _, createErr := leaseClient.Create(ctx, lease, metav1.CreateOptions{})
+ return createErr == nil
+ }
+ if lease.Spec.HolderIdentity != nil && *lease.Spec.HolderIdentity != s.holder && lease.Spec.RenewTime != nil {
+ expires := lease.Spec.RenewTime.Time.Add(s.ttl)
+ if time.Now().Before(expires) {
+ return false
+ }
+ }
+ lease.Spec.HolderIdentity = &s.holder
+ lease.Spec.RenewTime = &now
+ _, err = leaseClient.Update(ctx, lease, metav1.UpdateOptions{})
+ return err == nil
+}
+
+func int32Ptr32(v int32) *int32 {
+ return &v
+}
+```
+
+- [ ] **Step 8: Run runtime deployment and leader tests**
+
+Run:
+
+```powershell
+go test ./backend/internal/services ./backend/internal/services/k8s -run 'Runtime|Leader' -count=1
+```
+
+Expected: pass.
+
+- [ ] **Step 9: Commit runtime capacity and Kubernetes services**
+
+Run:
+
+```powershell
+git add backend/internal/config backend/internal/services backend/internal/services/k8s
+git commit -m "feat: add runtime pool infrastructure services"
+```
+
+## Task 4: Runtime Scheduler And Redis Event Fanout
+
+**Files:**
+- Create: `backend/internal/services/redis_client.go`
+- Create: `backend/internal/services/runtime_events.go`
+- Create: `backend/internal/services/runtime_scheduler.go`
+- Create: `backend/internal/services/runtime_scheduler_test.go`
+- Modify: `backend/cmd/server/main.go`
+
+- [ ] **Step 1: Extract a platform Redis client**
+
+Create `backend/internal/services/redis_client.go` by reusing the existing RESP parsing pattern from `team_redis.go`. The exported interface must be:
+
+```go
+type PlatformRedisClient interface {
+ SetNX(ctx context.Context, key, value string, ttl time.Duration) (bool, error)
+ Del(ctx context.Context, key string) error
+ XAdd(ctx context.Context, key string, fields map[string]string) (string, error)
+ XRead(ctx context.Context, key, lastID string, block time.Duration) ([]redisStreamMessage, error)
+}
+```
+
+Use the same URL parsing behavior as `newRedisBus`, and add:
+
+```go
+func NewPlatformRedisClient(rawURL string) (PlatformRedisClient, error) {
+ return newRedisBus(rawURL)
+}
+```
+
+Extend `redisBus` in `team_redis.go` only if needed so the same type satisfies `SetNX` and `Del`:
+
+```go
+func (b *redisBus) SetNX(ctx context.Context, key, value string, ttl time.Duration) (bool, error) {
+ reply, err := b.do(ctx, "SET", key, value, "NX", "PX", fmt.Sprintf("%d", ttl.Milliseconds()))
+ if err != nil {
+ return false, err
+ }
+ return reply == "OK", nil
+}
+
+func (b *redisBus) Del(ctx context.Context, key string) error {
+ _, err := b.do(ctx, "DEL", key)
+ return err
+}
+```
+
+- [ ] **Step 2: Add runtime event service**
+
+Create `backend/internal/services/runtime_events.go`:
+
+```go
+package services
+
+import (
+ "context"
+ "encoding/json"
+ "time"
+)
+
+const runtimeEventStreamKey = "clawmanager:runtime-events"
+
+type RuntimeEvent struct {
+ Type string `json:"type"`
+ Payload json.RawMessage `json:"payload"`
+ CreatedAt time.Time `json:"created_at"`
+}
+
+type RuntimeEventService interface {
+ Publish(ctx context.Context, eventType string, payload any) error
+ Read(ctx context.Context, lastID string, block time.Duration) ([]redisStreamMessage, error)
+}
+
+type runtimeEventService struct {
+ redis PlatformRedisClient
+}
+
+func NewRuntimeEventService(redis PlatformRedisClient) RuntimeEventService {
+ return &runtimeEventService{redis: redis}
+}
+
+func (s *runtimeEventService) Publish(ctx context.Context, eventType string, payload any) error {
+ raw, err := json.Marshal(payload)
+ if err != nil {
+ return err
+ }
+ event := RuntimeEvent{Type: eventType, Payload: raw, CreatedAt: time.Now().UTC()}
+ eventRaw, err := json.Marshal(event)
+ if err != nil {
+ return err
+ }
+ _, err = s.redis.XAdd(ctx, runtimeEventStreamKey, map[string]string{"event": string(eventRaw)})
+ return err
+}
+
+func (s *runtimeEventService) Read(ctx context.Context, lastID string, block time.Duration) ([]redisStreamMessage, error) {
+ return s.redis.XRead(ctx, runtimeEventStreamKey, lastID, block)
+}
+```
+
+- [ ] **Step 3: Write scheduler assignment test with fakes**
+
+Create `backend/internal/services/runtime_scheduler_test.go` with fake repositories and this test:
+
+```go
+func TestRuntimeSchedulerAssignsCreatingInstanceToReadyPod(t *testing.T) {
+ instance := models.Instance{ID: 123, UserID: 45, Type: "openclaw", CPUCores: 2, MemoryGB: 4, DiskGB: 20, RuntimeGeneration: 1}
+ pod := models.RuntimePod{ID: 9, RuntimeType: "openclaw", PodIP: stringPtr("10.42.0.31"), AgentEndpoint: stringPtr("http://10.42.0.31:19090"), State: "ready", Capacity: 100, UsedSlots: 0}
+ s := newRuntimeSchedulerForTest(instance, pod)
+
+ if err := s.assignInstance(context.Background(), instance); err != nil {
+ t.Fatalf("assignInstance returned error: %v", err)
+ }
+ if s.fakeAgent.createGatewayCalls != 1 {
+ t.Fatalf("expected one create gateway call, got %d", s.fakeAgent.createGatewayCalls)
+ }
+ binding := s.fakeBindings.byInstance[123]
+ if binding.GatewayPort != 20017 || binding.RuntimePodID != 9 {
+ t.Fatalf("unexpected binding %#v", binding)
+ }
+}
+```
+
+The fake agent must return:
+
+```go
+&RuntimeAgentCreateGatewayResponse{
+ GatewayID: "gw-123-1",
+ Port: 20017,
+ PID: intPtr(8842),
+ Status: "running",
+}
+```
+
+- [ ] **Step 4: Implement scheduler**
+
+Create `backend/internal/services/runtime_scheduler.go` with:
+
+```go
+package services
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "time"
+
+ "clawreef/internal/models"
+ "clawreef/internal/repository"
+ k8ssvc "clawreef/internal/services/k8s"
+)
+
+type RuntimeScheduler struct {
+ instanceRepo repository.InstanceRepository
+ podRepo repository.RuntimePodRepository
+ bindingRepo repository.InstanceRuntimeBindingRepository
+ rolloutRepo repository.RuntimeRolloutRepository
+ agentClient RuntimeAgentClient
+ events RuntimeEventService
+ leader RuntimeLeaderService
+ deployments k8ssvc.RuntimeDeploymentService
+ tick time.Duration
+}
+
+func NewRuntimeScheduler(
+ instanceRepo repository.InstanceRepository,
+ podRepo repository.RuntimePodRepository,
+ bindingRepo repository.InstanceRuntimeBindingRepository,
+ rolloutRepo repository.RuntimeRolloutRepository,
+ agentClient RuntimeAgentClient,
+ events RuntimeEventService,
+ leader RuntimeLeaderService,
+ deployments k8ssvc.RuntimeDeploymentService,
+ tick time.Duration,
+) *RuntimeScheduler {
+ return &RuntimeScheduler{
+ instanceRepo: instanceRepo,
+ podRepo: podRepo,
+ bindingRepo: bindingRepo,
+ rolloutRepo: rolloutRepo,
+ agentClient: agentClient,
+ events: events,
+ leader: leader,
+ deployments: deployments,
+ tick: tick,
+ }
+}
+
+func (s *RuntimeScheduler) Start(ctx context.Context) {
+ go func() {
+ ticker := time.NewTicker(s.tick)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ if !s.leader.IsLeader(ctx) {
+ continue
+ }
+ if err := s.reconcile(ctx); err != nil {
+ log.Printf("runtime scheduler reconcile failed: %v", err)
+ }
+ }
+ }
+ }()
+}
+
+func (s *RuntimeScheduler) reconcile(ctx context.Context) error {
+ creating, err := s.instanceRepo.GetV2Creating(ctx, 100)
+ if err != nil {
+ return err
+ }
+ for _, instance := range creating {
+ if err := s.assignInstance(ctx, instance); err != nil {
+ msg := err.Error()
+ _ = s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "error", instance.RuntimeGeneration, &msg)
+ }
+ }
+ running, err := s.instanceRepo.GetV2DesiredRunning(ctx, 200)
+ if err != nil {
+ return err
+ }
+ for _, instance := range running {
+ if _, err := s.bindingRepo.GetRunningByInstanceID(ctx, instance.ID); err != nil {
+ if assignErr := s.assignInstance(ctx, instance); assignErr != nil {
+ msg := assignErr.Error()
+ _ = s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "error", instance.RuntimeGeneration, &msg)
+ }
+ }
+ }
+ return nil
+}
+
+func (s *RuntimeScheduler) assignInstance(ctx context.Context, instance models.Instance) error {
+ runtimeType, ok := NormalizeV2RuntimeType(instance.Type)
+ if !ok {
+ return fmt.Errorf("unsupported V2 runtime type %s", instance.Type)
+ }
+ pods, err := s.podRepo.ListSchedulable(ctx, runtimeType)
+ if err != nil {
+ return err
+ }
+ for _, pod := range pods {
+ if pod.AgentEndpoint == nil {
+ continue
+ }
+ claimed, err := s.podRepo.TryClaimSlot(ctx, pod.ID)
+ if err != nil {
+ return err
+ }
+ if !claimed {
+ continue
+ }
+ if err := s.createGatewayOnPod(ctx, instance, pod); err != nil {
+ _ = s.podRepo.ReleaseSlot(ctx, pod.ID)
+ continue
+ }
+ return nil
+ }
+ return fmt.Errorf("no schedulable runtime pod for %s", runtimeType)
+}
+
+func (s *RuntimeScheduler) createGatewayOnPod(ctx context.Context, instance models.Instance, pod models.RuntimePod) error {
+ workspacePath := RuntimeWorkspacePath(pod.RuntimeType, instance.UserID, instance.ID)
+ linuxID := RuntimeLinuxID(instance.ID)
+ resp, err := s.agentClient.CreateGateway(ctx, *pod.AgentEndpoint, RuntimeAgentCreateGatewayRequest{
+ InstanceID: instance.ID,
+ UserID: instance.UserID,
+ AgentType: pod.RuntimeType,
+ WorkspacePath: workspacePath,
+ PortRange: RuntimeAgentPortRange{Start: RuntimeGatewayPortStart, End: RuntimeGatewayPortEnd},
+ UID: linuxID,
+ GID: linuxID,
+ CPUCores: instance.CPUCores,
+ MemoryMB: instance.MemoryGB * 1024,
+ DiskQuotaMB: instance.DiskGB * 1024,
+ Generation: instance.RuntimeGeneration,
+ })
+ if err != nil {
+ return err
+ }
+ binding := &models.InstanceRuntimeBinding{
+ InstanceID: instance.ID,
+ RuntimePodID: pod.ID,
+ RuntimeType: pod.RuntimeType,
+ GatewayID: resp.GatewayID,
+ GatewayPort: resp.Port,
+ GatewayPID: resp.PID,
+ WorkspacePath: workspacePath,
+ State: "running",
+ Generation: instance.RuntimeGeneration,
+ }
+ if err := s.bindingRepo.Create(ctx, binding); err != nil {
+ return err
+ }
+ if err := s.instanceRepo.SetWorkspacePath(ctx, instance.ID, workspacePath); err != nil {
+ return err
+ }
+ return s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "running", instance.RuntimeGeneration, nil)
+}
+```
+
+Add failover and rollout in the same file with these public methods:
+
+```go
+func (s *RuntimeScheduler) DrainPod(ctx context.Context, podID int64) error
+func (s *RuntimeScheduler) FailoverPod(ctx context.Context, podID int64, reason string) error
+func (s *RuntimeScheduler) StartRollout(ctx context.Context, rolloutID int64) error
+```
+
+`FailoverPod` must mark the Pod `unhealthy`, list its bindings, delete each binding, release the old slot, increment instance generation, set instance status `creating`, and let the next scheduler tick recreate the gateway on another Pod.
+
+- [ ] **Step 5: Wire scheduler in `main.go`**
+
+In `backend/cmd/server/main.go`, create the new repositories after the existing repositories:
+
+```go
+runtimePodRepo := repository.NewRuntimePodRepository(sess)
+bindingRepo := repository.NewInstanceRuntimeBindingRepository(sess)
+rolloutRepo := repository.NewRuntimeRolloutRepository(sess)
+```
+
+Create Redis and runtime services:
+
+```go
+platformRedis, err := services.NewPlatformRedisClient(cfg.Runtime.RedisURL)
+if err != nil {
+ log.Printf("platform redis disabled: %v", err)
+}
+runtimeEvents := services.NewRuntimeEventService(platformRedis)
+agentClient := services.NewRuntimeAgentClient(cfg.Runtime.AgentControlToken)
+leader := services.NewRuntimeLeaderService(k8sClient, cfg.Runtime.Namespace, cfg.Runtime.BackendReplicaID)
+runtimeDeployments := k8s.NewRuntimeDeploymentService(k8sClient)
+runtimeScheduler := services.NewRuntimeScheduler(instanceRepo, runtimePodRepo, bindingRepo, rolloutRepo, agentClient, runtimeEvents, leader, runtimeDeployments, cfg.Runtime.SchedulerTick)
+if cfg.Runtime.SchedulerEnabled {
+ runtimeScheduler.Start(ctx)
+}
+```
+
+If `platformRedis` can be nil in local development, make `NewRuntimeEventService` accept nil and return a no-op service that never panics.
+
+- [ ] **Step 6: Run scheduler tests**
+
+Run:
+
+```powershell
+go test ./backend/internal/services -run RuntimeScheduler -count=1
+```
+
+Expected: pass.
+
+- [ ] **Step 7: Commit scheduler**
+
+Run:
+
+```powershell
+git add backend/internal/services backend/cmd/server/main.go
+git commit -m "feat: add runtime scheduler"
+```
+
+## Task 5: V2 Instance Lifecycle And Pod-IP Proxy
+
+**Files:**
+- Modify: `backend/internal/services/instance_service.go`
+- Modify: `backend/internal/services/instance_proxy_service.go`
+- Modify: `backend/internal/services/sync_service.go`
+- Modify: `backend/internal/handlers/instance_handler.go`
+- Modify: `backend/cmd/server/main.go`
+- Create: `backend/internal/services/instance_proxy_service_v2_test.go`
+
+- [ ] **Step 1: Add proxy test for binding target**
+
+Create `backend/internal/services/instance_proxy_service_v2_test.go` with a test that builds an instance, a binding, and a runtime Pod:
+
+```go
+func TestInstanceProxyServiceUsesRuntimeBindingForV2(t *testing.T) {
+ target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/apps/openclaw" {
+ t.Fatalf("unexpected path %s", r.URL.Path)
+ }
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte("ok"))
+ }))
+ defer target.Close()
+
+ podIP, port := splitHostPortForTest(t, target.URL)
+ instance := models.Instance{ID: 123, UserID: 45, Type: "openclaw", Status: "running"}
+ binding := models.InstanceRuntimeBinding{InstanceID: 123, RuntimePodID: 9, GatewayPort: port, State: "running"}
+ pod := models.RuntimePod{ID: 9, PodIP: &podIP, State: "ready"}
+
+ service := newProxyServiceForV2Test(instance, binding, pod)
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/instances/123/proxy/apps/openclaw", nil)
+ rec := httptest.NewRecorder()
+
+ service.Proxy(rec, req, 123, "/apps/openclaw")
+ if rec.Code != http.StatusOK || rec.Body.String() != "ok" {
+ t.Fatalf("unexpected proxy response %d %q", rec.Code, rec.Body.String())
+ }
+}
+```
+
+- [ ] **Step 2: Modify instance creation for V2**
+
+In `backend/internal/services/instance_service.go`, keep legacy behavior for non-V2 types and add this branch near the start of create:
+
+```go
+runtimeType, isV2 := NormalizeV2RuntimeType(req.Type)
+if isV2 {
+ instance := &models.Instance{
+ UserID: userID,
+ Name: strings.TrimSpace(req.Name),
+ Description: normalizeStringPtr(req.Description),
+ Type: runtimeType,
+ RuntimeType: "gateway",
+ Status: "creating",
+ CPUCores: req.CPUCores,
+ MemoryGB: req.MemoryGB,
+ DiskGB: req.DiskGB,
+ StorageClass: strings.TrimSpace(req.StorageClass),
+ MountPath: "/workspaces",
+ RuntimeGeneration: 1,
+ }
+ if err := s.instanceRepo.Create(ctx, instance); err != nil {
+ return nil, err
+ }
+ workspacePath := RuntimeWorkspacePath(runtimeType, userID, instance.ID)
+ if err := os.MkdirAll(workspacePath, 0750); err != nil {
+ return nil, err
+ }
+ if err := s.instanceRepo.SetWorkspacePath(ctx, instance.ID, workspacePath); err != nil {
+ return nil, err
+ }
+ instance.WorkspacePath = &workspacePath
+ return instance, nil
+}
+```
+
+Keep validation so new creation accepts only `openclaw` and `hermes`; old records with `ubuntu`, `webtop`, `debian`, `centos`, or `custom` remain readable and manageable through legacy paths.
+
+- [ ] **Step 3: Modify start, stop, restart, delete**
+
+For V2 instances in `instance_service.go`:
+
+```go
+func (s *instanceService) Start(ctx context.Context, userID int, instanceID int) error {
+ instance, err := s.mustOwnInstance(ctx, userID, instanceID)
+ if err != nil {
+ return err
+ }
+ if _, ok := NormalizeV2RuntimeType(instance.Type); ok {
+ return s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "creating", instance.RuntimeGeneration+1, nil)
+ }
+ return s.startLegacy(ctx, instance)
+}
+
+func (s *instanceService) Stop(ctx context.Context, userID int, instanceID int) error {
+ instance, err := s.mustOwnInstance(ctx, userID, instanceID)
+ if err != nil {
+ return err
+ }
+ if _, ok := NormalizeV2RuntimeType(instance.Type); ok {
+ binding, err := s.bindingRepo.GetByInstanceID(ctx, instance.ID)
+ if err == nil {
+ pod, podErr := s.runtimePodRepo.GetByID(ctx, binding.RuntimePodID)
+ if podErr == nil && pod.AgentEndpoint != nil {
+ _ = s.agentClient.DeleteGateway(ctx, *pod.AgentEndpoint, binding.GatewayID)
+ }
+ _ = s.bindingRepo.DeleteByInstanceID(ctx, instance.ID)
+ _ = s.runtimePodRepo.ReleaseSlot(ctx, binding.RuntimePodID)
+ }
+ return s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "stopped", instance.RuntimeGeneration, nil)
+ }
+ return s.stopLegacy(ctx, instance)
+}
+```
+
+Use the existing legacy code by moving old logic into private helpers `startLegacy`, `stopLegacy`, and `deleteLegacy`.
+
+- [ ] **Step 4: Modify status for ordinary users**
+
+In `backend/internal/handlers/instance_handler.go`, make `/instances/:id/status` return user-safe availability for V2:
+
+```json
+{
+ "instance_status": {
+ "instance_id": 123,
+ "status": "running",
+ "availability": "available",
+ "agent_type": "openclaw",
+ "workspace_usage_bytes": 123456
+ }
+}
+```
+
+Do not include Pod name, namespace, Pod IP, service name, port, capacity, node, or runtime scheduling fields for normal users.
+
+- [ ] **Step 5: Modify proxy for V2**
+
+In `backend/internal/services/instance_proxy_service.go`, resolve target for V2:
+
+```go
+func (s *instanceProxyService) resolveV2Target(ctx context.Context, instanceID int) (*url.URL, error) {
+ binding, err := s.bindingRepo.GetRunningByInstanceID(ctx, instanceID)
+ if err != nil {
+ return nil, fmt.Errorf("instance gateway is not available")
+ }
+ pod, err := s.runtimePodRepo.GetByID(ctx, binding.RuntimePodID)
+ if err != nil {
+ return nil, err
+ }
+ if pod.PodIP == nil || *pod.PodIP == "" {
+ return nil, fmt.Errorf("runtime pod ip is not available")
+ }
+ return &url.URL{Scheme: "http", Host: net.JoinHostPort(*pod.PodIP, strconv.Itoa(binding.GatewayPort))}, nil
+}
+```
+
+Use this target when `NormalizeV2RuntimeType(instance.Type)` succeeds, otherwise execute the existing Service-based proxy path.
+
+- [ ] **Step 6: Run lifecycle and proxy tests**
+
+Run:
+
+```powershell
+go test ./backend/internal/services -run 'Instance|Proxy|RuntimeScheduler' -count=1
+```
+
+Expected: pass.
+
+- [ ] **Step 7: Commit lifecycle and proxy**
+
+Run:
+
+```powershell
+git add backend/internal/services backend/internal/handlers backend/cmd/server/main.go
+git commit -m "feat: route V2 instances through runtime gateways"
+```
+
+## Task 6: Workspace File Manager Backend
+
+**Files:**
+- Create: `backend/internal/services/workspace_path_guard.go`
+- Create: `backend/internal/services/workspace_path_guard_test.go`
+- Create: `backend/internal/services/workspace_file_service.go`
+- Create: `backend/internal/services/workspace_file_service_test.go`
+- Create: `backend/internal/handlers/workspace_file_handler.go`
+- Modify: `backend/internal/utils/response.go`
+- Modify: `backend/cmd/server/main.go`
+
+- [ ] **Step 1: Write path guard tests**
+
+Create `backend/internal/services/workspace_path_guard_test.go`:
+
+```go
+package services
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+func TestResolveWorkspacePathRejectsTraversal(t *testing.T) {
+ root := t.TempDir()
+ _, err := ResolveWorkspacePath(root, "../outside.txt")
+ if err == nil || err.Error() != "workspace path escapes instance workspace" {
+ t.Fatalf("unexpected error %v", err)
+ }
+}
+
+func TestResolveWorkspacePathRejectsSymlinkEscape(t *testing.T) {
+ root := t.TempDir()
+ outside := t.TempDir()
+ if err := os.Symlink(outside, filepath.Join(root, "escape")); err != nil {
+ t.Skipf("symlink unavailable: %v", err)
+ }
+ _, err := ResolveWorkspacePath(root, "escape/file.txt")
+ if err == nil || err.Error() != "workspace path escapes instance workspace" {
+ t.Fatalf("unexpected error %v", err)
+ }
+}
+
+func TestResolveWorkspacePathAllowsNestedFile(t *testing.T) {
+ root := t.TempDir()
+ got, err := ResolveWorkspacePath(root, "a/b.txt")
+ if err != nil {
+ t.Fatalf("ResolveWorkspacePath returned error: %v", err)
+ }
+ want := filepath.Join(root, "a", "b.txt")
+ if got != want {
+ t.Fatalf("want %s got %s", want, got)
+ }
+}
+```
+
+- [ ] **Step 2: Implement path guard**
+
+Create `backend/internal/services/workspace_path_guard.go`:
+
+```go
+package services
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+)
+
+func ResolveWorkspacePath(workspaceRoot, relativePath string) (string, error) {
+ workspaceRoot = filepath.Clean(workspaceRoot)
+ cleanRelative := filepath.Clean("/" + strings.TrimSpace(relativePath))
+ cleanRelative = strings.TrimPrefix(cleanRelative, string(filepath.Separator))
+ fullPath := filepath.Join(workspaceRoot, cleanRelative)
+
+ rootReal, err := filepath.EvalSymlinks(workspaceRoot)
+ if err != nil {
+ return "", err
+ }
+ parent := fullPath
+ if info, statErr := os.Lstat(fullPath); statErr == nil && !info.IsDir() {
+ parent = filepath.Dir(fullPath)
+ }
+ if _, statErr := os.Lstat(parent); statErr == nil {
+ realParent, evalErr := filepath.EvalSymlinks(parent)
+ if evalErr != nil {
+ return "", evalErr
+ }
+ if !isPathInside(rootReal, realParent) {
+ return "", fmt.Errorf("workspace path escapes instance workspace")
+ }
+ }
+ if !isPathInside(workspaceRoot, fullPath) {
+ return "", fmt.Errorf("workspace path escapes instance workspace")
+ }
+ return fullPath, nil
+}
+
+func isPathInside(root, candidate string) bool {
+ rel, err := filepath.Rel(root, candidate)
+ if err != nil {
+ return false
+ }
+ return rel == "." || (!strings.HasPrefix(rel, ".."+string(filepath.Separator)) && rel != "..")
+}
+```
+
+- [ ] **Step 3: Write workspace service tests**
+
+Create `backend/internal/services/workspace_file_service_test.go`:
+
+```go
+func TestWorkspaceFileServicePreviewTextLimit(t *testing.T) {
+ root := t.TempDir()
+ if err := os.WriteFile(filepath.Join(root, "small.txt"), []byte("hello"), 0640); err != nil {
+ t.Fatal(err)
+ }
+ service := NewWorkspaceFileService(fakeAuditRepo{})
+ preview, err := service.Preview(context.Background(), WorkspaceFileScope{InstanceID: 1, UserID: 2, WorkspacePath: root}, "small.txt")
+ if err != nil {
+ t.Fatalf("Preview returned error: %v", err)
+ }
+ if preview.Kind != "text" || preview.Text != "hello" {
+ t.Fatalf("unexpected preview %#v", preview)
+ }
+}
+
+func TestWorkspaceFileServiceRejectsLargeTextPreview(t *testing.T) {
+ root := t.TempDir()
+ large := bytes.Repeat([]byte("a"), 1024*1024+1)
+ if err := os.WriteFile(filepath.Join(root, "large.log"), large, 0640); err != nil {
+ t.Fatal(err)
+ }
+ service := NewWorkspaceFileService(fakeAuditRepo{})
+ _, err := service.Preview(context.Background(), WorkspaceFileScope{InstanceID: 1, UserID: 2, WorkspacePath: root}, "large.log")
+ if err == nil || err.Error() != "text preview exceeds 1 MiB" {
+ t.Fatalf("unexpected error %v", err)
+ }
+}
+```
+
+- [ ] **Step 4: Implement workspace file service**
+
+Create `backend/internal/services/workspace_file_service.go` with these exported types:
+
+```go
+type WorkspaceFileScope struct {
+ InstanceID int
+ UserID int
+ WorkspacePath string
+}
+
+type WorkspaceEntry struct {
+ Name string `json:"name"`
+ Path string `json:"path"`
+ IsDir bool `json:"is_dir"`
+ Size int64 `json:"size"`
+ ModifiedAt time.Time `json:"modified_at"`
+ Previewable bool `json:"previewable"`
+ Downloadable bool `json:"downloadable"`
+}
+
+type WorkspacePreview struct {
+ Kind string `json:"kind"`
+ ContentType string `json:"content_type"`
+ Text string `json:"text,omitempty"`
+ URL string `json:"url,omitempty"`
+}
+
+type WorkspaceFileService interface {
+ List(ctx context.Context, scope WorkspaceFileScope, relativePath string) ([]WorkspaceEntry, error)
+ Preview(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*WorkspacePreview, error)
+ OpenDownload(ctx context.Context, scope WorkspaceFileScope, relativePath string) (*os.File, string, int64, error)
+ Upload(ctx context.Context, scope WorkspaceFileScope, relativeDir string, filename string, reader io.Reader, size int64) error
+ Mkdir(ctx context.Context, scope WorkspaceFileScope, relativePath string) error
+ Rename(ctx context.Context, scope WorkspaceFileScope, oldPath, newPath string) error
+ Delete(ctx context.Context, scope WorkspaceFileScope, relativePath string) error
+}
+```
+
+Rules to implement:
+- Text preview extensions: `.txt`, `.md`, `.json`, `.yaml`, `.yml`, `.log`, `.py`, `.js`, `.ts`, `.go`, `.sh`; max 1 MiB.
+- Image preview extensions: `.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, `.svg`; max 10 MiB.
+- PDF preview uses iframe URL and content type `application/pdf`.
+- Unknown binary files are download-only.
+- Upload max is read from env `WORKSPACE_UPLOAD_MAX_BYTES`, default `524288000`.
+- Audit upload, mkdir, rename, delete, and download. Do not audit preview.
+
+- [ ] **Step 5: Implement handler routes**
+
+Create `backend/internal/handlers/workspace_file_handler.go` with endpoints:
+
+```go
+GET /api/v1/instances/:id/workspace/files
+GET /api/v1/instances/:id/workspace/preview
+GET /api/v1/instances/:id/workspace/download
+POST /api/v1/instances/:id/workspace/upload
+POST /api/v1/instances/:id/workspace/folders
+PATCH /api/v1/instances/:id/workspace/entries
+DELETE /api/v1/instances/:id/workspace/entries
+```
+
+Each handler must load the instance, confirm ownership from auth context, require `WorkspacePath != nil`, and create:
+
+```go
+scope := services.WorkspaceFileScope{
+ InstanceID: instance.ID,
+ UserID: userID,
+ WorkspacePath: *instance.WorkspacePath,
+}
+```
+
+- [ ] **Step 6: Wire workspace routes**
+
+In `backend/cmd/server/main.go`:
+
+```go
+workspaceAuditRepo := repository.NewWorkspaceFileAuditRepository(sess)
+workspaceFileService := services.NewWorkspaceFileService(workspaceAuditRepo)
+workspaceFileHandler := handlers.NewWorkspaceFileHandler(instanceService, workspaceFileService)
+
+instances := api.Group("/instances")
+instances.GET("/:id/workspace/files", workspaceFileHandler.List)
+instances.GET("/:id/workspace/preview", workspaceFileHandler.Preview)
+instances.GET("/:id/workspace/download", workspaceFileHandler.Download)
+instances.POST("/:id/workspace/upload", workspaceFileHandler.Upload)
+instances.POST("/:id/workspace/folders", workspaceFileHandler.Mkdir)
+instances.PATCH("/:id/workspace/entries", workspaceFileHandler.Rename)
+instances.DELETE("/:id/workspace/entries", workspaceFileHandler.Delete)
+```
+
+- [ ] **Step 7: Run workspace backend tests**
+
+Run:
+
+```powershell
+go test ./backend/internal/services ./backend/internal/handlers -run 'Workspace|ResolveWorkspacePath' -count=1
+```
+
+Expected: pass.
+
+- [ ] **Step 8: Commit workspace backend**
+
+Run:
+
+```powershell
+git add backend/internal/services backend/internal/handlers backend/internal/repository backend/internal/utils backend/cmd/server/main.go
+git commit -m "feat: add workspace file APIs"
+```
+
+## Task 7: Runtime Agent Report And Admin APIs
+
+**Files:**
+- Create: `backend/internal/handlers/runtime_agent_handler.go`
+- Create: `backend/internal/handlers/runtime_pool_handler.go`
+- Create: `backend/internal/handlers/runtime_pool_handler_test.go`
+- Modify: `backend/internal/services/websocket_service.go`
+- Modify: `backend/cmd/server/main.go`
+
+- [ ] **Step 1: Implement runtime agent report handler**
+
+Create `backend/internal/handlers/runtime_agent_handler.go` with:
+
+```go
+type RuntimeAgentHandler struct {
+ cfg config.RuntimeConfig
+ podRepo repository.RuntimePodRepository
+ bindingRepo repository.InstanceRuntimeBindingRepository
+ events services.RuntimeEventService
+}
+
+func (h *RuntimeAgentHandler) requireAgentToken(c *gin.Context) bool {
+ if h.cfg.AgentReportToken == "" || c.GetHeader("X-ClawManager-Agent-Token") != h.cfg.AgentReportToken {
+ utils.Unauthorized(c, "invalid runtime agent token")
+ return false
+ }
+ return true
+}
+```
+
+Add handlers:
+- `Register` upserts `runtime_pods`.
+- `Heartbeat` updates state, used slots, draining, and last seen.
+- `ReportMetrics` updates metrics and binding health, then publishes `runtime_pod_metrics` to Redis events.
+- `ReportGateways` reconciles binding state from agent.
+- `ReportSkills` stores skill reports using existing skill service hooks if available; if no hook exists, accept the request and publish a `runtime_agent_skills_reported` event.
+
+- [ ] **Step 2: Implement admin runtime handler**
+
+Create `backend/internal/handlers/runtime_pool_handler.go`:
+
+```go
+type RuntimePoolHandler struct {
+ podRepo repository.RuntimePodRepository
+ bindingRepo repository.InstanceRuntimeBindingRepository
+ rolloutRepo repository.RuntimeRolloutRepository
+ scheduler *services.RuntimeScheduler
+}
+
+func (h *RuntimePoolHandler) ListPods(c *gin.Context)
+func (h *RuntimePoolHandler) GetPodGateways(c *gin.Context)
+func (h *RuntimePoolHandler) DrainPod(c *gin.Context)
+func (h *RuntimePoolHandler) StartRollout(c *gin.Context)
+```
+
+`ListPods` returns:
+
+```json
+{
+ "pods": [
+ {
+ "id": 9,
+ "runtime_type": "openclaw",
+ "pod_name": "openclaw-runtime-abcde",
+ "pod_ip": "10.42.0.31",
+ "node_name": "node-a",
+ "state": "ready",
+ "used_slots": 37,
+ "capacity": 100,
+ "draining": false,
+ "cpu_millis_used": 13600,
+ "memory_bytes_used": 42949672960,
+ "disk_bytes_used": 214748364800,
+ "network_rx_bytes": 9223372,
+ "network_tx_bytes": 19223372,
+ "last_seen_at": "2026-06-01T10:00:00Z"
+ }
+ ]
+}
+```
+
+- [ ] **Step 3: Add admin handler tests**
+
+Create `backend/internal/handlers/runtime_pool_handler_test.go` and verify non-admin access is rejected and admin access returns metrics. Use existing auth middleware test helpers if present; otherwise call the handler directly with `gin.CreateTestContext`.
+
+- [ ] **Step 4: Extend websocket service**
+
+Modify `backend/internal/services/websocket_service.go`:
+
+```go
+type WebSocketTopic string
+
+const (
+ WebSocketTopicUser WebSocketTopic = "user"
+ WebSocketTopicRuntimeAdmin WebSocketTopic = "runtime_admin"
+)
+```
+
+Add subscription filtering:
+
+```go
+type Client struct {
+ UserID int
+ Role string
+ Topic WebSocketTopic
+ Send chan []byte
+}
+
+func (h *Hub) BroadcastRuntimeAdmin(message []byte) {
+ h.mu.RLock()
+ defer h.mu.RUnlock()
+ for client := range h.clients {
+ if client.Role == "admin" && client.Topic == WebSocketTopicRuntimeAdmin {
+ select {
+ case client.Send <- message:
+ default:
+ }
+ }
+ }
+}
+```
+
+Start one goroutine per backend replica to `XREAD` `clawmanager:runtime-events` and broadcast runtime admin messages to local websocket clients.
+
+- [ ] **Step 5: Wire routes**
+
+In `backend/cmd/server/main.go`:
+
+```go
+runtimeAgentHandler := handlers.NewRuntimeAgentHandler(cfg.Runtime, runtimePodRepo, bindingRepo, runtimeEvents)
+api.POST("/runtime-agent/register", runtimeAgentHandler.Register)
+api.POST("/runtime-agent/heartbeat", runtimeAgentHandler.Heartbeat)
+api.POST("/runtime-agent/gateways/report", runtimeAgentHandler.ReportGateways)
+api.POST("/runtime-agent/skills/report", runtimeAgentHandler.ReportSkills)
+api.POST("/runtime-agent/metrics/report", runtimeAgentHandler.ReportMetrics)
+
+runtimePoolHandler := handlers.NewRuntimePoolHandler(runtimePodRepo, bindingRepo, rolloutRepo, runtimeScheduler)
+admin.GET("/runtime-pods", runtimePoolHandler.ListPods)
+admin.GET("/runtime-pods/:id/gateways", runtimePoolHandler.GetPodGateways)
+admin.POST("/runtime-pods/:id/drain", runtimePoolHandler.DrainPod)
+admin.POST("/runtime-rollouts", runtimePoolHandler.StartRollout)
+```
+
+- [ ] **Step 6: Run admin API tests**
+
+Run:
+
+```powershell
+go test ./backend/internal/handlers ./backend/internal/services -run 'RuntimePool|RuntimeAgent|WebSocket' -count=1
+```
+
+Expected: pass.
+
+- [ ] **Step 7: Commit admin runtime APIs**
+
+Run:
+
+```powershell
+git add backend/internal/handlers backend/internal/services backend/cmd/server/main.go
+git commit -m "feat: add runtime admin APIs"
+```
+
+## Task 8: Frontend Simple UI Foundation
+
+**Files:**
+- Modify: `frontend/package.json`
+- Modify: package lock file if present after install.
+- Modify: `frontend/src/index.css`
+- Modify: `frontend/src/components/UserLayout.tsx`
+- Modify: `frontend/src/components/AdminLayout.tsx`
+- Modify: `frontend/src/App.css` if it still injects decorative gradients.
+
+- [ ] **Step 1: Add icon dependency**
+
+Run:
+
+```powershell
+cd frontend
+npm install lucide-react
+cd ..
+```
+
+Expected: `frontend/package.json` and the lock file include `lucide-react`.
+
+- [ ] **Step 2: Replace global visual tokens**
+
+Modify `frontend/src/index.css` so the design uses a neutral, simple base:
+
+```css
+:root {
+ font-family:
+ Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI",
+ sans-serif;
+ color: #111827;
+ background: #f7f8fa;
+ font-synthesis: none;
+ text-rendering: optimizeLegibility;
+ -webkit-font-smoothing: antialiased;
+ -moz-osx-font-smoothing: grayscale;
+}
+
+body {
+ margin: 0;
+ min-width: 320px;
+ min-height: 100vh;
+ background: #f7f8fa;
+}
+
+button,
+input,
+select,
+textarea {
+ font: inherit;
+}
+
+.cm-surface {
+ background: #ffffff;
+ border: 1px solid #e5e7eb;
+ border-radius: 8px;
+}
+
+.cm-button {
+ display: inline-flex;
+ align-items: center;
+ justify-content: center;
+ gap: 6px;
+ min-height: 36px;
+ border-radius: 6px;
+ border: 1px solid #d1d5db;
+ background: #ffffff;
+ color: #111827;
+}
+
+.cm-button-primary {
+ border-color: #2563eb;
+ background: #2563eb;
+ color: #ffffff;
+}
+```
+
+Remove decorative gradient or orb classes from `App.css`.
+
+- [ ] **Step 3: Simplify user layout**
+
+Modify `frontend/src/components/UserLayout.tsx`:
+- Use a plain top bar and left nav.
+- Keep border radius at `rounded-lg` or smaller.
+- Use `lucide-react` icons for nav and command buttons.
+- Remove runtime details from ordinary user navigation labels.
+- Keep language switcher and account actions.
+
+Use this layout shape:
+
+```tsx
+
+```
+
+- [ ] **Step 4: Simplify admin layout**
+
+Modify `frontend/src/components/AdminLayout.tsx` with the same visual system and add a nav entry:
+
+```tsx
+{
+ to: "/admin/runtime-pods",
+ label: "Runtime Pods",
+ icon: Server,
+}
+```
+
+Do not expose admin-only runtime metrics in `UserLayout`.
+
+- [ ] **Step 5: Run frontend lint**
+
+Run:
+
+```powershell
+cd frontend
+npm run lint
+cd ..
+```
+
+Expected: pass.
+
+- [ ] **Step 6: Commit UI foundation**
+
+Run:
+
+```powershell
+git add frontend/package.json frontend/package-lock.json frontend/src/index.css frontend/src/App.css frontend/src/components/UserLayout.tsx frontend/src/components/AdminLayout.tsx
+git commit -m "style: simplify ClawManager UI shell"
+```
+
+## Task 9: Frontend Workspace Manager And V2 User Pages
+
+**Files:**
+- Modify: `frontend/src/types/instance.ts`
+- Create: `frontend/src/types/workspace.ts`
+- Create: `frontend/src/services/workspaceService.ts`
+- Create: `frontend/src/components/WorkspaceFileManager.tsx`
+- Create: `frontend/src/components/InstanceServiceFrame.tsx`
+- Modify: `frontend/src/pages/instances/CreateInstancePage.tsx`
+- Modify: `frontend/src/pages/instances/InstanceListPage.tsx`
+- Modify: `frontend/src/pages/instances/InstanceDetailPage.tsx`
+- Modify: `frontend/src/services/instanceService.ts`
+
+- [ ] **Step 1: Add TypeScript types**
+
+Create `frontend/src/types/workspace.ts`:
+
+```ts
+export interface WorkspaceEntry {
+ name: string;
+ path: string;
+ is_dir: boolean;
+ size: number;
+ modified_at: string;
+ previewable: boolean;
+ downloadable: boolean;
+}
+
+export interface WorkspacePreview {
+ kind: "text" | "image" | "pdf" | "download";
+ content_type: string;
+ text?: string;
+ url?: string;
+}
+```
+
+Modify `frontend/src/types/instance.ts`:
+
+```ts
+export type V2InstanceType = "openclaw" | "hermes";
+export type InstanceAvailability = "available" | "starting" | "unavailable";
+
+export interface InstanceStatus {
+ instance_id: number;
+ status: string;
+ availability?: InstanceAvailability;
+ agent_type?: V2InstanceType;
+ workspace_usage_bytes?: number;
+ created_at?: string;
+ started_at?: string;
+}
+```
+
+Keep legacy optional Pod fields only if existing pages still need them for admin legacy screens.
+
+- [ ] **Step 2: Add workspace service**
+
+Create `frontend/src/services/workspaceService.ts`:
+
+```ts
+import api from "./api";
+import type { WorkspaceEntry, WorkspacePreview } from "../types/workspace";
+
+export const workspaceService = {
+ async list(instanceId: number, path = ""): Promise {
+ const response = await api.get(`/instances/${instanceId}/workspace/files`, {
+ params: { path },
+ });
+ return response.data.data.entries;
+ },
+
+ async preview(instanceId: number, path: string): Promise {
+ const response = await api.get(`/instances/${instanceId}/workspace/preview`, {
+ params: { path },
+ });
+ return response.data.data.preview;
+ },
+
+ downloadUrl(instanceId: number, path: string): string {
+ return `/api/v1/instances/${instanceId}/workspace/download?path=${encodeURIComponent(path)}`;
+ },
+
+ async upload(instanceId: number, path: string, file: File): Promise {
+ const formData = new FormData();
+ formData.append("file", file);
+ await api.post(`/instances/${instanceId}/workspace/upload`, formData, {
+ params: { path },
+ headers: { "Content-Type": "multipart/form-data" },
+ });
+ },
+
+ async mkdir(instanceId: number, path: string): Promise {
+ await api.post(`/instances/${instanceId}/workspace/folders`, { path });
+ },
+
+ async rename(instanceId: number, oldPath: string, newPath: string): Promise {
+ await api.patch(`/instances/${instanceId}/workspace/entries`, {
+ old_path: oldPath,
+ new_path: newPath,
+ });
+ },
+
+ async remove(instanceId: number, path: string): Promise {
+ await api.delete(`/instances/${instanceId}/workspace/entries`, {
+ params: { path },
+ });
+ },
+};
+```
+
+- [ ] **Step 3: Build independent file manager**
+
+Create `frontend/src/components/WorkspaceFileManager.tsx`. It must:
+- Fetch entries with React Query.
+- Show breadcrumb path.
+- Upload via hidden file input.
+- Provide icon buttons for preview, download, rename, delete, and new folder.
+- Show text preview in a compact preformatted pane.
+- Show image/PDF preview in a modal or right panel.
+- Never render an arbitrary path input that can bypass the breadcrumb.
+
+Core state:
+
+```tsx
+const [currentPath, setCurrentPath] = useState("");
+const [previewPath, setPreviewPath] = useState(null);
+const entriesQuery = useQuery({
+ queryKey: ["workspace", instanceId, currentPath],
+ queryFn: () => workspaceService.list(instanceId, currentPath),
+});
+```
+
+- [ ] **Step 4: Build service frame**
+
+Create `frontend/src/components/InstanceServiceFrame.tsx`:
+
+```tsx
+interface InstanceServiceFrameProps {
+ instanceId: number;
+ availability: "available" | "starting" | "unavailable";
+}
+
+export function InstanceServiceFrame({ instanceId, availability }: InstanceServiceFrameProps) {
+ if (availability === "starting") {
+ return Starting
;
+ }
+ if (availability === "unavailable") {
+ return Unavailable
;
+ }
+ return (
+
+ );
+}
+```
+
+- [ ] **Step 5: Restrict create page to V2 types**
+
+Modify `frontend/src/pages/instances/CreateInstancePage.tsx` so the visible type choices are only:
+
+```ts
+const CREATE_INSTANCE_TYPES = INSTANCE_TYPES.filter((type) =>
+ ["openclaw", "hermes"].includes(type.id),
+);
+```
+
+Mark existing non-V2 types as legacy only in admin/history contexts, not in creation.
+
+- [ ] **Step 6: Simplify ordinary user list and detail pages**
+
+Modify `InstanceListPage.tsx` and `InstanceDetailPage.tsx`:
+- Remove service resource, recent status, Pod name, Pod IP, namespace, service, gateway port, scheduler details from ordinary user pages.
+- Show status as `Available`, `Starting`, or `Unavailable`.
+- Show instance type, name, updated time, and workspace usage.
+- Put `InstanceServiceFrame` and `WorkspaceFileManager` in tabs or a split layout.
+- Keep start, stop, restart, delete actions.
+
+Detail page core:
+
+```tsx
+const availability =
+ status?.availability ??
+ (instance.status === "running" ? "available" : instance.status === "creating" ? "starting" : "unavailable");
+
+return (
+
+);
+```
+
+- [ ] **Step 7: Run frontend checks**
+
+Run:
+
+```powershell
+cd frontend
+npm run lint
+npm run build
+cd ..
+```
+
+Expected: both commands pass.
+
+- [ ] **Step 8: Commit user V2 UI**
+
+Run:
+
+```powershell
+git add frontend/src/types frontend/src/services frontend/src/components frontend/src/pages/instances
+git commit -m "feat: add V2 user workspace UI"
+```
+
+## Task 10: Admin Runtime Pods And Rollout UI
+
+**Files:**
+- Create: `frontend/src/types/runtimePool.ts`
+- Create: `frontend/src/services/runtimePoolService.ts`
+- Modify: `frontend/src/hooks/useWebSocket.ts`
+- Create: `frontend/src/pages/admin/RuntimePodsPage.tsx`
+- Modify: `frontend/src/pages/admin/SystemSettingsPage.tsx`
+- Modify: `frontend/src/router/index.tsx`
+- Modify: `frontend/src/components/AdminLayout.tsx`
+
+- [ ] **Step 1: Add runtime pool types**
+
+Create `frontend/src/types/runtimePool.ts`:
+
+```ts
+export interface RuntimePod {
+ id: number;
+ runtime_type: "openclaw" | "hermes";
+ pod_name: string;
+ pod_ip?: string;
+ node_name?: string;
+ state: "pending" | "ready" | "draining" | "unhealthy" | "deleted";
+ used_slots: number;
+ capacity: number;
+ draining: boolean;
+ cpu_millis_used: number;
+ memory_bytes_used: number;
+ disk_bytes_used: number;
+ network_rx_bytes: number;
+ network_tx_bytes: number;
+ last_seen_at?: string;
+}
+
+export interface RuntimeGateway {
+ instance_id: number;
+ gateway_id: string;
+ gateway_port: number;
+ state: string;
+ workspace_path: string;
+ last_health_at?: string;
+}
+
+export interface StartRuntimeRolloutRequest {
+ runtime_type: "openclaw" | "hermes";
+ target_image_ref: string;
+ batch_size: number;
+ max_unavailable: number;
+}
+```
+
+- [ ] **Step 2: Add admin service client**
+
+Create `frontend/src/services/runtimePoolService.ts`:
+
+```ts
+import api from "./api";
+import type { RuntimeGateway, RuntimePod, StartRuntimeRolloutRequest } from "../types/runtimePool";
+
+export const runtimePoolService = {
+ async listPods(runtimeType?: "openclaw" | "hermes"): Promise {
+ const response = await api.get("/admin/runtime-pods", {
+ params: runtimeType ? { runtime_type: runtimeType } : undefined,
+ });
+ return response.data.data.pods;
+ },
+
+ async listGateways(podId: number): Promise {
+ const response = await api.get(`/admin/runtime-pods/${podId}/gateways`);
+ return response.data.data.gateways;
+ },
+
+ async drainPod(podId: number): Promise {
+ await api.post(`/admin/runtime-pods/${podId}/drain`);
+ },
+
+ async startRollout(data: StartRuntimeRolloutRequest): Promise {
+ await api.post("/admin/runtime-rollouts", data);
+ },
+};
+```
+
+- [ ] **Step 3: Create Runtime Pods page**
+
+Modify `frontend/src/hooks/useWebSocket.ts` first:
+
+```ts
+export interface RuntimeAdminMessage {
+ type: "runtime_pod_metrics" | "runtime_pod_state" | "runtime_rollout";
+ data: unknown;
+ timestamp: string;
+}
+
+export function useRuntimeAdminWebSocket(
+ onRuntimeEvent?: (message: RuntimeAdminMessage) => void,
+) {
+ const { addMessageHandler, isConnected } = useWebSocket();
+
+ useEffect(() => {
+ if (!onRuntimeEvent) return;
+
+ const handler = (message: WebSocketMessage) => {
+ if (
+ message.type === "runtime_pod_metrics" ||
+ message.type === "runtime_pod_state" ||
+ message.type === "runtime_rollout"
+ ) {
+ onRuntimeEvent(message as RuntimeAdminMessage);
+ }
+ };
+
+ return addMessageHandler(handler);
+ }, [onRuntimeEvent, addMessageHandler]);
+
+ return { isConnected };
+}
+```
+
+Create `frontend/src/pages/admin/RuntimePodsPage.tsx`. It must:
+- Show segmented filter: All, OpenClaw, Hermes.
+- Show a dense table with Pod, type, state, slots, CPU, memory, disk, network, last seen.
+- Refresh with React Query every 3 seconds.
+- Subscribe to admin runtime websocket through `useRuntimeAdminWebSocket`; invalidate the `["runtime-pods"]` query when a runtime event arrives.
+- Provide a Drain button for each Pod.
+- Show a gateway drawer/table for a selected Pod.
+
+Use simple table layout:
+
+```tsx
+
+```
+
+- [ ] **Step 4: Add route**
+
+Modify `frontend/src/router/index.tsx`:
+
+```tsx
+{
+ path: "runtime-pods",
+ element: ,
+}
+```
+
+under the admin route group.
+
+- [ ] **Step 5: Add gray rollout controls in system settings**
+
+Modify `frontend/src/pages/admin/SystemSettingsPage.tsx`:
+- Add a section named `Runtime Images`.
+- Include only OpenClaw and Hermes runtime image fields for V2.
+- Add `Start gray rollout` action using `runtimePoolService.startRollout`.
+- Inputs: runtime type, target image, batch size default 1, max unavailable default 1.
+- Keep existing image settings if needed for legacy visibility, but visually separate them under `Legacy Images`.
+
+- [ ] **Step 6: Run frontend checks**
+
+Run:
+
+```powershell
+cd frontend
+npm run lint
+npm run build
+cd ..
+```
+
+Expected: pass.
+
+- [ ] **Step 7: Commit admin runtime UI**
+
+Run:
+
+```powershell
+git add frontend/src/types/runtimePool.ts frontend/src/services/runtimePoolService.ts frontend/src/hooks/useWebSocket.ts frontend/src/pages/admin/RuntimePodsPage.tsx frontend/src/pages/admin/SystemSettingsPage.tsx frontend/src/router/index.tsx frontend/src/components/AdminLayout.tsx
+git commit -m "feat: add admin runtime pod view"
+```
+
+## Task 11: One-YAML Deployment Manifests
+
+**Files:**
+- Modify: `deployments/k8s/clawmanager.yaml`
+- Modify: `deployments/k3s/clawmanager.yaml`
+
+- [ ] **Step 1: Rename Redis service while keeping compatibility**
+
+In both manifests:
+- Keep the Redis Deployment selector as a stable label.
+- Add Service `clawmanager-redis`.
+- Keep Service `clawmanager-team-redis` pointing to the same selector.
+- Set backend env:
+
+```yaml
+- name: PLATFORM_REDIS_URL
+ value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0"
+- name: TEAM_REDIS_URL
+ value: "redis://clawmanager-redis.clawmanager-system.svc.cluster.local:6379/0"
+```
+
+- [ ] **Step 2: Add workspace store**
+
+Add a lightweight workspace store Deployment, PVC, and Service:
+
+```yaml
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: clawmanager-workspaces
+ namespace: clawmanager-system
+spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 500Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: workspace-store
+ namespace: clawmanager-system
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: workspace-store
+ template:
+ metadata:
+ labels:
+ app: workspace-store
+ spec:
+ containers:
+ - name: nfs
+ image: itsthenetwork/nfs-server-alpine:12
+ securityContext:
+ privileged: true
+ env:
+ - name: SHARED_DIRECTORY
+ value: /exports/workspaces
+ ports:
+ - name: nfs
+ containerPort: 2049
+ volumeMounts:
+ - name: workspaces
+ mountPath: /exports/workspaces
+ volumes:
+ - name: workspaces
+ persistentVolumeClaim:
+ claimName: clawmanager-workspaces
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: workspace-store
+ namespace: clawmanager-system
+spec:
+ selector:
+ app: workspace-store
+ ports:
+ - name: nfs
+ port: 2049
+ targetPort: nfs
+```
+
+- [ ] **Step 3: Scale backend replicas and add runtime env**
+
+Set backend Deployment:
+
+```yaml
+spec:
+ replicas: 3
+```
+
+Add env:
+
+```yaml
+- name: RUNTIME_NAMESPACE
+ value: "clawmanager-system"
+- name: RUNTIME_WORKSPACE_ROOT
+ value: "/workspaces"
+- name: RUNTIME_MAX_GATEWAYS_PER_POD
+ value: "100"
+- name: RUNTIME_GATEWAY_PORT_START
+ value: "20000"
+- name: RUNTIME_GATEWAY_PORT_END
+ value: "20099"
+- name: RUNTIME_SCHEDULER_ENABLED
+ value: "true"
+- name: RUNTIME_AGENT_CONTROL_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: clawmanager-secrets
+ key: runtime-agent-control-token
+- name: RUNTIME_AGENT_REPORT_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: clawmanager-secrets
+ key: runtime-agent-report-token
+```
+
+Mount `/workspaces` from the NFS service in backend:
+
+```yaml
+volumes:
+ - name: workspaces
+ nfs:
+ server: workspace-store.clawmanager-system.svc.cluster.local
+ path: /exports/workspaces
+```
+
+- [ ] **Step 4: Add runtime Deployments**
+
+Add `openclaw-runtime` and `hermes-runtime` Deployments, each initial `replicas: 1`, each mounting the same NFS workspace and using the matching image env:
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: openclaw-runtime
+ namespace: clawmanager-system
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: openclaw-runtime
+ clawmanager.io/runtime-type: openclaw
+ template:
+ metadata:
+ labels:
+ app: openclaw-runtime
+ clawmanager.io/runtime-type: openclaw
+ spec:
+ containers:
+ - name: runtime
+ image: ghcr.io/yuan-lab-llm/clawmanager-openclaw-image/openclaw:latest
+ ports:
+ - name: agent
+ containerPort: 19090
+ env:
+ - name: CLAWMANAGER_RUNTIME_TYPE
+ value: openclaw
+ - name: CLAWMANAGER_GATEWAY_PORT_START
+ value: "20000"
+ - name: CLAWMANAGER_GATEWAY_PORT_END
+ value: "20099"
+ volumeMounts:
+ - name: workspaces
+ mountPath: /workspaces
+ volumes:
+ - name: workspaces
+ nfs:
+ server: workspace-store.clawmanager-system.svc.cluster.local
+ path: /exports/workspaces
+```
+
+Repeat for `hermes-runtime` with `clawmanager.io/runtime-type: hermes`.
+
+- [ ] **Step 5: Extend RBAC**
+
+Add permissions for:
+
+```yaml
+- apiGroups: ["coordination.k8s.io"]
+ resources: ["leases"]
+ verbs: ["get", "create", "update"]
+- apiGroups: ["apps"]
+ resources: ["deployments", "deployments/scale"]
+ verbs: ["get", "list", "watch", "create", "update", "patch"]
+- apiGroups: [""]
+ resources: ["pods"]
+ verbs: ["get", "list", "watch"]
+```
+
+- [ ] **Step 6: Validate YAML parse**
+
+Run:
+
+```powershell
+kubectl apply --dry-run=client -f deployments/k8s/clawmanager.yaml
+kubectl apply --dry-run=client -f deployments/k3s/clawmanager.yaml
+```
+
+Expected: both commands return `configured (dry run)` or `created (dry run)` lines. If `kubectl` is unavailable, run:
+
+```powershell
+python - <<'PY'
+import yaml
+for path in ["deployments/k8s/clawmanager.yaml", "deployments/k3s/clawmanager.yaml"]:
+ with open(path, "r", encoding="utf-8") as f:
+ docs = [d for d in yaml.safe_load_all(f) if d]
+ print(path, len(docs))
+PY
+```
+
+Expected: both files parse and print a document count.
+
+- [ ] **Step 7: Commit manifests**
+
+Run:
+
+```powershell
+git add deployments/k8s/clawmanager.yaml deployments/k3s/clawmanager.yaml
+git commit -m "feat: install V2 runtime pool components"
+```
+
+## Task 12: Full Verification And Cleanup
+
+**Files:**
+- Modify: `docs/superpowers/specs/2026-06-01-clawmanager-v2-runtime-pool-design.md` only if implementation behavior differs from the accepted spec.
+- Modify: `README.md` or deployment docs if they already contain install steps for the single YAML path.
+
+- [ ] **Step 1: Run full backend tests**
+
+Run:
+
+```powershell
+go test ./backend/... -count=1
+```
+
+Expected: pass.
+
+- [ ] **Step 2: Run full frontend checks**
+
+Run:
+
+```powershell
+cd frontend
+npm run lint
+npm run build
+cd ..
+```
+
+Expected: pass.
+
+- [ ] **Step 3: Run repository-wide status check**
+
+Run:
+
+```powershell
+git status --short
+```
+
+Expected: only intended implementation files are changed since the last task commit.
+
+- [ ] **Step 4: Smoke test local frontend**
+
+Run:
+
+```powershell
+cd frontend
+npm run dev -- --host 127.0.0.1
+```
+
+Open `http://127.0.0.1:9002/` in the in-app browser. Verify:
+- User instance pages do not show Pod IP, Pod name, namespace, service, gateway port, scheduler capacity, or runtime resource internals.
+- User detail page shows whether the instance can be used, the frame, and the workspace file manager.
+- Admin navigation contains Runtime Pods.
+- Runtime Pods page is admin-only and shows Pod status, usage, disk, network, and gateway count.
+- System Settings has V2 runtime image and gray rollout controls.
+
+- [ ] **Step 5: Commit final docs if changed**
+
+Run:
+
+```powershell
+git add README.md docs/superpowers/specs/2026-06-01-clawmanager-v2-runtime-pool-design.md
+git commit -m "docs: align V2 runtime pool implementation notes"
+```
+
+Skip this commit only when neither file changed.
+
+## Self-Review
+
+Spec coverage:
+- Shared runtime Pods with 100 gateways and scale-out at the 101st request: Task 3, Task 4, Task 11.
+- Agent-managed gateway processes and HTTP dual channel: Task 2, Task 7.
+- Pod IP plus port routing instead of Services: Task 5.
+- 5-10 second failover and gray upgrade drain: Task 4, Task 7, Task 10.
+- One YAML install with embedded Redis and workspace store: Task 11.
+- Backend multi-replica with leader election and Redis fanout: Task 3, Task 4, Task 7, Task 11.
+- No VNC; frame directly maps OpenClaw/Hermes services: Task 5, Task 9.
+- Workspace file list, preview, upload, download, rename, delete with path isolation: Task 6, Task 9.
+- Ordinary user UI hides runtime scheduling details: Task 5, Task 8, Task 9.
+- Admin Runtime Pods page with status, usage, disk, network realtime data: Task 7, Task 10.
+- Only OpenClaw and Hermes are creatable in V2; legacy remains readable: Task 5, Task 9.
+
+Placeholder scan:
+- The plan contains concrete file paths, endpoint paths, payloads, method signatures, test names, commands, and expected outcomes.
+- There are no open-ended markers requiring the implementer to invent missing product behavior.
+
+Type consistency:
+- Runtime types are consistently `openclaw` and `hermes`.
+- Runtime Pod capacity is consistently 100.
+- Gateway port range is consistently `20000-20099`.
+- Workspace paths are consistently `/workspaces/{runtime}/user-{user_id}/instance-{instance_id}`.
+- Runtime user and group IDs are consistently `200000 + instance_id`.
diff --git a/docs/superpowers/plans/2026-06-03-clawmanager-v2-e2e-p0.md b/docs/superpowers/plans/2026-06-03-clawmanager-v2-e2e-p0.md
new file mode 100644
index 0000000..5deb062
--- /dev/null
+++ b/docs/superpowers/plans/2026-06-03-clawmanager-v2-e2e-p0.md
@@ -0,0 +1,691 @@
+# ClawManager V2 E2E P0 Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Build the first e2e development stage: the reusable Playwright/TypeScript harness plus the P0 pull-request gate.
+
+**Architecture:** Add a root `e2e/` package that owns Playwright tests, shared fixtures, page objects, lifecycle scripts, Docker MySQL dependencies, logs, reports, and P0 specs. The harness starts isolated MySQL, resets the e2e database, runs the Go backend and Vite frontend, executes Playwright, writes logs/reports, and integrates `ci:p0` into GitHub Actions.
+
+**Tech Stack:** TypeScript, Playwright Test, Node.js, Docker Compose, MySQL 8, Go backend, Vite frontend, GitHub Actions.
+
+---
+
+## Scope
+
+This plan implements Batch 0 and Batch 1 from `docs/superpowers/specs/2026-06-03-clawmanager-v2-e2e-design.md`.
+
+Included:
+- Root `e2e/` package and `package-lock.json`.
+- Reusable TypeScript fixture modules.
+- Shared page objects.
+- Lifecycle scripts with logs.
+- P0 Playwright specs.
+- GitHub Actions `e2e-p0` PR check.
+- Verification by running `npm run ci:p0` locally when dependencies are available.
+
+Excluded:
+- P1 workspace file tests.
+- P2 k8s browser tests.
+- Any changes to production backend or frontend behavior unless a P0 test exposes a blocking issue.
+
+## File Structure
+
+- Create: `e2e/package.json` - e2e scripts and dependencies.
+- Create: `e2e/package-lock.json` - generated by `npm install`.
+- Create: `e2e/playwright.config.ts` - reporters, browser project, output dirs.
+- Create: `e2e/tsconfig.json` - TypeScript config for Playwright and scripts.
+- Create: `e2e/.gitignore` - generated report and state ignores.
+- Create: `e2e/README.md` - local and CI usage.
+- Create: `e2e/docker/docker-compose.e2e.yml` - isolated MySQL service.
+- Create: `e2e/scripts/reset-db.ts` - drop/create the e2e database.
+- Create: `e2e/scripts/wait-for.ts` - reusable TCP/HTTP readiness utilities.
+- Create: `e2e/scripts/run-with-log.ts` - orchestrate dependencies, run Playwright, write `reports/run.log`.
+- Create: `e2e/scripts/e2e-up.ps1` and `e2e/scripts/e2e-down.ps1` - Windows helper scripts.
+- Create: `e2e/scripts/e2e-up.sh` and `e2e/scripts/e2e-down.sh` - Unix helper scripts.
+- Create: `e2e/setup/auth.setup.ts` - write admin storage state.
+- Create: `e2e/setup/db.setup.ts` - register one deterministic runtime pod through API.
+- Create: `e2e/fixtures/env.ts` - typed environment config.
+- Create: `e2e/fixtures/users.ts` - seeded user constants.
+- Create: `e2e/fixtures/apiClient.ts` - shared API client helpers.
+- Create: `e2e/fixtures/test.ts` - shared Playwright test export.
+- Create: `e2e/pages/LoginPage.ts` - login page object.
+- Create: `e2e/pages/AdminRuntimePodsPage.ts` - runtime Pods page object.
+- Create: `e2e/tests/smoke/login.spec.ts` - P0 login UI tests.
+- Create: `e2e/tests/smoke/navigation.spec.ts` - P0 route protection/navigation tests.
+- Create: `e2e/tests/admin/access-control.spec.ts` - P0 admin route access tests.
+- Create: `e2e/tests/admin/runtime-pods.spec.ts` - P0 runtime Pods UI tests.
+- Create: `e2e/tests/api/auth.spec.ts` - P0 auth API tests.
+- Create: `e2e/tests/api/runtime-agent.spec.ts` - P0 runtime-agent API tests.
+- Create: `e2e/reports/.gitkeep` - keep reports directory tracked.
+- Modify: `.github/workflows/build.yml` - add `workflow_dispatch` input and `e2e-p0` job.
+
+## Shared Conventions
+
+Environment values:
+
+```text
+E2E_FRONTEND_URL=http://127.0.0.1:9002
+E2E_BACKEND_URL=http://127.0.0.1:9001/api/v1
+E2E_DB_HOST=127.0.0.1
+E2E_DB_PORT=13307
+E2E_DB_USER=root
+E2E_DB_PASSWORD=123456
+E2E_DB_NAME=clawmanager_e2e
+E2E_RUNTIME_AGENT_TOKEN=e2e-runtime-agent-token
+E2E_WORKSPACE_ROOT=/.cache/e2e/workspaces
+```
+
+Backend runtime env overrides:
+
+```text
+SERVER_ADDRESS=:9001
+DB_HOST=127.0.0.1
+DB_PORT=13307
+DB_USER=root
+DB_PASSWORD=123456
+DB_NAME=clawmanager_e2e
+JWT_SECRET=clawmanager-e2e-secret
+RUNTIME_SCHEDULER_ENABLED=false
+RUNTIME_AGENT_REPORT_TOKEN=e2e-runtime-agent-token
+RUNTIME_WORKSPACE_ROOT=/.cache/e2e/workspaces
+OBJECT_STORAGE_LOCAL_FALLBACK=/.cache/e2e/object-storage
+SKILL_SCANNER_ENABLED=false
+```
+
+## Task 1: Scaffold The E2E Package
+
+**Files:**
+- Create: `e2e/package.json`
+- Create: `e2e/tsconfig.json`
+- Create: `e2e/.gitignore`
+- Create: `e2e/reports/.gitkeep`
+- Create: `e2e/README.md`
+- Generate: `e2e/package-lock.json`
+
+- [ ] **Step 1: Create package metadata and scripts**
+
+Create `e2e/package.json`:
+
+```json
+{
+ "name": "clawmanager-e2e",
+ "private": true,
+ "version": "0.0.0",
+ "type": "module",
+ "scripts": {
+ "test:p0": "tsx scripts/run-with-log.ts p0",
+ "test:p1": "tsx scripts/run-with-log.ts p1",
+ "test:p2": "tsx scripts/run-with-log.ts p2",
+ "test:all": "tsx scripts/run-with-log.ts all",
+ "ci:p0": "tsx scripts/run-with-log.ts p0 --ci",
+ "pw:p0": "playwright test --grep @p0",
+ "pw:p1": "playwright test --grep @p1",
+ "pw:p2": "playwright test --grep @p2",
+ "pw:all": "playwright test",
+ "test:report": "playwright show-report playwright-report",
+ "db:reset": "tsx scripts/reset-db.ts"
+ },
+ "devDependencies": {
+ "@playwright/test": "^1.49.1",
+ "@types/node": "^24.12.0",
+ "mysql2": "^3.11.5",
+ "tsx": "^4.19.2",
+ "typescript": "^5.9.3"
+ }
+}
+```
+
+- [ ] **Step 2: Create TypeScript config**
+
+Create `e2e/tsconfig.json`:
+
+```json
+{
+ "compilerOptions": {
+ "target": "ES2022",
+ "module": "NodeNext",
+ "moduleResolution": "NodeNext",
+ "strict": true,
+ "types": ["node", "@playwright/test"],
+ "esModuleInterop": true,
+ "forceConsistentCasingInFileNames": true,
+ "skipLibCheck": true,
+ "resolveJsonModule": true,
+ "noEmit": true
+ },
+ "include": ["**/*.ts"]
+}
+```
+
+- [ ] **Step 3: Create ignores and report marker**
+
+Create `e2e/.gitignore`:
+
+```gitignore
+node_modules/
+playwright-report/
+test-results/
+reports/*
+!reports/.gitkeep
+.auth/
+.cache/
+```
+
+Create empty file `e2e/reports/.gitkeep`.
+
+- [ ] **Step 4: Create README**
+
+Create `e2e/README.md` with:
+
+```markdown
+# ClawManager E2E
+
+This directory contains Playwright-based TypeScript e2e tests.
+
+## First-Time Setup
+
+```bash
+npm ci
+npx playwright install chromium
+```
+
+## Run Locally
+
+```bash
+npm run test:p0
+npm run test:report
+```
+
+The default local gate is P0. P1, P2, and all tests are explicit:
+
+```bash
+npm run test:p1
+npm run test:p2
+npm run test:all
+```
+
+## Reports
+
+- Live progress appears in the terminal.
+- Run log: `reports/run.log`
+- JSON: `reports/results.json`
+- JUnit: `reports/junit.xml`
+- HTML: `playwright-report/`
+- Raw artifacts: `test-results/`
+```
+
+- [ ] **Step 5: Install dependencies and generate lockfile**
+
+Run:
+
+```powershell
+npm install
+```
+
+Working directory: `e2e`.
+
+Expected: `package-lock.json` is generated and npm exits with code 0.
+
+- [ ] **Step 6: Commit scaffold**
+
+Run:
+
+```powershell
+git add e2e/package.json e2e/package-lock.json e2e/tsconfig.json e2e/.gitignore e2e/README.md e2e/reports/.gitkeep
+git commit -m "test: scaffold e2e package"
+```
+
+## Task 2: Add Environment, API, And Page Object Modules
+
+**Files:**
+- Create: `e2e/fixtures/env.ts`
+- Create: `e2e/fixtures/users.ts`
+- Create: `e2e/fixtures/apiClient.ts`
+- Create: `e2e/fixtures/test.ts`
+- Create: `e2e/pages/LoginPage.ts`
+- Create: `e2e/pages/AdminRuntimePodsPage.ts`
+
+- [ ] **Step 1: Create environment helper**
+
+Create `e2e/fixtures/env.ts` with typed parsing for URLs, database config, runtime token, workspace root, and CI flag. It must export `env`, `repoRoot`, `e2eRoot`, and `reportsDir`.
+
+- [ ] **Step 2: Create user constants**
+
+Create `e2e/fixtures/users.ts`:
+
+```typescript
+export const users = {
+ admin: {
+ username: "admin",
+ password: "admin123",
+ role: "admin",
+ storageState: ".auth/admin.json"
+ }
+} as const;
+```
+
+- [ ] **Step 3: Create API client helper**
+
+Create `e2e/fixtures/apiClient.ts` with helpers:
+
+```typescript
+export interface LoginTokens {
+ access_token: string;
+ refresh_token: string;
+ expires_in: number;
+}
+
+export async function login(request: APIRequestContext, user = users.admin): Promise;
+export async function getMe(request: APIRequestContext, accessToken: string): Promise<{ username: string; role: string }>;
+export async function registerRuntimePod(request: APIRequestContext, overrides?: Partial): Promise;
+export async function sendRuntimeHeartbeat(request: APIRequestContext, identity: RuntimePodIdentity, overrides?: Partial): Promise;
+export async function listRuntimePods(request: APIRequestContext, accessToken: string): Promise;
+```
+
+The implementation must centralize the `X-ClawManager-Agent-Token` header and never log token values.
+
+- [ ] **Step 4: Create shared Playwright export**
+
+Create `e2e/fixtures/test.ts`:
+
+```typescript
+import { test as base, expect } from "@playwright/test";
+import { env } from "./env";
+
+export const test = base.extend({
+ baseURL: async ({}, use) => {
+ await use(env.frontendUrl);
+ }
+});
+
+export { expect };
+```
+
+- [ ] **Step 5: Create page objects**
+
+Create `e2e/pages/LoginPage.ts` with `goto()`, `login(username, password)`, and `expectVisible()`.
+
+Create `e2e/pages/AdminRuntimePodsPage.ts` with `goto()`, `expectEmptyState()`, and `expectPodVisible(podName)`.
+
+- [ ] **Step 6: Type-check helper modules**
+
+Run:
+
+```powershell
+npx tsc --noEmit
+```
+
+Working directory: `e2e`.
+
+Expected: TypeScript exits with code 0.
+
+- [ ] **Step 7: Commit reusable modules**
+
+Run:
+
+```powershell
+git add e2e/fixtures e2e/pages
+git commit -m "test: add reusable e2e modules"
+```
+
+## Task 3: Add Docker And Lifecycle Scripts
+
+**Files:**
+- Create: `e2e/docker/docker-compose.e2e.yml`
+- Create: `e2e/scripts/reset-db.ts`
+- Create: `e2e/scripts/wait-for.ts`
+- Create: `e2e/scripts/run-with-log.ts`
+- Create: `e2e/scripts/e2e-up.ps1`
+- Create: `e2e/scripts/e2e-down.ps1`
+- Create: `e2e/scripts/e2e-up.sh`
+- Create: `e2e/scripts/e2e-down.sh`
+
+- [ ] **Step 1: Add MySQL compose file**
+
+Create `e2e/docker/docker-compose.e2e.yml` with a MySQL 8 service named `mysql`, host port `13307`, root password `123456`, database `clawmanager_e2e`, and a healthcheck using `mysqladmin ping`.
+
+- [ ] **Step 2: Add database reset script**
+
+Create `e2e/scripts/reset-db.ts` that connects to MySQL without selecting a database, drops `clawmanager_e2e`, creates it with `utf8mb4_unicode_ci`, and exits with a non-zero code on failure.
+
+- [ ] **Step 3: Add readiness helpers**
+
+Create `e2e/scripts/wait-for.ts` exporting `waitForTcpPort`, `waitForHttpStatus`, and `sleep`.
+
+- [ ] **Step 4: Add run-with-log orchestrator**
+
+Create `e2e/scripts/run-with-log.ts` that:
+
+1. Parses suite argument `p0`, `p1`, `p2`, or `all`.
+2. Creates `reports/run.log`.
+3. Starts Docker Compose MySQL.
+4. Runs `reset-db.ts`.
+5. Starts backend with e2e env.
+6. Waits for backend port `9001`.
+7. Starts frontend on `127.0.0.1:9002`.
+8. Waits for frontend HTTP 200.
+9. Runs `npx playwright test` with the requested suite.
+10. Writes terminal output to both console and `reports/run.log`.
+11. Stops backend, frontend, and Docker Compose in `finally`.
+
+- [ ] **Step 5: Add platform helper scripts**
+
+Create PowerShell and shell wrappers that run Docker Compose up/down for local manual control:
+
+```powershell
+docker compose -f docker/docker-compose.e2e.yml up -d mysql
+```
+
+```bash
+docker compose -f docker/docker-compose.e2e.yml up -d mysql
+```
+
+The down scripts should run Docker Compose `down --remove-orphans`.
+
+- [ ] **Step 6: Verify DB reset can run**
+
+Run:
+
+```powershell
+docker compose -f docker/docker-compose.e2e.yml up -d mysql
+npm run db:reset
+docker compose -f docker/docker-compose.e2e.yml down --remove-orphans
+```
+
+Working directory: `e2e`.
+
+Expected: database reset succeeds.
+
+- [ ] **Step 7: Commit lifecycle scripts**
+
+Run:
+
+```powershell
+git add e2e/docker e2e/scripts
+git commit -m "test: add e2e lifecycle scripts"
+```
+
+## Task 4: Add Playwright Configuration And Setup Projects
+
+**Files:**
+- Create: `e2e/playwright.config.ts`
+- Create: `e2e/setup/auth.setup.ts`
+- Create: `e2e/setup/db.setup.ts`
+
+- [ ] **Step 1: Add Playwright config**
+
+Create `e2e/playwright.config.ts` with:
+
+- `testDir: "./tests"`
+- reporters: `list`, `html`, `json`, `junit`
+- output dir `test-results`
+- HTML report dir `playwright-report`
+- retries `1` in CI, `0` locally
+- Chromium project
+- setup projects `auth setup` and `db setup`
+- storage state path `.auth/admin.json` for UI projects that require admin auth
+
+- [ ] **Step 2: Add auth setup**
+
+Create `e2e/setup/auth.setup.ts`. It should login through the UI using `LoginPage`, verify localStorage has `access_token`, and write `.auth/admin.json`.
+
+- [ ] **Step 3: Add runtime pod setup**
+
+Create `e2e/setup/db.setup.ts`. It should call `registerRuntimePod` once with pod name `e2e-openclaw-runtime-p0`, namespace `clawmanager-e2e`, runtime type `openclaw`, state `ready`, capacity `100`, used slots `0`, and image `e2e/openclaw:latest`.
+
+- [ ] **Step 4: Verify Playwright list command**
+
+Run:
+
+```powershell
+npx playwright test --list
+```
+
+Working directory: `e2e`.
+
+Expected: setup projects are listed and no config parse error occurs.
+
+- [ ] **Step 5: Commit config and setup**
+
+Run:
+
+```powershell
+git add e2e/playwright.config.ts e2e/setup
+git commit -m "test: configure e2e playwright setup"
+```
+
+## Task 5: Add P0 API Tests
+
+**Files:**
+- Create: `e2e/tests/api/auth.spec.ts`
+- Create: `e2e/tests/api/runtime-agent.spec.ts`
+
+- [ ] **Step 1: Add auth API tests**
+
+Create `auth.spec.ts` with P0 tests:
+
+- `@p0 auth login returns tokens`
+- `@p0 auth me returns current user with valid token`
+- `@p0 auth me rejects unauthenticated requests`
+
+Use `apiClient.login` and `apiClient.getMe`.
+
+- [ ] **Step 2: Add runtime-agent API tests**
+
+Create `runtime-agent.spec.ts` with P0 tests:
+
+- `@p0 runtime-agent register persists an openclaw runtime pod`
+- `@p0 runtime-agent heartbeat updates pod health state`
+- `@p0 runtime-agent register rejects unsupported runtime type`
+
+Use `apiClient.registerRuntimePod` and `apiClient.sendRuntimeHeartbeat`.
+
+- [ ] **Step 3: Run P0 API tests**
+
+Run:
+
+```powershell
+npm run test:p0 -- --grep "auth|runtime-agent"
+```
+
+Working directory: `e2e`.
+
+Expected: API tests pass after services start.
+
+- [ ] **Step 4: Commit API tests**
+
+Run:
+
+```powershell
+git add e2e/tests/api
+git commit -m "test: add p0 e2e api coverage"
+```
+
+## Task 6: Add P0 UI Tests
+
+**Files:**
+- Create: `e2e/tests/smoke/login.spec.ts`
+- Create: `e2e/tests/smoke/navigation.spec.ts`
+- Create: `e2e/tests/admin/access-control.spec.ts`
+- Create: `e2e/tests/admin/runtime-pods.spec.ts`
+
+- [ ] **Step 1: Add login smoke tests**
+
+Create tests for:
+
+- `@p0 login page renders`
+- `@p0 admin can log in`
+- `@p0 failed login shows an error`
+
+Use `LoginPage`.
+
+- [ ] **Step 2: Add navigation tests**
+
+Create tests for:
+
+- `@p0 unauthenticated users are redirected to login`
+- `@p0 authenticated admin reaches admin dashboard`
+- `@p0 authenticated admin can reach instances and runtime pods routes`
+
+Use storage state from `.auth/admin.json`.
+
+- [ ] **Step 3: Add access control tests**
+
+Create tests for:
+
+- `@p0 unauthenticated admin route redirects to login`
+- `@p0 admin can open runtime pods route`
+
+- [ ] **Step 4: Add runtime Pods UI tests**
+
+Create tests for:
+
+- `@p0 admin runtime pods page renders no runtime pods when unseeded`
+- `@p0 admin runtime pods page renders seeded runtime pod`
+
+If setup always seeds one pod, implement the empty-state test by directly clearing runtime Pods through `db.setup.ts` helper before visiting the page, then reseed for the seeded-state test.
+
+- [ ] **Step 5: Run P0 UI tests**
+
+Run:
+
+```powershell
+npm run test:p0 -- --grep "login|navigation|runtime pods|access"
+```
+
+Working directory: `e2e`.
+
+Expected: UI tests pass after services start.
+
+- [ ] **Step 6: Commit UI tests**
+
+Run:
+
+```powershell
+git add e2e/tests/smoke e2e/tests/admin
+git commit -m "test: add p0 e2e ui coverage"
+```
+
+## Task 7: Add GitHub Actions E2E Job
+
+**Files:**
+- Modify: `.github/workflows/build.yml`
+
+- [ ] **Step 1: Add workflow dispatch input**
+
+Modify `.github/workflows/build.yml` to add `workflow_dispatch` with `e2e_suite` choices `p0`, `p1`, `p2`, and `all`, defaulting to `p0`.
+
+- [ ] **Step 2: Add e2e-p0 job**
+
+Add job `e2e-p0`:
+
+- `runs-on: ubuntu-latest`
+- `needs: [frontend, backend]`
+- checkout
+- setup Node 20 with cache for `e2e/package-lock.json`
+- setup Go using `backend/go.mod`
+- install frontend deps with `npm ci` in `frontend`
+- install e2e deps with `npm ci` in `e2e`
+- install Chromium with `npx playwright install --with-deps chromium`
+- run selected suite. For PRs use `npm run ci:p0`; for manual dispatch map `e2e_suite` to the requested command.
+- upload `e2e/playwright-report`, `e2e/test-results`, and `e2e/reports` on failure.
+
+- [ ] **Step 3: Validate workflow syntax**
+
+Run:
+
+```powershell
+git diff -- .github/workflows/build.yml
+```
+
+Expected: e2e job is present, YAML indentation is valid on visual inspection, existing jobs are untouched except for workflow triggers.
+
+- [ ] **Step 4: Commit GitHub Actions integration**
+
+Run:
+
+```powershell
+git add .github/workflows/build.yml
+git commit -m "ci: add p0 e2e pr check"
+```
+
+## Task 8: Full P0 Verification
+
+**Files:**
+- Verify all files changed in Tasks 1-7.
+
+- [ ] **Step 1: Run TypeScript check**
+
+Run:
+
+```powershell
+npx tsc --noEmit
+```
+
+Working directory: `e2e`.
+
+Expected: exits with code 0.
+
+- [ ] **Step 2: Run P0 suite**
+
+Run:
+
+```powershell
+npm run ci:p0
+```
+
+Working directory: `e2e`.
+
+Expected: P0 tests pass, `reports/run.log`, `reports/results.json`, `reports/junit.xml`, and `playwright-report/` are generated.
+
+- [ ] **Step 3: Verify report opens**
+
+Run:
+
+```powershell
+npm run test:report -- --host 127.0.0.1
+```
+
+Expected: Playwright report server starts. Stop it after confirming no report generation error.
+
+- [ ] **Step 4: Run repository build checks affected by CI**
+
+Run:
+
+```powershell
+npm run build
+```
+
+Working directory: `frontend`.
+
+Run:
+
+```powershell
+go test ./...
+```
+
+Working directory: `backend`.
+
+Expected: both commands exit with code 0, or any pre-existing failures are reported with exact output.
+
+- [ ] **Step 5: Final status review**
+
+Run:
+
+```powershell
+git status --short
+git log --oneline -5
+```
+
+Expected: only intended files remain changed or all e2e work is committed. Existing unrelated dirty files from before this work remain untouched and unstaged.
+
+## Self-Review Checklist
+
+- Every P0 requirement in the spec maps to a test file.
+- Repeated login and API logic is centralized.
+- Runtime-agent payload construction exists in one helper.
+- URLs and tokens come from `fixtures/env.ts`.
+- Report artifacts match the spec paths.
+- GitHub PR check uses `npm run ci:p0`.
+- Local manual commands are documented.
+- No production frontend/backend code is changed for test convenience.
diff --git a/docs/superpowers/plans/2026-06-04-lite-pro-runtime-rework.md b/docs/superpowers/plans/2026-06-04-lite-pro-runtime-rework.md
new file mode 100644
index 0000000..2b89e54
--- /dev/null
+++ b/docs/superpowers/plans/2026-06-04-lite-pro-runtime-rework.md
@@ -0,0 +1,328 @@
+# Lite / Pro Runtime Rework Plan
+
+## Goal
+
+Rework ClawManager instance modes so users choose between:
+
+- `lite`: shared runtime Pod plus per-instance gateway process.
+- `pro`: dedicated desktop Deployment plus Service plus PVC.
+
+Both modes must keep the same user-facing product capabilities: instance access, AI Gateway, channel management, skill management, workspace import/export, and admin operations. The modes differ only in runtime backend, scheduling, and resource/capacity controls.
+
+## Progress Protocol
+
+Implementation should update this plan as each phase completes. Each phase must include:
+
+- Files changed.
+- Tests or checks run.
+- Known gaps or follow-up work.
+- Whether user-visible behavior changed.
+
+Do not revert unrelated existing workspace changes.
+
+## Architecture Decisions
+
+- User-facing mode is `instance_mode`: `lite` or `pro`.
+- Internal backend remains `runtime_type`: `gateway`, `desktop`, or `shell`.
+- `lite` always maps to `runtime_type = gateway`.
+- `pro` always maps to `runtime_type = desktop`.
+- Frontend should show Lite/Pro and avoid exposing gateway/desktop as ordinary user choices.
+- Scheduler must manage only Lite instances.
+- Dedicated K8s lifecycle must manage only Pro and shell instances.
+- External access should still proxy through ClawManager, not directly expose Pod IP or runtime Pod gateway ports.
+- AI Gateway telemetry must include instance-level and gateway-level identity for Lite so multiple Lite instances on one runtime Pod never mix audit, cost, risk, or trace records.
+
+## Phase 0: Planning
+
+- [x] Create this implementation plan.
+- [x] Confirm dirty worktree impact before each implementation phase.
+
+## Phase 1: Instance Mode Data Model And Routing Boundary
+
+### Backend
+
+- [x] Add migration for `instances.instance_mode ENUM('lite','pro') NOT NULL DEFAULT 'lite'`.
+- [x] Add `InstanceMode` to `models.Instance`.
+- [x] Add constants and helpers:
+ - `InstanceModeLite`
+ - `InstanceModePro`
+ - `RuntimeTypeGateway`
+ - `RuntimeTypeDesktop`
+ - `NormalizeInstanceMode`
+ - `RuntimeTypeForInstanceMode`
+ - `InstanceModeForRuntimeType`
+- [x] Extend create request validation to accept `mode` or `instance_mode`.
+- [x] Preserve backward compatibility:
+ - If request has `runtime_type=gateway`, infer `lite`.
+ - If request has `runtime_type=desktop`, infer `pro`.
+ - If neither is present, default to `lite` for OpenClaw/Hermes unless system config says otherwise.
+- [x] Change create flow:
+ - Lite -> existing gateway-pool creation path.
+ - Pro -> existing dedicated runtime creation path.
+- [x] Change `v2RuntimeTypeForInstance` to return true only for Lite/gateway instances.
+- [x] Change scheduler selectors to require `runtime_type='gateway'` and `instance_mode='lite'`.
+- [x] Change sync service to ignore Lite/gateway and continue managing Pro/shell dedicated workloads.
+
+### Frontend
+
+- [x] Add `mode` / `instance_mode` type fields.
+- [x] Show Lite/Pro selection on create page.
+- [x] Submit `mode` to backend.
+- [ ] Keep runtime image selection compatible with Pro desktop images and Lite runtime pool images.
+
+### Tests
+
+- [x] Unit test Lite create produces `instance_mode=lite`, `runtime_type=gateway`, workspace path.
+- [ ] Unit test Pro create produces `instance_mode=pro`, `runtime_type=desktop`, and does not enter gateway scheduler.
+- [x] Unit test scheduler ignores Pro instances even if workspace_path exists.
+
+## Phase 2: Pro Dedicated Deployment + Service + PVC
+
+### K8s Services
+
+- [x] Add per-instance deployment service, separate from shared runtime pool deployment service.
+- [x] Build Deployment from the existing PodConfig shape:
+ - labels: `instance-id`, `instance-name`, `user-id`, `instance-type`, `runtime-type=desktop`, `managed-by=clawreef`
+ - selector should be immutable and match Service selector.
+ - container name remains `desktop`.
+ - probes, resources, env, envFrom, PVC mounts, config map mounts, SHM, and security mode should match current Pod behavior.
+- [x] Add operations:
+ - Ensure Deployment.
+ - Scale Deployment to 1.
+ - Scale Deployment to 0.
+ - Delete Deployment.
+ - Restart/Rollout restart.
+ - Get Deployment status.
+ - Get active Pod for instance.
+
+### Instance Lifecycle
+
+- [x] Pro create: PVC -> Deployment(replicas=1) -> Service.
+- [x] Pro start: scale Deployment to 1 and ensure Service.
+- [x] Pro stop: scale Deployment to 0, retain PVC and Service.
+- [x] Pro restart: rollout restart or scale 0 -> 1.
+- [x] Pro delete: delete Deployment, Service, PVC, Secrets, ConfigMaps.
+- [x] Sync Pro status from Deployment available replicas instead of raw Pod phase.
+
+### Tests
+
+- [x] K8s fake-client tests for BuildInstanceDeployment.
+- [ ] Lifecycle tests for Pro start/stop/delete paths.
+- [ ] Proxy tests still resolve Pro through stable Service.
+
+## Phase 3: Mode-Specific Capacity And Resource Limits
+
+- [x] Add system-level mode limits:
+ - Lite runtime pod count / gateways per pod / max active instances.
+ - Pro capacity / max active instances / max CPU / max memory / max storage / max GPU.
+- [ ] Consider user-level overrides after system-level support is stable.
+- [x] Enforce Lite capacity before accepting Lite create/start.
+- [x] Enforce Pro capacity before accepting Pro create/start.
+- [x] Treat `pro_capacity = 0` as Pro create/start disabled, while keeping existing Pro instances visible and deletable.
+- [ ] Add admin API to read and update mode limits.
+- [x] Add tests for capacity edge cases.
+
+## Phase 4: External Instance Access
+
+### Data And API
+
+- [x] Add `instance_external_access` table:
+ - `instance_id`
+ - `enabled`
+ - `auth_mode`
+ - `public_slug`
+ - `public_token_hash`
+ - `api_key_hash`
+ - `api_key_prefix`
+ - `expires_at`
+ - `created_by`
+ - `last_used_at`
+ - timestamps
+- [x] Owner APIs:
+ - Enable public link.
+ - Disable external access.
+ - Rotate public link by enabling again.
+ - Create API key.
+ - Revoke current API key by disabling external access.
+ - Set expiration.
+- [ ] Admin APIs:
+ - View external access state.
+ - Revoke external access.
+ - Configure global policy: disabled, api-key-only, public-link-allowed.
+
+### Proxy
+
+- [x] Add public route such as `/public/instances/{slug}/proxy/`.
+- [x] Authenticate by capability URL token or API key.
+- [x] Resolve target with the same Lite/Pro proxy routing:
+ - Lite -> runtime binding and gateway port.
+ - Pro -> stable Service.
+- [ ] Record access audit:
+ - instance_id
+ - instance_mode
+ - runtime_type
+ - gateway_id for Lite
+ - api_key prefix/id if applicable
+ - source IP
+ - user agent
+
+### Tests
+
+- [x] Public link succeeds when enabled and unexpired.
+- [x] Public link fails after expiration.
+- [x] API key succeeds and disabled key fails.
+- [ ] Public link fails after disable/rotate.
+- [ ] Lite and Pro proxy targets remain separated.
+
+### Phase 4 Progress Notes
+
+- Files changed:
+ - `backend/internal/db/migrations/027_add_instance_external_access.sql`
+ - `backend/internal/models/instance_external_access.go`
+ - `backend/internal/repository/instance_external_access_repository.go`
+ - `backend/internal/services/instance_external_access_service.go`
+ - `backend/internal/services/instance_external_access_service_test.go`
+ - `backend/internal/handlers/instance_handler.go`
+ - `backend/cmd/server/main.go`
+- Tests/checks run:
+ - `cd backend; $env:GOCACHE='D:\code\github\ClawManagerV2\.cache\go-build'; go test ./internal/services/k8s ./internal/services ./internal/repository ./internal/handlers -run "(InstanceExternalAccess|ExternalAccess|InstanceDeployment|RuntimeDeployment|InstanceMode|V2|Scheduler|BuildV2|UserSafe|WorkspaceArchive|RuntimeLinuxID|RuntimeWorkspacePath|NormalizeV2RuntimeType|Proxy)" -count=1`
+- Known gaps:
+ - Admin external access policy/revoke APIs are still pending.
+ - Public access audit payload is still pending.
+ - Public proxy separation needs a handler/proxy-level regression test.
+- User-visible behavior:
+ - Instance owners can enable public-link or API-key external access.
+ - Public routes proxy through ClawManager using the existing Lite/Pro proxy resolver.
+
+## Phase 5: AI Gateway Lite Gateway-Level Attribution
+
+- [x] Extend gateway auth context to include:
+ - `instance_id`
+ - `instance_mode`
+ - `runtime_type`
+ - `gateway_id`
+ - `runtime_pod_id`
+ - `user_id`
+- [x] Ensure Lite gateway create injects identity into environment/token.
+- [x] Ensure each Lite gateway has a distinct token or claims set.
+- [x] Extend model invocation, audit event, cost record, and risk hit payloads where needed.
+- [x] Add tests proving gateway attribution fields are populated and stripped before provider calls.
+- [ ] Add integration/e2e test proving two Lite instances on one runtime Pod produce separate invocation records.
+
+### Phase 5 Progress Notes
+
+- Files changed:
+ - `backend/internal/middleware/auth_middleware.go`
+ - `backend/internal/handlers/ai_gateway_handler.go`
+ - `backend/internal/aigateway/service.go`
+ - `backend/internal/aigateway/service_test.go`
+ - `backend/internal/models/model_invocation.go`
+ - `backend/internal/models/audit_event.go`
+ - `backend/internal/models/cost_record.go`
+ - `backend/internal/models/risk_hit.go`
+ - `backend/internal/repository/model_invocation_repository.go`
+ - `backend/internal/repository/audit_event_repository.go`
+ - `backend/internal/repository/cost_record_repository.go`
+ - `backend/internal/repository/risk_hit_repository.go`
+ - `backend/internal/db/migrations/028_add_ai_gateway_runtime_attribution.sql`
+- Tests/checks run:
+ - `cd backend; $env:GOCACHE='D:\code\github\ClawManagerV2\.cache\go-build'; go test ./internal/aigateway ./internal/services/k8s ./internal/services ./internal/repository ./internal/handlers ./internal/middleware -run "(RuntimeAttribution|BuildProvider|InstanceExternalAccess|ExternalAccess|InstanceDeployment|RuntimeDeployment|InstanceMode|V2|Scheduler|BuildV2|UserSafe|WorkspaceArchive|RuntimeLinuxID|RuntimeWorkspacePath|NormalizeV2RuntimeType|Proxy|GatewayAuth)" -count=1`
+- Known gaps:
+ - Full multi-gateway e2e attribution proof is still pending.
+- User-visible behavior:
+ - Admin AI audit/cost records can carry Lite gateway identity instead of only instance ID.
+
+## Phase 6: Unified Channel, Skill, Import/Export For Lite And Pro
+
+- [x] Treat channel and skill management as instance capabilities, not runtime backend features.
+- [x] Lite:
+ - Apply config through runtime agent to gateway.
+ - Inventory reports include `instance_id` and `gateway_id`.
+ - Import/export through shared workspace backend.
+- [x] Pro:
+ - Apply config through instance agent/control plane.
+ - Inventory reports include `instance_id`.
+ - Import/export through dedicated PVC workspace backend or instance agent.
+- [x] Add a workspace backend abstraction:
+ - Lite shared workspace path.
+ - Pro dedicated PVC/workspace path.
+- [x] Keep existing OpenClaw/Hermes import/export API shape.
+- [x] Add tests for Lite and Pro import/export workspace selection.
+
+### Phase 6 Progress Notes
+
+- Files changed:
+ - `backend/internal/services/openclaw_transfer_service.go`
+ - `backend/internal/services/openclaw_transfer_service_test.go`
+ - `backend/cmd/server/main.go`
+- Tests/checks run:
+ - `cd backend; $env:GOCACHE='D:\code\github\ClawManagerV2\.cache\go-build'; go test ./internal/aigateway ./internal/services/k8s ./internal/services ./internal/repository ./internal/handlers ./internal/middleware -run "(RuntimeAttribution|BuildProvider|OpenClawTransfer|WorkspaceSpec|InstanceExternalAccess|ExternalAccess|InstanceDeployment|RuntimeDeployment|InstanceMode|V2|Scheduler|BuildV2|UserSafe|WorkspaceArchive|RuntimeLinuxID|RuntimeWorkspacePath|NormalizeV2RuntimeType|Proxy|GatewayAuth)" -count=1`
+- Known gaps:
+ - Real runtime import/export smoke test still needs a running cluster.
+- User-visible behavior:
+ - Lite and Pro keep the same OpenClaw/Hermes import/export endpoints.
+ - Lite import/export targets its per-instance workspace inside the shared runtime pod.
+
+## Phase 7: Admin Monitoring And Management
+
+### Admin Instances
+
+- [x] Show mode: Lite/Pro.
+- [x] Show runtime backend: gateway/desktop for admins only.
+- [ ] Show external access status.
+- [x] Show AI usage/cost/risk by instance through existing observability surfaces with new attribution fields.
+- [x] Support start, stop, restart, delete, sync for Lite/Pro instances.
+- [ ] Support export, logs, revoke external access directly from admin list.
+
+### Runtime Pools
+
+- [x] Keep Lite runtime Pod view:
+ - runtime Pod status.
+ - used/capacity.
+ - gateway distribution.
+ - drain/migrate/failover.
+- [ ] Add Pro capacity view:
+ - pro_capacity.
+ - running Pro instances.
+ - Deployment status.
+ - Service/PVC summary.
+ - resource usage.
+- [x] Add Pro running/creating/total summary in admin instance page.
+
+### Tests
+
+- [x] Frontend build and targeted lint for changed files.
+- [x] Admin instance page covers both Lite and Pro.
+- [ ] E2E smoke navigation includes mode labels and admin operations.
+
+### Phase 7 Progress Notes
+
+- Files changed:
+ - `frontend/src/pages/admin/InstanceManagementPage.tsx`
+ - `frontend/src/pages/instances/InstanceDetailPage.tsx`
+ - `frontend/src/services/instanceService.ts`
+ - `frontend/src/types/instance.ts`
+- Tests/checks run:
+ - `cd frontend; npx eslint src/pages/admin/InstanceManagementPage.tsx src/pages/instances/InstanceDetailPage.tsx src/services/instanceService.ts src/types/instance.ts`
+ - `cd frontend; npm run build`
+- Known gaps:
+ - Full frontend lint still fails on pre-existing unrelated files.
+ - Admin external access status/revoke actions need a dedicated UI/API policy pass.
+- User-visible behavior:
+ - Admin instance page now has Lite/Pro and backend filters plus mode summary.
+ - Instance detail page now lets owners generate public links/API keys and disable external access.
+
+## Phase 8: Verification
+
+- [x] `cd backend && go test ./... -count=1`
+- [x] `cd frontend && npm run lint` attempted; fails on pre-existing unrelated lint debt.
+- [x] `cd frontend && npx eslint `
+- [x] `cd frontend && npm run build`
+- [ ] YAML parse/dry-run for deployment manifests if changed.
+- [ ] Manual smoke:
+ - Create Lite OpenClaw.
+ - Create Pro OpenClaw.
+ - Access both through ClawManager.
+ - Enable external access for both.
+ - Verify AI Gateway attribution for Lite is instance/gateway-specific.
+ - Verify admin can see/manage both modes.
diff --git a/docs/superpowers/plans/2026-06-04-short-sharelink.md b/docs/superpowers/plans/2026-06-04-short-sharelink.md
new file mode 100644
index 0000000..01e4da0
--- /dev/null
+++ b/docs/superpowers/plans/2026-06-04-short-sharelink.md
@@ -0,0 +1,65 @@
+# Short Share Link Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Replace exposed public instance URLs with `/s//` short links and add fixed/custom/permanent expiration choices.
+
+**Architecture:** Public links become capability URLs: the plaintext short code is returned only at creation time and only its SHA-256 hash is stored. The backend resolves `/s/:code/*path` to an enabled external access record, rewrites the request to the internal instance proxy path, and proxies the request. The old `/api/v1/public/instances/:slug/proxy` route is removed.
+
+**Tech Stack:** Go/Gin backend, MySQL migrations, React/Vite frontend, existing Kubernetes image build and rollout flow.
+
+---
+
+### Task 1: Backend Short Link Storage and Service Behavior
+
+**Files:**
+- Modify: `backend/internal/models/instance_external_access.go`
+- Modify: `backend/internal/repository/instance_external_access_repository.go`
+- Modify: `backend/internal/services/instance_external_access_service.go`
+- Modify: `backend/internal/services/instance_external_access_service_test.go`
+- Create: `backend/internal/db/migrations/029_add_short_external_access_links.sql`
+
+- [ ] Write tests that public links return `/s//`, do not return `token=`, and persist only a code hash.
+- [ ] Write tests for preset, custom, and permanent expiration resolution.
+- [ ] Add `short_code_hash` storage and lookup repository methods.
+- [ ] Generate random short codes, hash them, and return only the short URL.
+- [ ] Keep API key mode available, but do not make Public Link depend on a separate query token.
+
+### Task 2: Backend Short Link Routing
+
+**Files:**
+- Modify: `backend/cmd/server/main.go`
+- Modify: `backend/internal/handlers/instance_handler.go`
+- Modify: `backend/internal/handlers/instance_handler_test.go`
+- Modify: `deployments/nginx/nginx.conf`
+
+- [ ] Write handler tests proving `/s//` and `/s//assets/app.js` resolve without query tokens.
+- [ ] Remove old `/api/v1/public/instances/:slug...` public routes.
+- [ ] Add root-level `/s/:code` and `/s/:code/*path` routes.
+- [ ] Rewrite short-link request paths to `/api/v1/instances/:id/proxy/...` before using the existing proxy service.
+- [ ] Add an nginx `/s/` proxy location so the SPA does not consume short links.
+
+### Task 3: Frontend Expiration UI
+
+**Files:**
+- Modify: `frontend/src/types/instance.ts`
+- Modify: `frontend/src/services/instanceService.ts`
+- Modify: `frontend/src/pages/instances/InstanceDetailPage.tsx`
+
+- [ ] Add request types for `preset`, `custom`, and `permanent` expiration modes.
+- [ ] Show fixed choices: 1 hour, 24 hours, 7 days, 30 days, custom, permanent.
+- [ ] Default Public Link creation to 24 hours.
+- [ ] Send expiration selection to `POST /instances/:id/external-access/public-link`.
+- [ ] Display the returned short URL and expiration state.
+
+### Task 4: Verification and Deployment
+
+**Files:**
+- No source-only changes beyond Tasks 1-3.
+
+- [ ] Run targeted backend tests for external access and handler routing.
+- [ ] Run `go test ./... -count=1`.
+- [ ] Run frontend lint/build.
+- [ ] Build and push image to `172.16.1.12:5010`.
+- [ ] Roll out `clawmanager-app`.
+- [ ] Verify on the real cluster that generated links use `/s//`, the long public route is not registered, and `/s//assets/...` no longer returns 401.
diff --git a/docs/superpowers/specs/2026-06-01-clawmanager-v2-runtime-pool-design.md b/docs/superpowers/specs/2026-06-01-clawmanager-v2-runtime-pool-design.md
new file mode 100644
index 0000000..b1c0fb0
--- /dev/null
+++ b/docs/superpowers/specs/2026-06-01-clawmanager-v2-runtime-pool-design.md
@@ -0,0 +1,719 @@
+# ClawManager V2 Runtime Pool Design
+
+## Overview
+
+ClawManager V2 changes the runtime model from one Kubernetes Pod per user instance to shared runtime pools. A user instance becomes metadata plus a workspace directory plus a gateway subprocess. A runtime Pod hosts one `clawmanager-agent` and many gateway subprocesses.
+
+Runtime pools are split by agent type:
+
+- `openclaw-runtime` Deployment, initially 1 replica.
+- `hermes-runtime` Deployment, initially 1 replica.
+- Each runtime Pod hosts up to 100 gateways by default.
+- The 101st instance of the same type causes ClawManager to scale the matching Deployment.
+
+The user-facing proxy path remains stable:
+
+```text
+/api/v1/instances/{id}/proxy/
+```
+
+Backend proxying uses the active binding:
+
+```text
+instance_id -> runtime_pod.pod_ip + gateway_port
+```
+
+Workspaces use a shared root:
+
+```text
+/workspaces/{runtime}/user-{user_id}/instance-{instance_id}
+```
+
+ClawManager owns scheduling, persistence, proxying, permissions, file APIs, UI, rollout control, and deployment manifests. `clawmanager-agent` owns gateway subprocess lifecycle, port allocation, cgroups, Linux user isolation, health checks, metrics, and gateway logs.
+
+## Deployment Model
+
+V2 keeps the one-command install goal:
+
+```bash
+kubectl apply -f clawmanager-v2.yaml
+```
+
+The YAML includes:
+
+- Namespace, RBAC, ConfigMaps, Secrets, Services, PVCs.
+- MySQL.
+- Redis.
+- ClawManager backend, default 3 replicas.
+- ClawManager frontend/nginx.
+- AI Gateway / control plane.
+- `workspace-store`.
+- `openclaw-runtime` Deployment.
+- `hermes-runtime` Deployment.
+
+The default storage mode is an embedded lightweight shared store, implemented as an NFS/Ganesha-style service:
+
+- `workspace-store` persists data on its own RWO PVC.
+- Backend and runtime Pods mount `/workspaces`.
+- This satisfies one-YAML deployment and runtime Pod failover.
+- `workspace-store` is a lightweight single point. Storage-layer HA is outside the default one-YAML mode and belongs to a future external RWX mode.
+
+Default runtime parameters:
+
+```text
+backend replicas: 3
+openclaw runtime replicas: 1
+hermes runtime replicas: 1
+gateways per runtime pod: 100
+gateway port range per pod: 20000-20099
+scheduler lease renew: 2s
+scheduler failover: 6-8s
+agent heartbeat interval: 2s
+agent unhealthy threshold: 8s
+gateway health interval: 3s
+migration batch size: 10
+redis lock ttl: 30s
+```
+
+All defaults should be configurable through environment variables or ConfigMap entries.
+
+## Persistence And Hot State
+
+MySQL is the only source of truth. Redis is a hot-path accelerator.
+
+MySQL stores:
+
+- Instances.
+- Runtime Pods.
+- Runtime bindings.
+- Runtime rollouts.
+- Workspace audit records.
+- Users, quotas, and permissions.
+
+Redis stores:
+
+- Runtime Pod heartbeat TTLs.
+- Runtime capacity hot cache.
+- Gateway status snapshots.
+- Scheduler event streams.
+- WebSocket fanout.
+- Short locks.
+
+Redis may be lost and rebuilt. On ClawManager restart, the scheduler reconciles MySQL records with agent-reported `/gateways` state.
+
+## Data Model
+
+The existing `instances` table remains the user instance fact table. V2 adds fields or equivalent migration support for:
+
+```text
+workspace_path
+last_error
+```
+
+In V2, resource fields change meaning:
+
+- `cpu_cores`: cgroup CPU quota for one gateway.
+- `memory_gb`: cgroup memory limit for one gateway.
+- `disk_gb`: logical workspace quota.
+
+New `runtime_pods` table:
+
+```text
+id
+runtime_type: openclaw | hermes
+pod_name
+pod_namespace
+pod_ip
+agent_id
+status: ready | draining | unhealthy | offline | retired
+version
+capacity
+used
+port_start
+port_end
+management_url
+management_token_encrypted
+last_heartbeat_at
+created_at
+updated_at
+```
+
+New `instance_runtime_bindings` table:
+
+```text
+id
+instance_id
+runtime_pod_id
+runtime_type
+gateway_port
+workspace_path
+gateway_status: starting | running | stopped | error
+status: active | released | migrating | error
+reason: create | start | stop | failover | upgrade | delete | recovery
+bound_at
+released_at
+last_health_check_at
+last_error
+```
+
+New `runtime_rollouts` table:
+
+```text
+id
+runtime_type
+from_image
+to_image
+status: idle | creating_new_pods | draining_old_pods | migrating_gateways | paused | failed | completed | rolling_back
+strategy
+batch_size
+started_by
+started_at
+completed_at
+paused_at
+error_message
+```
+
+## State Machines
+
+Instance statuses:
+
+```text
+creating -> running gateway scheduled and healthy
+creating -> error workspace, scheduler, or agent failure
+running -> stopped user stops instance; slot and port are released
+running -> running failover or upgrade migration succeeds
+running -> error repeated recovery failure
+running -> deleting
+stopped -> creating user starts instance; scheduler chooses a new Pod
+stopped -> deleting
+error -> creating user retries start
+error -> deleting
+```
+
+Runtime Pod statuses:
+
+```text
+ready accepts new gateways
+draining accepts no new gateways; existing gateways are migrated away
+unhealthy agent or Pod health failure; active bindings migrate away
+offline Pod is unreachable
+retired Pod has no active gateways and can be removed or ignored
+```
+
+## Scheduler
+
+The backend runs as multiple replicas. All replicas serve API traffic, but only the backend holding a Kubernetes `Lease` runs `RuntimeScheduler`.
+
+The scheduler handles:
+
+- Newly created instances in `creating`.
+- Start requests.
+- Stop/Delete cleanup.
+- Runtime Pod capacity changes.
+- Runtime Pod heartbeat timeout.
+- Gateway health failure.
+- Draining Pod migration.
+- Rollout batches.
+- ClawManager restart recovery.
+- Orphan gateway cleanup.
+
+Concurrency protection uses:
+
+- Kubernetes Lease for scheduler leadership.
+- Redis short locks:
+ - `lock:instance:{id}`
+ - `lock:runtime:scale:{type}`
+ - `lock:runtime:pod:{pod_id}`
+ - `lock:workspace:{instance_id}:{path_hash}`
+- Conditional DB updates as the final consistency guard.
+
+## Instance Lifecycle
+
+Create is asynchronous:
+
+1. API creates the instance record with `creating`.
+2. API creates the workspace directory.
+3. API writes a scheduler event to Redis.
+4. Scheduler chooses a `ready` runtime Pod with capacity.
+5. If no Pod has capacity, scheduler scales the matching runtime Deployment.
+6. Scheduler waits for the new agent to register.
+7. Scheduler calls agent `POST /gateways`.
+8. Scheduler writes an active binding.
+9. Scheduler updates the instance to `running`.
+
+Stop:
+
+- Calls agent to delete the gateway.
+- Releases slot and port.
+- Marks active binding `released`.
+- Updates the instance to `stopped`.
+- Leaves workspace data intact.
+
+Start:
+
+- Moves instance to `creating`.
+- Schedules a fresh gateway.
+- May use a different Pod and port.
+
+Restart:
+
+- Prefer restart on the current healthy Pod.
+- If the current Pod is unhealthy, reschedule to a healthy Pod.
+
+Delete:
+
+- Stops the gateway if active.
+- Deletes workspace contents.
+- Deletes or marks instance according to existing repository behavior.
+- Writes audit events.
+
+## Failover
+
+Failover triggers:
+
+- Agent heartbeat exceeds 8 seconds.
+- Pod is NotReady, deleted, or restarted.
+- Gateway health fails repeatedly.
+- Proxy detects active binding is unreachable.
+
+Failover flow:
+
+1. Mark runtime Pod `unhealthy` or `offline`.
+2. Select a healthy same-runtime Pod.
+3. If capacity exists, call target agent `recover`.
+4. If capacity is full, scale the Deployment first.
+5. Write a new active binding.
+6. Release or retire the old binding.
+7. Clean orphan gateway if the old Pod later returns.
+
+Target interruption is 5-10 seconds.
+
+## Gray Upgrade / Rollout
+
+OpenClaw and Hermes upgrades use the same drain-and-migrate machinery as failover.
+
+Admin starts rollout from System Settings / System Images:
+
+1. Create a `runtime_rollouts` row.
+2. Update the runtime Deployment image/template.
+3. New version Pods register as ready.
+4. Old version Pods are marked `draining`.
+5. Draining Pods receive no new gateways.
+6. Scheduler migrates gateways in batches of 10.
+7. Each batch is health-checked.
+8. Failure pauses rollout.
+9. Admin can continue, pause, or roll back.
+10. Empty old Pods become `retired`.
+
+Deployment strategy:
+
+```text
+maxUnavailable: 0
+maxSurge: 1
+```
+
+ClawManager controls gateway migration instead of allowing Kubernetes rolling update to kill Pods that still host gateways.
+
+## Agent Contract
+
+Each runtime Pod runs one `clawmanager-agent`.
+
+Agent keeps existing active reporting abilities:
+
+```http
+POST /api/v1/agent/register
+POST /api/v1/agent/heartbeat
+POST /api/v1/agent/state/report
+POST /api/v1/agent/skills/inventory
+```
+
+Reporting expands from one instance to Pod plus many gateways:
+
+```json
+{
+ "agent_id": "openclaw-runtime-abc",
+ "runtime_type": "openclaw",
+ "pod_name": "openclaw-runtime-abc",
+ "pod_ip": "10.42.1.83",
+ "capacity": 100,
+ "used": 37,
+ "port_range": { "start": 20000, "end": 20099 },
+ "version": "openclaw:2.1.0",
+ "gateways": [
+ {
+ "instance_id": 42,
+ "port": 20017,
+ "pid": 812,
+ "status": "running",
+ "workspace_path": "/workspaces/openclaw/user-7/instance-42"
+ }
+ ]
+}
+```
+
+ClawManager actively calls agent on:
+
+```text
+http://:19090
+```
+
+All management calls use:
+
+```http
+Authorization: Bearer
+```
+
+Required agent management APIs:
+
+```http
+GET /healthz
+GET /capacity
+GET /gateways
+GET /gateways/{instance_id}
+POST /gateways
+POST /gateways/{instance_id}/stop
+POST /gateways/{instance_id}/restart
+POST /gateways/{instance_id}/recover
+DELETE /gateways/{instance_id}
+GET /gateways/{instance_id}/logs?tail=200
+```
+
+Create gateway request:
+
+```json
+{
+ "instance_id": 42,
+ "user_id": 7,
+ "agent_type": "openclaw",
+ "workspace_path": "/workspaces/openclaw/user-7/instance-42",
+ "port_range": { "start": 20000, "end": 20099 },
+ "env": {
+ "CLAWMANAGER_INSTANCE_ID": "42",
+ "CLAWMANAGER_LLM_BASE_URL": "http://clawmanager-gateway.clawmanager-system.svc.cluster.local:9001/api/v1/gateway/llm"
+ },
+ "resources": {
+ "cpu_cores": 0.5,
+ "memory_mb": 512,
+ "pids_limit": 256
+ }
+}
+```
+
+Create gateway response:
+
+```json
+{
+ "instance_id": 42,
+ "port": 20017,
+ "status": "running",
+ "pid": 812,
+ "url": "http://127.0.0.1:20017"
+}
+```
+
+Agent port requirements:
+
+- Pod-local gateway range is `20000-20099`.
+- Agent assigns the actual port.
+- Agent skips conflicts or returns structured errors.
+- Agent persists enough state to recover the port occupation map after restart.
+- Repeated `create` for the same instance is idempotent.
+
+Agent security requirements:
+
+- Each gateway runs as its own Linux user.
+- Deterministic identity:
+
+```text
+uid = 200000 + instance_id
+gid = 200000 + instance_id
+```
+
+- Agent creates uid/gid as needed.
+- Agent `chown`s workspace to that uid/gid.
+- Workspace permission should be `0700` or `0750`.
+- Gateway subprocess must not run as root.
+- Agent may run as root to manage users, ownership, and cgroups.
+- Agent must defend against `../`, absolute paths, and symlink escape.
+
+Agent resource requirements:
+
+- Each gateway uses a dedicated cgroup or systemd scope.
+- Enforce CPU quota, memory limit, and pids limit.
+- IO weight is outside the first release and can be added in a separate resource-control enhancement.
+- Agent reports cgroup OOM/limit failures.
+- Agent may restart failed gateways, then mark error after retry exhaustion.
+
+Agent health and recovery:
+
+- Check gateway health every 3 seconds.
+- Restart locally before reporting hard failure.
+- Support `recover` on a different Pod using the same workspace path.
+
+Agent metrics:
+
+- Pod-level CPU, memory, disk I/O, network in/out, used/capacity, workspace total.
+- Gateway-level instance_id, port, pid, status, CPU, memory, pids, network in/out, workspace_used_bytes, restart_count, last_error.
+
+## Proxying
+
+Frontend and iframe use:
+
+```text
+/api/v1/instances/{id}/proxy/
+```
+
+Backend checks user permission and active binding, then proxies to:
+
+```text
+http://{pod_ip}:{gateway_port}
+```
+
+HTTP and WebSocket traffic both use this path. The frontend path does not change after migration; only the binding changes.
+
+V2 does not create per-instance Kubernetes Services. It uses direct Pod IP plus gateway port.
+
+## Workspace File Management
+
+ClawManager backend directly reads and writes the shared workspace. File management is independent of the OpenClaw/Hermes iframe and works even when the instance is stopped.
+
+Supported operations:
+
+- List files and directories.
+- Text preview.
+- Image preview.
+- PDF preview.
+- Download.
+- Upload.
+- Create folder.
+- Rename file or directory.
+- Delete file or directory.
+
+API shape:
+
+```http
+GET /api/v1/instances/{id}/workspace/files?path=/
+GET /api/v1/instances/{id}/workspace/preview?path=/README.md
+GET /api/v1/instances/{id}/workspace/download?path=/README.md
+POST /api/v1/instances/{id}/workspace/upload
+POST /api/v1/instances/{id}/workspace/folders
+PATCH /api/v1/instances/{id}/workspace/rename
+DELETE /api/v1/instances/{id}/workspace/files
+```
+
+Preview support:
+
+- Text, maximum 1 MiB:
+ - `.txt`, `.md`, `.json`, `.yaml`, `.yml`, `.log`, `.py`, `.js`, `.ts`, `.go`, `.sh`
+- Images, maximum 10 MiB:
+ - `.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, `.svg`
+- PDF through browser iframe.
+- Other binary files are download-only.
+
+Upload defaults:
+
+- Maximum upload size: 500 MiB.
+- Configurable through environment variables.
+
+Security rules:
+
+1. API accepts relative paths only.
+2. Clean and join the user path with the instance workspace root.
+3. Resolve realpath.
+4. Final real path must remain inside the instance workspace root.
+5. Reject `../`.
+6. Reject absolute paths.
+7. Reject symlink escape.
+8. Upload filenames cannot contain path separators.
+9. Preview, download, delete, rename, and upload all use the same path guard.
+
+Workspace write operations use Redis path locks:
+
+```text
+lock:workspace:{instance_id}:{path_hash}
+```
+
+Logical disk quota:
+
+- `disk_gb` is the workspace quota.
+- Upload checks current size plus upload size before writing.
+- Agent periodically reports workspace usage.
+- If gateway writes beyond quota, agent reports `quota_exceeded`.
+- No filesystem project quota in the first version.
+
+Deletion:
+
+- No trash/recycle bin in V2 first version.
+- Deletes are direct and require confirmation.
+- Directory deletion displays item count before confirmation.
+
+Audit:
+
+- Audit all write operations: upload, delete, rename, create folder.
+- Audit download.
+- Do not audit preview.
+
+## Frontend Information Architecture
+
+Ordinary users only see usable product state.
+
+V2 also includes a UI simplification pass. The target style is quiet, direct, and work-focused:
+
+- Use a simple visual system with neutral backgrounds, restrained borders, clear spacing, and minimal decorative effects.
+- Avoid marketing-style hero areas, oversized titles, nested cards, ornamental gradients, and visual noise.
+- Prefer full-width sections and compact panels over many floating cards.
+- Use small, stable controls with clear labels and icons for common actions.
+- Keep ordinary user pages focused on the next useful action, not operational detail.
+- Keep admin pages denser and more table-oriented, with metrics designed for scanning and comparison.
+- Use consistent status language across pages: `Available`, `Starting`, `Unavailable` for users; `Ready`, `Draining`, `Unhealthy`, `Offline`, `Retired` for administrators.
+- Make responsive layouts predictable: iframe, file manager, tables, and toolbars must not overflow or overlap on mobile or desktop.
+
+User instance detail page shows:
+
+- Instance name.
+- Runtime type: OpenClaw or Hermes.
+- Availability: `Available`, `Starting`, `Unavailable`.
+- Start, Stop, Restart, Delete, Refresh, Fullscreen.
+- OpenClaw/Hermes iframe.
+- Workspace file manager.
+- Workspace usage.
+
+User page layout:
+
+```text
+[Compact title row: instance name, availability, primary actions]
+[OpenClaw/Hermes iframe with Refresh and Fullscreen]
+[Workspace file manager]
+```
+
+No separate service/resource/recent-status blocks appear on the ordinary user page. If an instance is unavailable, the page shows only actionable messages such as starting progress, retry start, or contact administrator.
+
+User instance detail page does not show:
+
+- Pod name or IP.
+- Gateway port.
+- Runtime capacity.
+- Runtime version.
+- Agent heartbeat.
+- Scheduler or migration details.
+- CPU, memory, network, disk runtime internals.
+- Service/resource/recent status operational cards.
+
+Admin adds a Runtime Pods page:
+
+- Runtime selector: OpenClaw / Hermes.
+- Pool summary: Pod count, gateway usage, CPU, memory, disk I/O, network.
+- Pod table: status, used/capacity, CPU, memory, disk I/O, network, version.
+- Selected Pod detail: Pod IP, agent heartbeat, port range, workspace mount, metrics chart.
+- Gateway distribution: instance, user, port, status.
+- Operations: Drain Pod, Resume Pod, Migrate Gateways, View Agent Logs, Scale Runtime Pool.
+
+Admin page style:
+
+- Use table-first layouts for Pods and gateways.
+- Use compact metric cells and sparklines rather than large decorative charts.
+- Keep dangerous operations grouped and confirm destructive or disruptive actions.
+- Do not hide operational detail from administrators, but keep it visually organized by pool, Pod, and gateway.
+
+Admin runtime data is pushed over WebSocket. Initial page load uses HTTP snapshot; updates use WebSocket from Redis-backed backend state. Agent reports metrics every 2 seconds.
+
+System Settings / System Images adds rollout controls:
+
+- OpenClaw runtime image.
+- Hermes runtime image.
+- Current version.
+- Target version.
+- Rollout status.
+- Old and new Pod counts.
+- Migrated and failed gateway counts.
+- Publish, pause, continue, rollback.
+
+## Legacy Compatibility
+
+V2 allows new instance creation only for:
+
+- `openclaw`
+- `hermes`
+
+Legacy types are not offered on the create page:
+
+- `ubuntu`
+- `debian`
+- `centos`
+- `custom`
+- `webtop`
+
+Existing legacy instances remain visible and are labeled `Legacy`. Their old detail and runtime behavior should remain available where practical. No automatic migration is performed. Admins can delete legacy instances. A legacy migration tool is out of scope for this V2 runtime-pool design.
+
+## Acceptance Criteria
+
+Instance creation:
+
+- Creating OpenClaw/Hermes does not create per-instance Pod/PVC/Service.
+- API returns `creating`.
+- Scheduler creates a gateway through agent.
+- Instance becomes `running`.
+- The 101st same-runtime instance scales the matching runtime Deployment.
+
+Port handling:
+
+- Gateway ports do not conflict within one Pod.
+- Agent restart restores port occupation state.
+- Binding records the actual assigned port.
+
+Access:
+
+- `/api/v1/instances/{id}/proxy/` opens OpenClaw/Hermes.
+- Frontend path remains unchanged after migration.
+- Ordinary users cannot see Pod/IP/port details.
+
+UI style:
+
+- Ordinary user pages use the simplified V2 style: compact title row, iframe, workspace file manager, and no runtime operational cards.
+- Admin pages use a simple operations-console style: dense tables, compact metrics, clear grouping, and no decorative dashboard chrome.
+- Text, buttons, toolbars, iframe, file manager, and runtime tables remain readable without overlap on mobile and desktop.
+- Status language is consistent with the V2 design vocabulary for user and admin contexts.
+
+Failover:
+
+- Runtime Pod deletion or restart migrates running instances within 5-10 seconds.
+- Workspace data remains intact.
+- Old orphan gateways are cleaned when old Pods return.
+
+Gray upgrade:
+
+- Publishing a new image starts new runtime Pods.
+- Old Pods enter draining.
+- Gateways migrate in batches.
+- Rollout can pause on failure.
+- Admin can continue or roll back.
+
+Workspace files:
+
+- Users can list, preview, download, upload, create folder, rename, and delete within their workspace.
+- Stopped instances still allow file operations.
+- Path escape and symlink escape are rejected.
+- Deletes require confirmation and do not use trash.
+- Write operations and downloads are audited.
+
+Admin runtime page:
+
+- Shows each runtime Pod status, gateway usage, CPU, memory, disk, and network.
+- Shows gateway distribution per Pod.
+- Supports drain, resume, migrate, log view, and scale operations.
+- Uses WebSocket for real-time updates.
+
+Deployment:
+
+- One YAML installs all required components.
+- Backend defaults to 3 replicas.
+- OpenClaw and Hermes runtime pools default to 1 replica each.
+- Redis, MySQL, and workspace-store are included and usable.
+
+## Open Risks
+
+- Default workspace-store is intentionally lightweight and not storage-layer HA.
+- Agent changes are on the critical path and must follow this contract.
+- Pod-level multi-tenancy requires careful Linux user, cgroup, and path isolation.
+- Rollout migration must be idempotent and pause safely on partial failures.
+- Metrics volume must be controlled through Redis aggregation and WebSocket subscriptions.
diff --git a/docs/superpowers/specs/2026-06-03-clawmanager-v2-e2e-design.md b/docs/superpowers/specs/2026-06-03-clawmanager-v2-e2e-design.md
new file mode 100644
index 0000000..18aa49f
--- /dev/null
+++ b/docs/superpowers/specs/2026-06-03-clawmanager-v2-e2e-design.md
@@ -0,0 +1,472 @@
+# ClawManager V2 E2E Test Design
+
+## Overview
+
+ClawManager V2 needs an end-to-end test mechanism that covers user-facing flows across the React frontend, Go backend, MySQL migrations, and selected V2 runtime-pool APIs. The test suite should live in a root-level `e2e/` directory because it validates cross-service behavior rather than frontend-only behavior.
+
+The initial implementation should use Playwright. It fits the current Vite frontend plus Go backend setup, supports multiple local web servers, can persist authenticated browser state, and produces CI-friendly traces, screenshots, videos, and HTML reports.
+
+## Language And Tooling
+
+E2E test code should be written in TypeScript.
+
+- Test specs use Playwright Test with `.spec.ts` files.
+- Fixtures, page objects, setup files, API helpers, and environment helpers use `.ts`.
+- Optional command wrappers can use PowerShell (`.ps1`) and shell (`.sh`) so Windows and CI/Linux users have clear entry points.
+- JavaScript should not be used for hand-written e2e source files unless a tool generates it.
+
+TypeScript is the preferred language because the frontend already uses TypeScript, Playwright has strong TypeScript support, and typed fixtures/page objects reduce test maintenance risk as V2 flows grow.
+
+## Goals
+
+- Add a dedicated root-level `e2e/` test directory.
+- Support local developer execution with an isolated database.
+- Support CI execution with stable smoke tests.
+- Cover login, route protection, core user navigation, admin access control, workspace file flows, and V2 runtime-pool API surfaces.
+- Provide readable run logs, clear progress output, structured test results, and a final human-readable test report.
+- Integrate the P0 e2e suite into GitHub pull request checks.
+- Support manual local execution for P0, P1, P2, and the full suite.
+- Keep the first version independent of a real Kubernetes cluster or real runtime agent.
+- Leave room for a later full-stack k8s e2e mode that targets the existing kind smoke deployment.
+
+## Non-Goals
+
+- Do not replace backend Go unit or integration tests.
+- Do not move frontend tests into `e2e/`.
+- Do not require a real OpenClaw or Hermes runtime Pod for the initial suite.
+- Do not make heavy k8s browser tests block every pull request in the first iteration.
+
+## Recommended Directory Structure
+
+```text
+e2e/
+ package.json
+ package-lock.json
+ playwright.config.ts
+ README.md
+ .gitignore
+
+ docker/
+ docker-compose.e2e.yml
+
+ scripts/
+ reset-db.ts
+ wait-for.ts
+ run-with-log.ts
+ e2e-up.ps1
+ e2e-down.ps1
+ e2e-up.sh
+ e2e-down.sh
+
+ setup/
+ auth.setup.ts
+ db.setup.ts
+
+ fixtures/
+ test.ts
+ apiClient.ts
+ users.ts
+ env.ts
+
+ pages/
+ LoginPage.ts
+ DashboardPage.ts
+ InstanceListPage.ts
+ InstanceDetailPage.ts
+ AdminRuntimePodsPage.ts
+
+ tests/
+ smoke/
+ login.spec.ts
+ navigation.spec.ts
+ user/
+ dashboard.spec.ts
+ instances.spec.ts
+ workspace-files.spec.ts
+ admin/
+ runtime-pods.spec.ts
+ access-control.spec.ts
+ api/
+ auth.spec.ts
+ runtime-agent.spec.ts
+
+ reports/
+ .gitkeep
+```
+
+## Responsibilities
+
+`playwright.config.ts` defines browser projects, base URLs, reporters, traces, retries, timeouts, and the local web server orchestration.
+
+`docker/docker-compose.e2e.yml` starts only test dependencies that should be isolated from a developer machine, beginning with MySQL on a non-default host port such as `13307`.
+
+`scripts/` contains explicit helper commands for resetting the e2e database, waiting for ports or health endpoints, and starting or stopping local dependencies on Windows and Unix-like systems.
+
+`scripts/run-with-log.ts` wraps Playwright execution when needed so console output is also written to `e2e/reports/run.log`.
+
+`setup/` contains Playwright project dependencies. `db.setup.ts` prepares deterministic data. `auth.setup.ts` logs in as seeded users and writes browser storage state files for later specs.
+
+`fixtures/` contains typed shared utilities: environment parsing, API request helpers, seeded user constants, and a custom Playwright `test` export.
+
+`pages/` contains page objects for stable selectors and common user actions. Page objects should stay small and focused on navigation and interactions, not assertions-heavy business logic.
+
+`tests/` groups specs by product surface rather than by technical layer. Smoke tests are intentionally tiny and run first in CI.
+
+`reports/` stores generated run logs and machine-readable report files. Generated report artifacts are ignored by git except for a `.gitkeep` tracked marker.
+
+## Reuse And Common Module Guidelines
+
+E2E implementation must prefer shared modules over repeated setup code. Repeated steps are expected across login, role switching, API calls, database setup, runtime pod seeding, navigation, and reporting, so the first implementation batch must establish reusable foundations before adding many specs.
+
+Required shared modules:
+
+- `fixtures/test.ts` exports the shared Playwright `test` and `expect`, wires common fixtures, and centralizes base URL, storage state, and artifact behavior.
+- `fixtures/env.ts` owns environment parsing for frontend URL, backend URL, database connection, runtime agent token, workspace root, and CI flags.
+- `fixtures/apiClient.ts` owns typed request helpers for auth, runtime-agent, admin runtime Pods, and authenticated API calls.
+- `fixtures/users.ts` owns seeded user constants such as `admin/admin123` and any generated non-admin test user data.
+- `setup/auth.setup.ts` performs reusable login setup and writes storage state files instead of logging in manually in every UI spec.
+- `setup/db.setup.ts` prepares deterministic database and workspace state for specs that need seeded V2 runtime data.
+- `pages/*.ts` page objects encapsulate repeated UI actions and selectors. Tests should describe user intent; page objects should hold selector details.
+- `scripts/*.ts` own lifecycle concerns such as database reset, server readiness, process orchestration, logging, and report paths.
+
+Reuse rules:
+
+- Do not duplicate login steps inside every spec when a storage state fixture can be used.
+- Do not hard-code backend or frontend URLs inside specs; read them from `fixtures/env.ts`.
+- Do not repeat raw runtime-agent request construction in multiple tests; use `apiClient.ts`.
+- Do not repeat selectors across specs; move repeated selectors and actions into page objects.
+- Do not add test-only helpers to production frontend or backend code for e2e convenience.
+- Keep page objects small: one page object should cover one page or stable product surface.
+- Keep shared helpers behavior-focused. Avoid large catch-all utility files.
+- When a repeated sequence appears in two specs, extract it before adding a third copy.
+
+## Runtime Model
+
+The initial local flow should start:
+
+1. MySQL through `e2e/docker/docker-compose.e2e.yml`.
+2. Go backend from `backend/cmd/server`, with environment overrides:
+ - `SERVER_ADDRESS=:9001`
+ - `DB_HOST=127.0.0.1`
+ - `DB_PORT=13307`
+ - `DB_USER=root`
+ - `DB_PASSWORD=123456`
+ - `DB_NAME=clawmanager_e2e`
+ - `JWT_SECRET=clawmanager-e2e-secret`
+ - `RUNTIME_SCHEDULER_ENABLED=false`
+ - `RUNTIME_WORKSPACE_ROOT=/.cache/e2e/workspaces`
+ - `OBJECT_STORAGE_LOCAL_FALLBACK=/.cache/e2e/object-storage`
+ - `SKILL_SCANNER_ENABLED=false`
+3. Vite frontend from `frontend`, using the existing `9002` dev port and `/api` proxy to backend `9001`.
+4. Playwright tests against `http://127.0.0.1:9002`.
+
+The backend already applies embedded migrations during startup, so e2e does not need a separate migration runner. Database reset can drop and recreate `clawmanager_e2e` before backend startup.
+
+## Run Logging, Progress, And Reports
+
+E2E execution must make progress and results easy to inspect during and after a run.
+
+Console progress:
+
+- Use Playwright's list-style console reporter for local runs so each test is visible as it starts and finishes.
+- Test titles should include the priority tag, such as `@p0`, `@p1`, or `@p2`.
+- Long tests should use `test.step(...)` for meaningful milestones such as login, seed data creation, navigation, API call, and assertion phases.
+- Setup scripts should print clear lifecycle messages for database reset, backend startup, frontend startup, auth setup, and report location.
+
+Run log:
+
+- Every standard script should write a timestamped log to `e2e/reports/run.log`.
+- CI should also retain raw job logs, but `run.log` is the repository-level e2e artifact.
+- Logs must not print access tokens, refresh tokens, passwords, or full authorization headers.
+- On failure, logs should include the failing test name, retry count, browser project, and the Playwright artifact path.
+
+Report artifacts:
+
+- Human-readable HTML report: `e2e/playwright-report/`.
+- Raw per-test artifacts: `e2e/test-results/`.
+- Machine-readable JSON report: `e2e/reports/results.json`.
+- CI-friendly JUnit report: `e2e/reports/junit.xml`.
+- Text run log: `e2e/reports/run.log`.
+
+Recommended package scripts:
+
+```json
+{
+ "scripts": {
+ "test:p0": "playwright test --grep @p0",
+ "test:p1": "playwright test --grep @p1",
+ "test:p2": "playwright test --grep @p2",
+ "test:all": "playwright test",
+ "ci:p0": "playwright test --grep @p0 --workers=1",
+ "test:report": "playwright show-report playwright-report"
+ }
+}
+```
+
+`playwright.config.ts` should configure at least these reporters:
+
+- `list` for clear live progress.
+- `html` for the final local/CI report.
+- `json` written to `e2e/reports/results.json`.
+- `junit` written to `e2e/reports/junit.xml`.
+
+CI must upload `e2e/playwright-report/`, `e2e/test-results/`, and `e2e/reports/` whenever e2e runs fail. Local users can open the final report with `npm run test:report`.
+
+## Test Data Strategy
+
+Use seeded migration data where it is stable. The default admin account is:
+
+```text
+username: admin
+password: admin123
+```
+
+Additional test data should be created through APIs when possible. For flows that require a specific V2 state not naturally reachable without k8s, `db.setup.ts` may insert deterministic rows directly after migrations have run. Direct database setup is acceptable for initial e2e because the target behavior is browser and API behavior, not migration correctness.
+
+Each spec should use unique names with a run prefix when it creates mutable data. Cleanup should happen through API calls where practical, with database reset reserved for suite setup.
+
+## Test Case Inventory And Priority
+
+The suite should be planned from a full test inventory, then implemented in priority batches. Each batch must be developed, run, fixed, and reviewed before starting the next batch. P0 is the first implementation batch and acts as the required core gate.
+
+### P0: Core Gate
+
+P0 proves the product can boot, authenticate, protect routes, and expose the most critical V2 surfaces without requiring a real k8s cluster or runtime agent.
+
+- `smoke/login.spec.ts`
+ - Login page renders.
+ - Admin can log in with `admin/admin123`.
+ - Login failure shows a user-visible error and stays on `/login`.
+- `smoke/navigation.spec.ts`
+ - Unauthenticated users are redirected to `/login`.
+ - Authenticated admin reaches `/dashboard`.
+ - Main authenticated navigation can reach instances and settings/admin entry points that are visible for the logged-in role.
+- `admin/access-control.spec.ts`
+ - Non-admin users cannot open admin routes.
+ - Admin users can open the admin runtime Pods route.
+- `api/auth.spec.ts`
+ - `/api/v1/auth/login` returns access and refresh tokens.
+ - `/api/v1/auth/me` returns the current user when called with a valid token.
+ - `/api/v1/auth/me` rejects unauthenticated requests.
+- `api/runtime-agent.spec.ts`
+ - Runtime-agent register persists an `openclaw` runtime pod.
+ - Runtime-agent heartbeat updates pod health state.
+ - Invalid runtime-agent payloads are rejected with a non-2xx response.
+- `admin/runtime-pods.spec.ts`
+ - Admin runtime Pods page renders an empty state.
+ - Admin runtime Pods page renders a seeded pod row with status and capacity fields.
+
+P0 completion criteria:
+
+- P0 passes locally on Chromium.
+- P0 passes in CI with one worker.
+- Console progress, `e2e/reports/run.log`, HTML report, JSON report, JUnit report, traces, and screenshots are produced as configured.
+- Playwright trace, report, and log artifacts are uploaded on CI failure.
+- No P1 or P2 tests are required for the first e2e gate.
+
+### P1: V2 User Workflows
+
+P1 covers the main user paths most likely to catch frontend/backend integration regressions after P0 is stable.
+
+- `user/dashboard.spec.ts`
+ - Dashboard loads quota and instance summary.
+ - Dashboard handles zero instances.
+ - Dashboard links to create instance and instance list.
+- `user/instances.spec.ts`
+ - Instance list handles empty state.
+ - Instance list renders seeded V2 `openclaw` and `hermes` instances.
+ - Instance detail shows the simplified V2 layout for a seeded V2 instance.
+ - Instance detail shows actionable unavailable state for a stopped or creating instance.
+- `user/workspace-files.spec.ts`
+ - Workspace file manager lists the seeded workspace root.
+ - User can create a folder.
+ - User can upload a small text file.
+ - User can preview the uploaded file.
+ - User can download the uploaded file.
+ - User can rename a file or folder.
+ - User can delete a file or folder.
+ - Path traversal attempts are rejected through the UI/API path.
+- `api/runtime-agent.spec.ts`
+ - Runtime-agent metrics report updates pod CPU, memory, disk, and network fields.
+ - Runtime-agent gateways report updates gateway rows for admin runtime view.
+
+P1 completion criteria:
+
+- P0 and P1 pass locally.
+- P0 remains the required PR gate.
+- P1 produces the same log and report artifacts as P0.
+- P1 can run in CI as required or manual depending on runtime and database timing stability.
+
+### P2: Expanded Operations And Full-Stack Confidence
+
+P2 validates broader operational flows and should be introduced only after P0 and P1 are reliable.
+
+- `admin/runtime-pods.spec.ts`
+ - Admin runtime Pods page renders seeded gateway rows.
+ - Runtime pod drain action calls the expected API and reflects the response.
+ - Runtime rollout action can be submitted with a test image and shows rollout feedback.
+- `user/instances.spec.ts`
+ - Create instance form validates required fields.
+ - Create instance submits a minimal V2 instance request.
+ - Start, stop, restart, and sync actions show expected UI transitions for seeded or mocked backend state.
+- `admin/access-control.spec.ts`
+ - User-scoped instance list excludes another user's seeded instances.
+ - Admin instance list includes cross-user seeded instances.
+- `k8s/smoke.spec.ts`
+ - Browser can load the app through `https://127.0.0.1:8443` after kind deployment.
+ - Admin can log in through the deployed nginx endpoint.
+ - Health page and frontend route fallback work through the deployed endpoint.
+
+P2 completion criteria:
+
+- P0 and P1 remain stable.
+- P2 can run manually or on a schedule.
+- P2 produces the same log and report artifacts as P0.
+- k8s browser tests are not required for every pull request until deployment timing and runtime image availability are proven stable.
+
+## Local Manual Runs
+
+Developers must be able to run e2e locally without GitHub.
+
+First-time setup:
+
+```bash
+cd e2e
+npm ci
+npx playwright install chromium
+```
+
+Standard local commands:
+
+```bash
+cd e2e
+npm run test:p0
+npm run test:p1
+npm run test:p2
+npm run test:all
+npm run test:report
+```
+
+Windows helpers should provide equivalent entry points:
+
+```powershell
+.\scripts\e2e-up.ps1
+npm run test:p0
+npm run test:report
+.\scripts\e2e-down.ps1
+```
+
+Unix-like helpers should provide equivalent entry points:
+
+```bash
+./scripts/e2e-up.sh
+npm run test:p0
+npm run test:report
+./scripts/e2e-down.sh
+```
+
+Local runs should produce the same log and report outputs as CI. The local default is P0. P1, P2, and `test:all` are explicit commands so longer tests are intentional.
+
+## GitHub Actions Integration
+
+Add an e2e PR check to GitHub Actions. The recommended implementation is to add an `e2e-p0` job to the existing `.github/workflows/build.yml` workflow so it appears alongside the existing frontend, backend, Docker, and k8s smoke checks on pull requests.
+
+The PR check requirements are:
+
+- Trigger on `pull_request`.
+- Also run on pushes to the same protected branch patterns as the existing build workflow.
+- Depend on successful `frontend` and `backend` jobs.
+- Run only the P0 suite initially.
+- Use Chromium with one worker for stability.
+- Upload logs and reports on failure.
+- Fail the PR check when P0 fails.
+
+The first GitHub PR command should be:
+
+Recommended CI commands:
+
+```bash
+cd e2e
+npm ci
+npx playwright install --with-deps chromium
+npm run ci:p0
+```
+
+P1 can be added to CI after it is stable locally. P2 should begin as manual or scheduled CI. The e2e job should not run the full k8s browser suite initially. A later workflow can reuse the existing kind deployment smoke path and then run a small Playwright suite against `https://127.0.0.1:8443`.
+
+GitHub manual runs should also be supported. The existing build workflow can add `workflow_dispatch` with an `e2e_suite` input:
+
+```text
+p0 | p1 | p2 | all
+```
+
+The manual run should map the input to:
+
+- `p0`: `npm run ci:p0`
+- `p1`: `npm run test:p1 -- --workers=1`
+- `p2`: `npm run test:p2 -- --workers=1`
+- `all`: `npm run test:all -- --workers=1`
+
+Artifact upload must include:
+
+- `e2e/playwright-report/`
+- `e2e/test-results/`
+- `e2e/reports/run.log`
+- `e2e/reports/results.json`
+- `e2e/reports/junit.xml`
+
+The GitHub summary should include the suite name, pass/fail status, total test count, failed test names, and a pointer to the uploaded HTML report artifact.
+
+## Phased Implementation
+
+Implementation must proceed batch by batch. A later batch should not start until the current batch is implemented, verified, and reviewed.
+
+Batch 0 prepares the harness:
+
+- Create the root `e2e/` package.
+- Add Playwright config, isolated MySQL compose file, setup projects, fixtures, scripts, and README.
+- Add local manual run commands and helper scripts.
+- Add GitHub artifact wiring for logs and reports.
+- Verify that the empty harness can start MySQL, backend, frontend, and one minimal smoke test.
+
+Batch 1 implements P0:
+
+- Add P0 smoke, auth API, access-control, runtime-agent, and admin runtime Pods tests.
+- Add an `e2e-p0` GitHub Actions PR check to the existing build workflow.
+- Make `npm run ci:p0` the first GitHub PR e2e command.
+- Fix selectors, setup data, and server orchestration until P0 passes locally and in CI.
+- Stop after P0 and review results before adding P1.
+
+Batch 2 implements P1:
+
+- Add dashboard, instance list/detail, workspace file, and runtime metrics/gateway reporting tests.
+- Add any deterministic V2 seed helpers required by those specs.
+- Keep P0 as the required PR gate until P1 has passed repeatedly.
+- Review P1 stability before making it required in CI.
+
+Batch 3 implements P2:
+
+- Add expanded admin operations, instance action, cross-user visibility, and k8s deployed smoke tests.
+- Keep k8s browser tests manual or scheduled first.
+- Promote P2 subsets into PR CI only after timing and runtime image stability are proven.
+
+## Acceptance Criteria
+
+- `e2e/` exists at the repository root and owns all browser e2e test assets.
+- Developers can run the suite without modifying local frontend or backend configuration files.
+- Developers can manually run P0, P1, P2, or all e2e tests locally from `e2e/`.
+- The suite uses an isolated database and workspace cache path.
+- GitHub Actions runs the P0 e2e suite as a pull request check.
+- GitHub Actions can also run e2e manually through `workflow_dispatch`.
+- CI can run the P0 suite and upload failure artifacts.
+- E2E source files are written in TypeScript.
+- E2E runs show readable live progress in the terminal.
+- E2E runs generate `e2e/reports/run.log`.
+- E2E runs generate a final HTML report in `e2e/playwright-report/`.
+- E2E runs generate JSON and JUnit result files for CI consumption.
+- Repeated login, API, setup, navigation, and runtime-agent operations are implemented through shared fixtures, helpers, setup files, or page objects.
+- Specs do not duplicate URLs, credentials, selectors, or raw runtime-agent payload construction when a shared module exists.
+- Tests are grouped by product surface, assigned P0/P1/P2 priority, and can be expanded without changing the directory model.
+- The full necessary test inventory is documented before implementation begins.
+- Each implementation batch is verified before the next batch starts.
+- Initial tests do not require a real k8s cluster or runtime agent.
diff --git a/e2e/.gitignore b/e2e/.gitignore
new file mode 100644
index 0000000..e403392
--- /dev/null
+++ b/e2e/.gitignore
@@ -0,0 +1,7 @@
+node_modules/
+playwright-report/
+test-results/
+reports/*
+!reports/.gitkeep
+.auth/
+.cache/
diff --git a/e2e/README.md b/e2e/README.md
new file mode 100644
index 0000000..31709c6
--- /dev/null
+++ b/e2e/README.md
@@ -0,0 +1,47 @@
+# ClawManager E2E
+
+This directory contains Playwright-based TypeScript e2e tests.
+
+## First-Time Setup
+
+```bash
+npm ci
+npx playwright install chromium
+```
+
+## Run Locally
+
+```bash
+npm run test:p0
+npm run test:report
+```
+
+The default local gate is P0. P1, P2, and all tests are explicit:
+
+```bash
+npm run test:p1
+npm run test:p2
+npm run test:all
+```
+
+## Run Against A Target Environment
+
+Pass only the ClawManager address. The runner defaults to the P0 suite, derives
+the backend API as `/api/v1`, runs serially, skips local-only data setup, and
+writes a dedicated target log.
+
+```bash
+npm run target -- 172.16.1.12:39443
+```
+
+The address may include a protocol. When omitted, `https://` is used.
+
+## Reports
+
+- Live progress appears in the terminal.
+- Local run log: `reports/run.log`
+- Target run log: `reports/target-run.log`
+- JSON: `reports/results.json`
+- JUnit: `reports/junit.xml`
+- HTML: `playwright-report/`
+- Raw artifacts: `test-results/`
diff --git a/e2e/docker/docker-compose.e2e.yml b/e2e/docker/docker-compose.e2e.yml
new file mode 100644
index 0000000..117e7c1
--- /dev/null
+++ b/e2e/docker/docker-compose.e2e.yml
@@ -0,0 +1,16 @@
+services:
+ mysql:
+ image: mysql:8.0
+ container_name: clawmanager-e2e-mysql
+ environment:
+ MYSQL_ROOT_PASSWORD: "123456"
+ MYSQL_DATABASE: "clawmanager_e2e"
+ ports:
+ - "13307:3306"
+ command: --default-authentication-plugin=mysql_native_password
+ healthcheck:
+ test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-uroot", "-p123456"]
+ interval: 5s
+ timeout: 5s
+ retries: 20
+
diff --git a/e2e/docs/lite-pro-feature-matrix.md b/e2e/docs/lite-pro-feature-matrix.md
new file mode 100644
index 0000000..038cc75
--- /dev/null
+++ b/e2e/docs/lite-pro-feature-matrix.md
@@ -0,0 +1,14 @@
+# Lite / Pro Feature Matrix
+
+This matrix is the source checklist for the Playwright coverage in
+`tests/instances/lite-pro-modes.spec.ts`.
+
+| Area | Lite | Pro | E2E coverage |
+| --- | --- | --- | --- |
+| Create wizard | Selectable `Lite` mode. Runtime backend is derived as `Gateway`. | Selectable `Pro` mode. Runtime backend is derived as `Desktop`. | `create wizard exposes the lite/pro mode contract` |
+| Resource selection | Does not require or show dedicated CPU, memory, storage, or GPU sizing. Dedicated resource quota checks are not applied. | Shows dedicated resource presets and GPU toggle. CPU, memory, storage, and GPU quota checks are applied. | `create wizard exposes the lite/pro mode contract` |
+| Runtime ownership | Uses the shared gateway runtime pool and keeps gateway-level identity separate. | Uses a dedicated desktop runtime backed by Kubernetes Deployment + Service. | `admin instances page surfaces mode and backend management controls` |
+| Detail layout | Shows the service frame and workspace manager without desktop-only management panels. | Shows the service frame plus a merged Runtime Overview card, Runtime Events, Instance Skills, and workspace manager rooted at `/config`. | `lite detail keeps gateway workspace layout`, `pro detail keeps desktop management layout` |
+| Channel / skill management | Supports OpenClaw injection, channel selection, skill injection, import, and export at create/manage time. | Supports OpenClaw injection, channel selection, skill injection, import, export, and instance skill attach/remove. | Create wizard assertions plus pro detail skill panel assertion |
+| External access | Supports short Share Link and Password access with fixed, custom, or permanent expiration. | Supports short Share Link and Password access with fixed, custom, or permanent expiration. | `password share link opens a password gate` |
+| Admin monitoring | Admin instance management can show/filter Lite records and identify them as `Gateway binding` / `Gateway`. | Admin instance management can show/filter Pro records and identify them as `Deployment` / `Desktop`. | `admin instances page surfaces mode and backend management controls` |
diff --git a/e2e/fixtures/apiClient.ts b/e2e/fixtures/apiClient.ts
new file mode 100644
index 0000000..f2e11cc
--- /dev/null
+++ b/e2e/fixtures/apiClient.ts
@@ -0,0 +1,398 @@
+import type { APIRequestContext, APIResponse } from "@playwright/test";
+import { env } from "./env.js";
+import { users } from "./users.js";
+
+interface ApiEnvelope {
+ success: boolean;
+ message?: string;
+ data: T;
+ error?: string;
+}
+
+export interface LoginTokens {
+ access_token: string;
+ refresh_token: string;
+ expires_in: number;
+}
+
+export interface CurrentUser {
+ id: number;
+ username: string;
+ email: string;
+ role: string;
+}
+
+export interface RuntimePod {
+ id: number;
+ runtime_type: "openclaw" | "hermes";
+ namespace: string;
+ pod_name: string;
+ pod_uid?: string;
+ pod_ip?: string;
+ node_name?: string;
+ deployment_name: string;
+ image_ref: string;
+ state: string;
+ capacity: number;
+ used_slots: number;
+ draining: boolean;
+}
+
+export interface InstanceRecord {
+ id: number;
+ user_id: number;
+ name: string;
+ type: string;
+ runtime_type: "desktop" | "shell" | "gateway" | string;
+ instance_mode?: "lite" | "pro" | string;
+ status: "creating" | "running" | "stopped" | "error" | "deleting" | string;
+ cpu_cores: number;
+ memory_gb: number;
+ disk_gb: number;
+ gpu_enabled: boolean;
+ gpu_count: number;
+ os_type?: string;
+ os_version?: string;
+ pod_name?: string;
+ pod_namespace?: string;
+ pod_ip?: string;
+ workspace_usage_bytes?: number;
+ created_at?: string;
+ updated_at?: string;
+}
+
+export interface InstanceListResponse {
+ instances: InstanceRecord[];
+ total: number;
+ page: number;
+ limit: number;
+}
+
+export interface TeamRecord {
+ id: number;
+ user_id: number;
+ name: string;
+ description?: string;
+ status: "creating" | "running" | "failed" | "deleting" | "deleted" | string;
+ communication_mode: string;
+ shared_mount_path: string;
+ created_at?: string;
+ updated_at?: string;
+}
+
+export interface TeamMemberRecord {
+ id: number;
+ team_id: number;
+ instance_id?: number;
+ user_id: number;
+ member_key: string;
+ display_name: string;
+ role: string;
+ runtime_type?: "openclaw" | "hermes" | string;
+ instance_mode?: "lite" | "pro" | string;
+ status: "creating" | "idle" | "busy" | "failed" | "offline" | "deleting" | "deleted" | string;
+}
+
+export interface TeamDetailsRecord {
+ team: TeamRecord;
+ leader_member_id?: string;
+ leader?: TeamMemberRecord;
+ members: TeamMemberRecord[];
+}
+
+export interface CreateTeamMemberRequest {
+ member_id?: string;
+ name?: string;
+ role: string;
+ mode?: "lite" | "pro";
+ instance_mode?: "lite" | "pro";
+ runtime_type?: "openclaw" | "hermes";
+ description?: string;
+ cpu_cores?: number;
+ memory_gb?: number;
+ disk_gb?: number;
+ gpu_enabled?: boolean;
+ gpu_count?: number;
+ is_leader?: boolean;
+}
+
+export interface CreateTeamRequest {
+ name: string;
+ description?: string;
+ communication_mode?: string;
+ shared_storage_gb?: number;
+ members: CreateTeamMemberRequest[];
+}
+
+export interface InstanceExternalAccess {
+ id: number;
+ instance_id: number;
+ enabled: boolean;
+ auth_mode: "share_link" | "password" | string;
+ password_hint?: string;
+ expires_at?: string;
+ created_by?: number;
+ last_used_at?: string;
+ created_at: string;
+ updated_at: string;
+}
+
+export interface ExternalAccessRequest {
+ expires_mode?: "preset" | "custom" | "permanent";
+ expires_preset?: "1h" | "24h" | "7d" | "30d";
+ expires_at?: string;
+}
+
+export interface PasswordExternalAccessResult {
+ access: InstanceExternalAccess;
+ password: string;
+ share_url?: string;
+}
+
+export interface RuntimePodRegisterRequest {
+ runtime_type: "openclaw" | "hermes" | string;
+ namespace: string;
+ pod_name: string;
+ pod_uid?: string;
+ pod_ip?: string;
+ node_name?: string;
+ deployment_name: string;
+ image_ref: string;
+ agent_endpoint?: string;
+ state: string;
+ capacity: number;
+ used_slots: number;
+ draining: boolean;
+}
+
+export interface RuntimePodIdentity {
+ pod_id?: number;
+ namespace?: string;
+ pod_name?: string;
+}
+
+export interface RuntimeHeartbeatRequest extends RuntimePodIdentity {
+ state: string;
+ used_slots: number;
+ draining: boolean;
+}
+
+export interface LoginUser {
+ username: string;
+ password: string;
+}
+
+export interface RegisterUser extends LoginUser {
+ email: string;
+}
+
+export const defaultRuntimePod: RuntimePodRegisterRequest = {
+ runtime_type: "openclaw",
+ namespace: "clawmanager-e2e",
+ pod_name: "e2e-openclaw-runtime-p0",
+ pod_uid: "e2e-openclaw-runtime-p0-uid",
+ pod_ip: "10.244.0.25",
+ node_name: "e2e-node",
+ deployment_name: "openclaw-runtime",
+ image_ref: "e2e/openclaw:latest",
+ agent_endpoint: "http://10.244.0.25:18080",
+ state: "ready",
+ capacity: 100,
+ used_slots: 0,
+ draining: false
+};
+
+function bearer(accessToken: string) {
+ return { Authorization: `Bearer ${accessToken}` };
+}
+
+function runtimeAgentHeaders() {
+ return { "X-ClawManager-Agent-Token": env.runtimeAgentToken };
+}
+
+async function parseJson(response: APIResponse): Promise {
+ const text = await response.text();
+ if (!text) {
+ return undefined as T;
+ }
+ return JSON.parse(text) as T;
+}
+
+async function expectOkEnvelope(response: APIResponse): Promise {
+ const payload = await parseJson>(response);
+ if (!response.ok() || !payload.success) {
+ throw new Error(payload?.error || `request failed with status ${response.status()}`);
+ }
+ return payload.data;
+}
+
+export async function login(
+ request: APIRequestContext,
+ user: LoginUser = users.admin
+): Promise {
+ const response = await request.post(`${env.backendUrl}/auth/login`, {
+ data: {
+ username: user.username,
+ password: user.password
+ }
+ });
+ return expectOkEnvelope(response);
+}
+
+export async function registerUser(
+ request: APIRequestContext,
+ user: RegisterUser = users.user
+): Promise {
+ const response = await request.post(`${env.backendUrl}/auth/register`, {
+ data: {
+ username: user.username,
+ email: user.email,
+ password: user.password
+ }
+ });
+ await expectOkEnvelope(response);
+}
+
+export async function getMe(
+ request: APIRequestContext,
+ accessToken: string
+): Promise {
+ const response = await request.get(`${env.backendUrl}/auth/me`, {
+ headers: bearer(accessToken)
+ });
+ return expectOkEnvelope(response);
+}
+
+export async function getMeRaw(
+ request: APIRequestContext,
+ accessToken?: string
+): Promise {
+ return request.get(`${env.backendUrl}/auth/me`, {
+ headers: accessToken ? bearer(accessToken) : undefined
+ });
+}
+
+export async function postRuntimePodRegisterRaw(
+ request: APIRequestContext,
+ overrides: Partial = {}
+): Promise {
+ return request.post(`${env.backendUrl}/runtime-agent/register`, {
+ headers: runtimeAgentHeaders(),
+ data: {
+ ...defaultRuntimePod,
+ ...overrides
+ }
+ });
+}
+
+export async function registerRuntimePod(
+ request: APIRequestContext,
+ overrides: Partial = {}
+): Promise {
+ const response = await postRuntimePodRegisterRaw(request, overrides);
+ const data = await expectOkEnvelope<{ pod: RuntimePod }>(response);
+ return data.pod;
+}
+
+export async function sendRuntimeHeartbeat(
+ request: APIRequestContext,
+ identity: RuntimePodIdentity,
+ overrides: Partial = {}
+): Promise {
+ const response = await request.post(`${env.backendUrl}/runtime-agent/heartbeat`, {
+ headers: runtimeAgentHeaders(),
+ data: {
+ state: "ready",
+ used_slots: 0,
+ draining: false,
+ ...identity,
+ ...overrides
+ }
+ });
+ await expectOkEnvelope(response);
+}
+
+export async function listRuntimePods(
+ request: APIRequestContext,
+ accessToken: string
+): Promise {
+ const response = await request.get(`${env.backendUrl}/admin/runtime-pods`, {
+ headers: bearer(accessToken)
+ });
+ const data = await expectOkEnvelope<{ pods: RuntimePod[] }>(response);
+ return data.pods;
+}
+
+export async function listInstances(
+ request: APIRequestContext,
+ accessToken: string,
+ options: { admin?: boolean; page?: number; limit?: number } = {}
+): Promise {
+ const endpoint = options.admin ? "/admin/instances" : "/instances";
+ const response = await request.get(`${env.backendUrl}${endpoint}`, {
+ headers: bearer(accessToken),
+ params: {
+ page: options.page ?? 1,
+ limit: options.limit ?? 100
+ }
+ });
+ return expectOkEnvelope(response);
+}
+
+export async function createTeam(
+ request: APIRequestContext,
+ accessToken: string,
+ data: CreateTeamRequest
+): Promise {
+ const response = await request.post(`${env.backendUrl}/teams`, {
+ headers: bearer(accessToken),
+ data
+ });
+ return expectOkEnvelope(response);
+}
+
+export async function getTeam(
+ request: APIRequestContext,
+ accessToken: string,
+ teamId: number
+): Promise {
+ const response = await request.get(`${env.backendUrl}/teams/${teamId}`, {
+ headers: bearer(accessToken)
+ });
+ return expectOkEnvelope(response);
+}
+
+export async function deleteTeam(
+ request: APIRequestContext,
+ accessToken: string,
+ teamId: number
+): Promise {
+ const response = await request.delete(`${env.backendUrl}/teams/${teamId}`, {
+ headers: bearer(accessToken)
+ });
+ await expectOkEnvelope<{ id: number }>(response);
+}
+
+export async function createExternalAccessPassword(
+ request: APIRequestContext,
+ accessToken: string,
+ instanceId: number,
+ data: ExternalAccessRequest = { expires_mode: "preset", expires_preset: "1h" }
+): Promise {
+ const response = await request.post(`${env.backendUrl}/instances/${instanceId}/external-access/password`, {
+ headers: bearer(accessToken),
+ data
+ });
+ return expectOkEnvelope(response);
+}
+
+export async function disableExternalAccess(
+ request: APIRequestContext,
+ accessToken: string,
+ instanceId: number
+): Promise {
+ const response = await request.delete(`${env.backendUrl}/instances/${instanceId}/external-access`, {
+ headers: bearer(accessToken)
+ });
+ await expectOkEnvelope(response);
+}
diff --git a/e2e/fixtures/env.ts b/e2e/fixtures/env.ts
new file mode 100644
index 0000000..28eb8e4
--- /dev/null
+++ b/e2e/fixtures/env.ts
@@ -0,0 +1,64 @@
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const fixturesDir = path.dirname(fileURLToPath(import.meta.url));
+export const e2eRoot = path.resolve(fixturesDir, "..");
+export const repoRoot = path.resolve(e2eRoot, "..");
+export const reportsDir = path.join(e2eRoot, "reports");
+export const testResultsDir = path.join(e2eRoot, "test-results");
+
+function readEnv(name: string, fallback: string): string {
+ const value = process.env[name]?.trim();
+ return value && value.length > 0 ? value : fallback;
+}
+
+function readNumberEnv(name: string, fallback: number): number {
+ const raw = process.env[name]?.trim();
+ if (!raw) {
+ return fallback;
+ }
+ const parsed = Number.parseInt(raw, 10);
+ if (!Number.isFinite(parsed)) {
+ throw new Error(`${name} must be a number`);
+ }
+ return parsed;
+}
+
+function readBooleanEnv(name: string, fallback: boolean): boolean {
+ const raw = process.env[name]?.trim();
+ if (!raw) {
+ return fallback;
+ }
+ return ["1", "true", "yes", "on"].includes(raw.toLowerCase());
+}
+
+function withoutTrailingSlash(value: string): string {
+ return value.replace(/\/+$/, "");
+}
+
+const isCI = process.env.CI === "true" || process.env.E2E_CI === "true";
+const debug = readBooleanEnv("E2E_DEBUG", false);
+
+export const env = {
+ frontendUrl: withoutTrailingSlash(readEnv("E2E_FRONTEND_URL", "http://127.0.0.1:9002")),
+ backendUrl: withoutTrailingSlash(readEnv("E2E_BACKEND_URL", "http://127.0.0.1:9001/api/v1")),
+ db: {
+ host: readEnv("E2E_DB_HOST", "127.0.0.1"),
+ port: readNumberEnv("E2E_DB_PORT", 13307),
+ user: readEnv("E2E_DB_USER", "root"),
+ password: readEnv("E2E_DB_PASSWORD", "123456"),
+ database: readEnv("E2E_DB_NAME", "clawmanager_e2e")
+ },
+ runtimeAgentToken: readEnv("E2E_RUNTIME_AGENT_TOKEN", "e2e-runtime-agent-token"),
+ workspaceRoot: readEnv("E2E_WORKSPACE_ROOT", path.join(repoRoot, ".cache", "e2e", "workspaces")),
+ objectStorageFallback: readEnv(
+ "E2E_OBJECT_STORAGE_FALLBACK",
+ path.join(repoRoot, ".cache", "e2e", "object-storage")
+ ),
+ ignoreHttpsErrors: readBooleanEnv("E2E_IGNORE_HTTPS_ERRORS", false),
+ headless: readBooleanEnv("E2E_HEADLESS", isCI),
+ debug,
+ slowMoMs: debug ? readNumberEnv("E2E_SLOW_MO_MS", 500) : 0,
+ remoteTarget: readBooleanEnv("E2E_REMOTE_TARGET", false),
+ isCI
+} as const;
diff --git a/e2e/fixtures/test.ts b/e2e/fixtures/test.ts
new file mode 100644
index 0000000..b015793
--- /dev/null
+++ b/e2e/fixtures/test.ts
@@ -0,0 +1,11 @@
+import { expect, test as base } from "@playwright/test";
+import { env } from "./env.js";
+
+export const test = base.extend({
+ baseURL: async ({}, use) => {
+ await use(env.frontendUrl);
+ }
+});
+
+export { expect };
+
diff --git a/e2e/fixtures/users.ts b/e2e/fixtures/users.ts
new file mode 100644
index 0000000..678a96b
--- /dev/null
+++ b/e2e/fixtures/users.ts
@@ -0,0 +1,14 @@
+export const users = {
+ admin: {
+ username: "admin",
+ password: "admin123",
+ role: "admin",
+ storageState: ".auth/admin.json"
+ },
+ user: {
+ username: "e2euser",
+ email: "e2euser@clawmanager.local",
+ password: "user12345",
+ role: "user"
+ }
+} as const;
diff --git a/e2e/package-lock.json b/e2e/package-lock.json
new file mode 100644
index 0000000..7003bff
--- /dev/null
+++ b/e2e/package-lock.json
@@ -0,0 +1,767 @@
+{
+ "name": "clawmanager-e2e",
+ "version": "0.0.0",
+ "lockfileVersion": 3,
+ "requires": true,
+ "packages": {
+ "": {
+ "name": "clawmanager-e2e",
+ "version": "0.0.0",
+ "devDependencies": {
+ "@playwright/test": "^1.49.1",
+ "@types/node": "^24.12.0",
+ "mysql2": "^3.11.5",
+ "tsx": "^4.19.2",
+ "typescript": "^5.9.3"
+ }
+ },
+ "node_modules/@esbuild/aix-ppc64": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.0.tgz",
+ "integrity": "sha512-lhRUCeuOyJQURhTxl4WkpFTjIsbDayJHih5kZC1giwE+MhIzAb7mEsQMqMf18rHLsrb5qI1tafG20mLxEWcWlA==",
+ "cpu": [
+ "ppc64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "aix"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@esbuild/android-arm": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.0.tgz",
+ "integrity": "sha512-wqh0ByljabXLKHeWXYLqoJ5jKC4XBaw6Hk08OfMrCRd2nP2ZQ5eleDZC41XHyCNgktBGYMbqnrJKq/K/lzPMSQ==",
+ "cpu": [
+ "arm"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "android"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@esbuild/android-arm64": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.0.tgz",
+ "integrity": "sha512-+WzIXQOSaGs33tLEgYPYe/yQHf0WTU0X42Jca3y8NWMbUVhp7rUnw+vAsRC/QiDrdD31IszMrZy+qwPOPjd+rw==",
+ "cpu": [
+ "arm64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "android"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@esbuild/android-x64": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.0.tgz",
+ "integrity": "sha512-+VJggoaKhk2VNNqVL7f6S189UzShHC/mR9EE8rDdSkdpN0KflSwWY/gWjDrNxxisg8Fp1ZCD9jLMo4m0OUfeUA==",
+ "cpu": [
+ "x64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "android"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@esbuild/darwin-arm64": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.0.tgz",
+ "integrity": "sha512-0T+A9WZm+bZ84nZBtk1ckYsOvyA3x7e2Acj1KdVfV4/2tdG4fzUp91YHx+GArWLtwqp77pBXVCPn2We7Letr0Q==",
+ "cpu": [
+ "arm64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "darwin"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@esbuild/darwin-x64": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.0.tgz",
+ "integrity": "sha512-fyzLm/DLDl/84OCfp2f/XQ4flmORsjU7VKt8HLjvIXChJoFFOIL6pLJPH4Yhd1n1gGFF9mPwtlN5Wf82DZs+LQ==",
+ "cpu": [
+ "x64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "darwin"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@esbuild/freebsd-arm64": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.0.tgz",
+ "integrity": "sha512-l9GeW5UZBT9k9brBYI+0WDffcRxgHQD8ShN2Ur4xWq/NFzUKm3k5lsH4PdaRgb2w7mI9u61nr2gI2mLI27Nh3Q==",
+ "cpu": [
+ "arm64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "freebsd"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@esbuild/freebsd-x64": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.0.tgz",
+ "integrity": "sha512-BXoQai/A0wPO6Es3yFJ7APCiKGc1tdAEOgeTNy3SsB491S3aHn4S4r3e976eUnPdU+NbdtmBuLncYir2tMU9Nw==",
+ "cpu": [
+ "x64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "freebsd"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@esbuild/linux-arm": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.0.tgz",
+ "integrity": "sha512-CjaaREJagqJp7iTaNQjjidaNbCKYcd4IDkzbwwxtSvjI7NZm79qiHc8HqciMddQ6CKvJT6aBd8lO9kN/ZudLlw==",
+ "cpu": [
+ "arm"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@esbuild/linux-arm64": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.0.tgz",
+ "integrity": "sha512-RVyzfb3FWsGA55n6WY0MEIEPURL1FcbhFE6BffZEMEekfCzCIMtB5yyDcFnVbTnwk+CLAgTujmV/Lgvih56W+A==",
+ "cpu": [
+ "arm64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@esbuild/linux-ia32": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.0.tgz",
+ "integrity": "sha512-KBnSTt1kxl9x70q+ydterVdl+Cn0H18ngRMRCEQfrbqdUuntQQ0LoMZv47uB97NljZFzY6HcfqEZ2SAyIUTQBQ==",
+ "cpu": [
+ "ia32"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@esbuild/linux-loong64": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.0.tgz",
+ "integrity": "sha512-zpSlUce1mnxzgBADvxKXX5sl8aYQHo2ezvMNI8I0lbblJtp8V4odlm3Yzlj7gPyt3T8ReksE6bK+pT3WD+aJRg==",
+ "cpu": [
+ "loong64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@esbuild/linux-mips64el": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.0.tgz",
+ "integrity": "sha512-2jIfP6mmjkdmeTlsX/9vmdmhBmKADrWqN7zcdtHIeNSCH1SqIoNI63cYsjQR8J+wGa4Y5izRcSHSm8K3QWmk3w==",
+ "cpu": [
+ "mips64el"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@esbuild/linux-ppc64": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.0.tgz",
+ "integrity": "sha512-bc0FE9wWeC0WBm49IQMPSPILRocGTQt3j5KPCA8os6VprfuJ7KD+5PzESSrJ6GmPIPJK965ZJHTUlSA6GNYEhg==",
+ "cpu": [
+ "ppc64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@esbuild/linux-riscv64": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.0.tgz",
+ "integrity": "sha512-SQPZOwoTTT/HXFXQJG/vBX8sOFagGqvZyXcgLA3NhIqcBv1BJU1d46c0rGcrij2B56Z2rNiSLaZOYW5cUk7yLQ==",
+ "cpu": [
+ "riscv64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@esbuild/linux-s390x": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.0.tgz",
+ "integrity": "sha512-SCfR0HN8CEEjnYnySJTd2cw0k9OHB/YFzt5zgJEwa+wL/T/raGWYMBqwDNAC6dqFKmJYZoQBRfHjgwLHGSrn3Q==",
+ "cpu": [
+ "s390x"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@esbuild/linux-x64": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.0.tgz",
+ "integrity": "sha512-us0dSb9iFxIi8srnpl931Nvs65it/Jd2a2K3qs7fz2WfGPHqzfzZTfec7oxZJRNPXPnNYZtanmRc4AL/JwVzHQ==",
+ "cpu": [
+ "x64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "linux"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@esbuild/netbsd-arm64": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.0.tgz",
+ "integrity": "sha512-CR/RYotgtCKwtftMwJlUU7xCVNg3lMYZ0RzTmAHSfLCXw3NtZtNpswLEj/Kkf6kEL3Gw+BpOekRX0BYCtklhUw==",
+ "cpu": [
+ "arm64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "netbsd"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@esbuild/netbsd-x64": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.0.tgz",
+ "integrity": "sha512-nU1yhmYutL+fQ71Kxnhg8uEOdC0pwEW9entHykTgEbna2pw2dkbFSMeqjjyHZoCmt8SBkOSvV+yNmm94aUrrqw==",
+ "cpu": [
+ "x64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "netbsd"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@esbuild/openbsd-arm64": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.0.tgz",
+ "integrity": "sha512-cXb5vApOsRsxsEl4mcZ1XY3D4DzcoMxR/nnc4IyqYs0rTI8ZKmW6kyyg+11Z8yvgMfAEldKzP7AdP64HnSC/6g==",
+ "cpu": [
+ "arm64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "openbsd"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@esbuild/openbsd-x64": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.0.tgz",
+ "integrity": "sha512-8wZM2qqtv9UP3mzy7HiGYNH/zjTA355mpeuA+859TyR+e+Tc08IHYpLJuMsfpDJwoLo1ikIJI8jC3GFjnRClzA==",
+ "cpu": [
+ "x64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "openbsd"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@esbuild/openharmony-arm64": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.0.tgz",
+ "integrity": "sha512-FLGfyizszcef5C3YtoyQDACyg95+dndv79i2EekILBofh5wpCa1KuBqOWKrEHZg3zrL3t5ouE5jgr94vA+Wb2w==",
+ "cpu": [
+ "arm64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "openharmony"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@esbuild/sunos-x64": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.0.tgz",
+ "integrity": "sha512-1ZgjUoEdHZZl/YlV76TSCz9Hqj9h9YmMGAgAPYd+q4SicWNX3G5GCyx9uhQWSLcbvPW8Ni7lj4gDa1T40akdlw==",
+ "cpu": [
+ "x64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "sunos"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@esbuild/win32-arm64": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.0.tgz",
+ "integrity": "sha512-Q9StnDmQ/enxnpxCCLSg0oo4+34B9TdXpuyPeTedN/6+iXBJ4J+zwfQI28u/Jl40nOYAxGoNi7mFP40RUtkmUA==",
+ "cpu": [
+ "arm64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "win32"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@esbuild/win32-ia32": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.0.tgz",
+ "integrity": "sha512-zF3ag/gfiCe6U2iczcRzSYJKH1DCI+ByzSENHlM2FcDbEeo5Zd2C86Aq0tKUYAJJ1obRP84ymxIAksZUcdztHA==",
+ "cpu": [
+ "ia32"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "win32"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@esbuild/win32-x64": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.0.tgz",
+ "integrity": "sha512-pEl1bO9mfAmIC+tW5btTmrKaujg3zGtUmWNdCw/xs70FBjwAL3o9OEKNHvNmnyylD6ubxUERiEhdsL0xBQ9efw==",
+ "cpu": [
+ "x64"
+ ],
+ "dev": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "win32"
+ ],
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@playwright/test": {
+ "version": "1.60.0",
+ "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.60.0.tgz",
+ "integrity": "sha512-O71yZIbAh/PxDMNGns37GHBIfrVkEVyn+AXyIa5dOTfb4/xNvRWV+Vv/NMbNCtODB/pO7vLlF2OTmMVLhmr7Ag==",
+ "dev": true,
+ "license": "Apache-2.0",
+ "dependencies": {
+ "playwright": "1.60.0"
+ },
+ "bin": {
+ "playwright": "cli.js"
+ },
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@types/node": {
+ "version": "24.12.4",
+ "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.4.tgz",
+ "integrity": "sha512-GUUEShf+PBCGW2KaXwcIt3Yk+e3pkKwWKb9GSyM9WQVE+ep2jzmHdGsHzu4wgcZy5fN9FBdVzjpBQsYlpfpgLA==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "undici-types": "~7.16.0"
+ }
+ },
+ "node_modules/aws-ssl-profiles": {
+ "version": "1.1.2",
+ "resolved": "https://registry.npmjs.org/aws-ssl-profiles/-/aws-ssl-profiles-1.1.2.tgz",
+ "integrity": "sha512-NZKeq9AfyQvEeNlN0zSYAaWrmBffJh3IELMZfRpJVWgrpEbtEpnjvzqBPf+mxoI287JohRDoa+/nsfqqiZmF6g==",
+ "dev": true,
+ "license": "MIT",
+ "engines": {
+ "node": ">= 6.0.0"
+ }
+ },
+ "node_modules/denque": {
+ "version": "2.1.0",
+ "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
+ "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
+ "dev": true,
+ "license": "Apache-2.0",
+ "engines": {
+ "node": ">=0.10"
+ }
+ },
+ "node_modules/esbuild": {
+ "version": "0.28.0",
+ "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.0.tgz",
+ "integrity": "sha512-sNR9MHpXSUV/XB4zmsFKN+QgVG82Cc7+/aaxJ8Adi8hyOac+EXptIp45QBPaVyX3N70664wRbTcLTOemCAnyqw==",
+ "dev": true,
+ "hasInstallScript": true,
+ "license": "MIT",
+ "bin": {
+ "esbuild": "bin/esbuild"
+ },
+ "engines": {
+ "node": ">=18"
+ },
+ "optionalDependencies": {
+ "@esbuild/aix-ppc64": "0.28.0",
+ "@esbuild/android-arm": "0.28.0",
+ "@esbuild/android-arm64": "0.28.0",
+ "@esbuild/android-x64": "0.28.0",
+ "@esbuild/darwin-arm64": "0.28.0",
+ "@esbuild/darwin-x64": "0.28.0",
+ "@esbuild/freebsd-arm64": "0.28.0",
+ "@esbuild/freebsd-x64": "0.28.0",
+ "@esbuild/linux-arm": "0.28.0",
+ "@esbuild/linux-arm64": "0.28.0",
+ "@esbuild/linux-ia32": "0.28.0",
+ "@esbuild/linux-loong64": "0.28.0",
+ "@esbuild/linux-mips64el": "0.28.0",
+ "@esbuild/linux-ppc64": "0.28.0",
+ "@esbuild/linux-riscv64": "0.28.0",
+ "@esbuild/linux-s390x": "0.28.0",
+ "@esbuild/linux-x64": "0.28.0",
+ "@esbuild/netbsd-arm64": "0.28.0",
+ "@esbuild/netbsd-x64": "0.28.0",
+ "@esbuild/openbsd-arm64": "0.28.0",
+ "@esbuild/openbsd-x64": "0.28.0",
+ "@esbuild/openharmony-arm64": "0.28.0",
+ "@esbuild/sunos-x64": "0.28.0",
+ "@esbuild/win32-arm64": "0.28.0",
+ "@esbuild/win32-ia32": "0.28.0",
+ "@esbuild/win32-x64": "0.28.0"
+ }
+ },
+ "node_modules/fsevents": {
+ "version": "2.3.2",
+ "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
+ "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
+ "dev": true,
+ "hasInstallScript": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "darwin"
+ ],
+ "engines": {
+ "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+ }
+ },
+ "node_modules/generate-function": {
+ "version": "2.3.1",
+ "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.3.1.tgz",
+ "integrity": "sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "is-property": "^1.0.2"
+ }
+ },
+ "node_modules/iconv-lite": {
+ "version": "0.7.2",
+ "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz",
+ "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "safer-buffer": ">= 2.1.2 < 3.0.0"
+ },
+ "engines": {
+ "node": ">=0.10.0"
+ },
+ "funding": {
+ "type": "opencollective",
+ "url": "https://opencollective.com/express"
+ }
+ },
+ "node_modules/is-property": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz",
+ "integrity": "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==",
+ "dev": true,
+ "license": "MIT"
+ },
+ "node_modules/long": {
+ "version": "5.3.2",
+ "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz",
+ "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==",
+ "dev": true,
+ "license": "Apache-2.0"
+ },
+ "node_modules/lru.min": {
+ "version": "1.1.4",
+ "resolved": "https://registry.npmjs.org/lru.min/-/lru.min-1.1.4.tgz",
+ "integrity": "sha512-DqC6n3QQ77zdFpCMASA1a3Jlb64Hv2N2DciFGkO/4L9+q/IpIAuRlKOvCXabtRW6cQf8usbmM6BE/TOPysCdIA==",
+ "dev": true,
+ "license": "MIT",
+ "engines": {
+ "bun": ">=1.0.0",
+ "deno": ">=1.30.0",
+ "node": ">=8.0.0"
+ },
+ "funding": {
+ "type": "github",
+ "url": "https://github.com/sponsors/wellwelwel"
+ }
+ },
+ "node_modules/mysql2": {
+ "version": "3.22.4",
+ "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.22.4.tgz",
+ "integrity": "sha512-CtXYlmL7ZamiYKbmqkamQHWJROUHSfm+f3kByzGfknw7kW51mcB2ouMUqYq1XfYxbXmnWo6RhPydx6OCqdgcmQ==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "aws-ssl-profiles": "^1.1.2",
+ "denque": "^2.1.0",
+ "generate-function": "^2.3.1",
+ "iconv-lite": "^0.7.2",
+ "long": "^5.3.2",
+ "lru.min": "^1.1.4",
+ "named-placeholders": "^1.1.6",
+ "sql-escaper": "^1.3.3"
+ },
+ "engines": {
+ "node": ">= 8.0"
+ },
+ "peerDependencies": {
+ "@types/node": ">= 8"
+ }
+ },
+ "node_modules/named-placeholders": {
+ "version": "1.1.6",
+ "resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.6.tgz",
+ "integrity": "sha512-Tz09sEL2EEuv5fFowm419c1+a/jSMiBjI9gHxVLrVdbUkkNUUfjsVYs9pVZu5oCon/kmRh9TfLEObFtkVxmY0w==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "lru.min": "^1.1.0"
+ },
+ "engines": {
+ "node": ">=8.0.0"
+ }
+ },
+ "node_modules/playwright": {
+ "version": "1.60.0",
+ "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.60.0.tgz",
+ "integrity": "sha512-hheHdokM8cdqCb0lcE3s+zT4t4W+vvjpGxsZlDnikarzx8tSzMebh3UiFtgqwFwnTnjYQcsyMF8ei2mCO/tpeA==",
+ "dev": true,
+ "license": "Apache-2.0",
+ "dependencies": {
+ "playwright-core": "1.60.0"
+ },
+ "bin": {
+ "playwright": "cli.js"
+ },
+ "engines": {
+ "node": ">=18"
+ },
+ "optionalDependencies": {
+ "fsevents": "2.3.2"
+ }
+ },
+ "node_modules/playwright-core": {
+ "version": "1.60.0",
+ "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.60.0.tgz",
+ "integrity": "sha512-9bW6zvX/m0lEbgTKJ6YppOKx8H3VOPBMOCFh2irXFOT4BbHgrx5hPjwJYLT40Lu+4qtD36qKc/Hn56StUW57IA==",
+ "dev": true,
+ "license": "Apache-2.0",
+ "bin": {
+ "playwright-core": "cli.js"
+ },
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/safer-buffer": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
+ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
+ "dev": true,
+ "license": "MIT"
+ },
+ "node_modules/sql-escaper": {
+ "version": "1.3.3",
+ "resolved": "https://registry.npmjs.org/sql-escaper/-/sql-escaper-1.3.3.tgz",
+ "integrity": "sha512-BsTCV265VpTp8tm1wyIm1xqQCS+Q9NHx2Sr+WcnUrgLrQ6yiDIvHYJV5gHxsj1lMBy2zm5twLaZao8Jd+S8JJw==",
+ "dev": true,
+ "license": "MIT",
+ "engines": {
+ "bun": ">=1.0.0",
+ "deno": ">=2.0.0",
+ "node": ">=12.0.0"
+ },
+ "funding": {
+ "type": "github",
+ "url": "https://github.com/mysqljs/sql-escaper?sponsor=1"
+ }
+ },
+ "node_modules/tsx": {
+ "version": "4.22.4",
+ "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.22.4.tgz",
+ "integrity": "sha512-X8EX+XV4QR5xCsrgxaED954zTDfY8KqlDtskKEL0cHhyS/P8b4IFOvGDQpsC9Q1XnLq915wEfwwY/zzskCtmhg==",
+ "dev": true,
+ "license": "MIT",
+ "dependencies": {
+ "esbuild": "~0.28.0"
+ },
+ "bin": {
+ "tsx": "dist/cli.mjs"
+ },
+ "engines": {
+ "node": ">=18.0.0"
+ },
+ "optionalDependencies": {
+ "fsevents": "~2.3.3"
+ }
+ },
+ "node_modules/tsx/node_modules/fsevents": {
+ "version": "2.3.3",
+ "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
+ "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",
+ "dev": true,
+ "hasInstallScript": true,
+ "license": "MIT",
+ "optional": true,
+ "os": [
+ "darwin"
+ ],
+ "engines": {
+ "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+ }
+ },
+ "node_modules/typescript": {
+ "version": "5.9.3",
+ "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
+ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
+ "dev": true,
+ "license": "Apache-2.0",
+ "bin": {
+ "tsc": "bin/tsc",
+ "tsserver": "bin/tsserver"
+ },
+ "engines": {
+ "node": ">=14.17"
+ }
+ },
+ "node_modules/undici-types": {
+ "version": "7.16.0",
+ "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz",
+ "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==",
+ "dev": true,
+ "license": "MIT"
+ }
+ }
+}
diff --git a/e2e/package.json b/e2e/package.json
new file mode 100644
index 0000000..713c1dd
--- /dev/null
+++ b/e2e/package.json
@@ -0,0 +1,28 @@
+{
+ "name": "clawmanager-e2e",
+ "private": true,
+ "version": "0.0.0",
+ "type": "module",
+ "scripts": {
+ "test:p0": "tsx scripts/run-with-log.ts p0",
+ "test:p1": "tsx scripts/run-with-log.ts p1",
+ "test:p2": "tsx scripts/run-with-log.ts p2",
+ "test:all": "tsx scripts/run-with-log.ts all",
+ "ci:p0": "tsx scripts/run-with-log.ts p0 --ci",
+ "target": "tsx scripts/run-target.ts",
+ "test:target-script": "tsx scripts/runner-utils.test.ts && tsx scripts/run-with-log.test.ts && tsx scripts/run-target.test.ts",
+ "pw:p0": "playwright test --grep @p0",
+ "pw:p1": "playwright test --grep @p1",
+ "pw:p2": "playwright test --grep @p2",
+ "pw:all": "playwright test",
+ "test:report": "playwright show-report playwright-report",
+ "db:reset": "tsx scripts/reset-db.ts"
+ },
+ "devDependencies": {
+ "@playwright/test": "^1.49.1",
+ "@types/node": "^24.12.0",
+ "mysql2": "^3.11.5",
+ "tsx": "^4.19.2",
+ "typescript": "^5.9.3"
+ }
+}
diff --git a/e2e/pages/AdminRuntimePodsPage.ts b/e2e/pages/AdminRuntimePodsPage.ts
new file mode 100644
index 0000000..9a7d72c
--- /dev/null
+++ b/e2e/pages/AdminRuntimePodsPage.ts
@@ -0,0 +1,64 @@
+import type { Page } from "@playwright/test";
+import type { RuntimePodRegisterRequest } from "../fixtures/apiClient.js";
+import { expect } from "../fixtures/test.js";
+
+export class AdminRuntimePodsPage {
+ constructor(private readonly page: Page) {}
+
+ private main() {
+ return this.page.getByRole("main");
+ }
+
+ private podCard(podName: string) {
+ return this.main().locator("article", { hasText: podName }).first();
+ }
+
+ private formatRuntimeType(runtimeType: RuntimePodRegisterRequest["runtime_type"]) {
+ return runtimeType === "hermes" ? "Hermes" : "OpenClaw";
+ }
+
+ async goto() {
+ await this.page.goto("/admin/runtime-pods");
+ }
+
+ async expectVisible() {
+ await expect(this.page.getByRole("heading", { name: /^runtime$/i })).toBeVisible();
+ }
+
+ async expectEmptyState() {
+ await this.expectVisible();
+ await expect(this.main().getByText("No active runtime pods")).toBeVisible();
+ }
+
+ async selectRuntimeFilter(label: "All" | "OpenClaw" | "Hermes") {
+ await this.main().getByRole("button", { name: label }).click();
+ }
+
+ async expectPodVisible(podName: string) {
+ await this.expectVisible();
+ const card = this.podCard(podName);
+
+ await expect(card).toBeVisible();
+ await expect(card.getByRole("heading", { name: podName, exact: true })).toBeVisible();
+ }
+
+ async expectPodStatusAndCapacity(pod: RuntimePodRegisterRequest) {
+ await this.expectPodVisible(pod.pod_name);
+ const card = this.podCard(pod.pod_name);
+
+ await expect(card.getByText(this.formatRuntimeType(pod.runtime_type), { exact: true })).toBeVisible();
+ await expect(card.getByText(pod.draining ? "draining" : pod.state, { exact: true })).toBeVisible();
+ await expect(card.getByText("Slots", { exact: true })).toBeVisible();
+ await expect(card.getByText(`${pod.used_slots} / ${pod.capacity}`, { exact: true })).toBeVisible();
+ }
+
+ async expectPodMetricsVisible(podName: string) {
+ const card = this.podCard(podName);
+
+ await expect(card.getByText("CPU", { exact: true })).toBeVisible();
+ await expect(card.getByText("Memory", { exact: true })).toBeVisible();
+ await expect(card.getByText("Disk", { exact: true })).toBeVisible();
+ await expect(card.getByText("Network", { exact: true })).toBeVisible();
+ await expect(card.getByText(/Last seen/i)).toBeVisible();
+ }
+}
diff --git a/e2e/pages/LoginPage.ts b/e2e/pages/LoginPage.ts
new file mode 100644
index 0000000..3304a74
--- /dev/null
+++ b/e2e/pages/LoginPage.ts
@@ -0,0 +1,23 @@
+import type { Page } from "@playwright/test";
+import { expect } from "../fixtures/test.js";
+
+export class LoginPage {
+ constructor(private readonly page: Page) {}
+
+ async goto() {
+ await this.page.goto("/login");
+ }
+
+ async expectVisible() {
+ await expect(this.page.getByRole("heading", { name: /sign in to clawmanager/i })).toBeVisible();
+ await expect(this.page.getByLabel(/username/i)).toBeVisible();
+ await expect(this.page.getByLabel(/password/i)).toBeVisible();
+ }
+
+ async login(username: string, password: string) {
+ await this.page.getByLabel(/username/i).fill(username);
+ await this.page.getByLabel(/password/i).fill(password);
+ await this.page.getByRole("button", { name: /^sign in$/i }).click();
+ }
+}
+
diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts
new file mode 100644
index 0000000..19e3287
--- /dev/null
+++ b/e2e/playwright.config.ts
@@ -0,0 +1,63 @@
+import path from "node:path";
+import { defineConfig, devices } from "@playwright/test";
+import { env, e2eRoot, reportsDir } from "./fixtures/env.js";
+
+const setupProjects = [
+ {
+ name: "auth setup",
+ testMatch: /setup\/auth\.setup\.ts/
+ },
+ ...(
+ env.remoteTarget
+ ? []
+ : [
+ {
+ name: "db setup",
+ testMatch: /setup\/db\.setup\.ts/
+ }
+ ]
+ )
+];
+
+export default defineConfig({
+ testDir: ".",
+ outputDir: "test-results",
+ timeout: env.remoteTarget ? 90_000 : 30_000,
+ expect: {
+ timeout: env.remoteTarget ? 45_000 : 10_000
+ },
+ fullyParallel: false,
+ retries: env.isCI ? 1 : 0,
+ workers: env.isCI ? 1 : undefined,
+ reporter: [
+ ["list"],
+ ["html", { outputFolder: "playwright-report", open: "never" }],
+ ["json", { outputFile: path.join(reportsDir, "results.json") }],
+ ["junit", { outputFile: path.join(reportsDir, "junit.xml") }]
+ ],
+ use: {
+ baseURL: env.frontendUrl,
+ headless: env.headless,
+ ignoreHTTPSErrors: env.ignoreHttpsErrors,
+ launchOptions: env.debug ? { slowMo: env.slowMoMs } : undefined,
+ trace: "retain-on-failure",
+ screenshot: "only-on-failure",
+ video: "retain-on-failure"
+ },
+ projects: [
+ ...setupProjects,
+ {
+ name: "chromium",
+ testMatch: /tests\/.*\.spec\.ts/,
+ dependencies: env.remoteTarget ? ["auth setup"] : ["auth setup", "db setup"],
+ use: {
+ ...devices["Desktop Chrome"]
+ }
+ }
+ ],
+ metadata: {
+ e2eRoot,
+ frontendUrl: env.frontendUrl,
+ backendUrl: env.backendUrl
+ }
+});
diff --git a/e2e/reports/.gitkeep b/e2e/reports/.gitkeep
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/e2e/reports/.gitkeep
@@ -0,0 +1 @@
+
diff --git a/e2e/scripts/e2e-down.ps1 b/e2e/scripts/e2e-down.ps1
new file mode 100644
index 0000000..9579a7f
--- /dev/null
+++ b/e2e/scripts/e2e-down.ps1
@@ -0,0 +1,10 @@
+Set-StrictMode -Version Latest
+$ErrorActionPreference = "Stop"
+
+Push-Location (Join-Path $PSScriptRoot "..")
+try {
+ docker compose -f docker/docker-compose.e2e.yml down --remove-orphans
+} finally {
+ Pop-Location
+}
+
diff --git a/e2e/scripts/e2e-down.sh b/e2e/scripts/e2e-down.sh
new file mode 100644
index 0000000..e8a7704
--- /dev/null
+++ b/e2e/scripts/e2e-down.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env sh
+set -eu
+
+cd "$(dirname "$0")/.."
+docker compose -f docker/docker-compose.e2e.yml down --remove-orphans
+
diff --git a/e2e/scripts/e2e-up.ps1 b/e2e/scripts/e2e-up.ps1
new file mode 100644
index 0000000..9a8c05e
--- /dev/null
+++ b/e2e/scripts/e2e-up.ps1
@@ -0,0 +1,10 @@
+Set-StrictMode -Version Latest
+$ErrorActionPreference = "Stop"
+
+Push-Location (Join-Path $PSScriptRoot "..")
+try {
+ docker compose -f docker/docker-compose.e2e.yml up -d mysql
+} finally {
+ Pop-Location
+}
+
diff --git a/e2e/scripts/e2e-up.sh b/e2e/scripts/e2e-up.sh
new file mode 100644
index 0000000..00b8282
--- /dev/null
+++ b/e2e/scripts/e2e-up.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env sh
+set -eu
+
+cd "$(dirname "$0")/.."
+docker compose -f docker/docker-compose.e2e.yml up -d mysql
+
diff --git a/e2e/scripts/reset-db.ts b/e2e/scripts/reset-db.ts
new file mode 100644
index 0000000..d3ceca4
--- /dev/null
+++ b/e2e/scripts/reset-db.ts
@@ -0,0 +1,50 @@
+import mysql from "mysql2/promise";
+import { env } from "../fixtures/env.js";
+import { sleep, waitForTcpPort } from "./wait-for.js";
+
+async function main() {
+ await waitForTcpPort(env.db.host, env.db.port, 120_000);
+
+ let connection: mysql.Connection | undefined;
+ let lastError: unknown;
+
+ for (let attempt = 1; attempt <= 30; attempt += 1) {
+ try {
+ connection = await mysql.createConnection({
+ host: env.db.host,
+ port: env.db.port,
+ user: env.db.user,
+ password: env.db.password,
+ multipleStatements: false
+ });
+ break;
+ } catch (error) {
+ lastError = error;
+ await sleep(2000);
+ }
+ }
+
+ if (!connection) {
+ throw new Error(
+ `failed to connect to MySQL: ${
+ lastError instanceof Error ? lastError.message : String(lastError)
+ }`
+ );
+ }
+
+ try {
+ await connection.query(`DROP DATABASE IF EXISTS \`${env.db.database}\``);
+ await connection.query(
+ `CREATE DATABASE \`${env.db.database}\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci`
+ );
+ console.log(`[e2e] reset database ${env.db.database}`);
+ } finally {
+ await connection.end();
+ }
+}
+
+main().catch((error: unknown) => {
+ console.error("[e2e] failed to reset database");
+ console.error(error instanceof Error ? error.message : String(error));
+ process.exit(1);
+});
diff --git a/e2e/scripts/run-target.test.ts b/e2e/scripts/run-target.test.ts
new file mode 100644
index 0000000..3418475
--- /dev/null
+++ b/e2e/scripts/run-target.test.ts
@@ -0,0 +1,47 @@
+import assert from "node:assert/strict";
+import { buildTargetRunOptions } from "./run-target.js";
+
+const defaultOptions = buildTargetRunOptions(["172.16.1.12:39443"], {});
+
+assert.equal(defaultOptions.frontendUrl, "https://172.16.1.12:39443");
+assert.equal(defaultOptions.backendUrl, "https://172.16.1.12:39443/api/v1");
+assert.deepEqual(defaultOptions.playwrightArgs, [
+ "playwright",
+ "test",
+ "--grep",
+ "@p0",
+ "--grep-invert",
+ "@local-only",
+ "--workers=1"
+]);
+assert.equal(defaultOptions.env.E2E_FRONTEND_URL, "https://172.16.1.12:39443");
+assert.equal(defaultOptions.env.E2E_BACKEND_URL, "https://172.16.1.12:39443/api/v1");
+assert.equal(defaultOptions.env.E2E_IGNORE_HTTPS_ERRORS, "true");
+assert.equal(defaultOptions.env.E2E_HEADLESS, "false");
+assert.equal(defaultOptions.env.E2E_DEBUG, "false");
+assert.equal(defaultOptions.env.E2E_SLOW_MO_MS, "0");
+assert.equal(defaultOptions.debug, false);
+assert.equal(defaultOptions.slowMoMs, 0);
+
+const httpOptions = buildTargetRunOptions(["http://172.16.1.12:39080/"]);
+assert.equal(httpOptions.frontendUrl, "http://172.16.1.12:39080");
+assert.equal(httpOptions.backendUrl, "http://172.16.1.12:39080/api/v1");
+
+const headlessOptions = buildTargetRunOptions(["172.16.1.12:39443", "--headless"], {});
+assert.equal(headlessOptions.env.E2E_HEADLESS, "true");
+
+const headedOptions = buildTargetRunOptions(["172.16.1.12:39443", "--headless=false"], {});
+assert.equal(headedOptions.env.E2E_HEADLESS, "false");
+
+const ciOptions = buildTargetRunOptions(["172.16.1.12:39443"], { CI: "true" });
+assert.equal(ciOptions.env.E2E_HEADLESS, "true");
+
+const debugOptions = buildTargetRunOptions(["172.16.1.12:39443", "--debug"], {});
+assert.equal(debugOptions.env.E2E_DEBUG, "true");
+assert.equal(debugOptions.env.E2E_SLOW_MO_MS, "500");
+assert.equal(debugOptions.env.E2E_HEADLESS, "false");
+assert.equal(debugOptions.debug, true);
+assert.equal(debugOptions.slowMoMs, 500);
+
+assert.throws(() => buildTargetRunOptions([]), /Usage/);
+assert.throws(() => buildTargetRunOptions(["https://a.example", "https://b.example"]), /only accepts one address/);
diff --git a/e2e/scripts/run-target.ts b/e2e/scripts/run-target.ts
new file mode 100644
index 0000000..beed1e1
--- /dev/null
+++ b/e2e/scripts/run-target.ts
@@ -0,0 +1,117 @@
+import { pathToFileURL } from "node:url";
+import { e2eRoot } from "../fixtures/env.js";
+import {
+ createLogger,
+ DEBUG_SLOW_MO_MS,
+ defaultHeadlessFromEnv,
+ parseDebugArgs,
+ parseHeadlessArgs,
+ runCommand,
+ suiteArgs,
+ writeFailureScreenshotSummary
+} from "./runner-utils.js";
+
+export interface TargetRunOptions {
+ frontendUrl: string;
+ backendUrl: string;
+ headless: boolean;
+ debug: boolean;
+ slowMoMs: number;
+ playwrightArgs: string[];
+ env: NodeJS.ProcessEnv;
+}
+
+function withoutTrailingSlash(value: string) {
+ return value.replace(/\/+$/, "");
+}
+
+function normalizeTargetAddress(rawAddress: string) {
+ const trimmed = rawAddress.trim();
+ if (!trimmed) {
+ throw new Error("Usage: npm run target -- ");
+ }
+
+ const withProtocol = /^[a-z][a-z\d+.-]*:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}`;
+ const url = new URL(withProtocol);
+ url.hash = "";
+ url.search = "";
+ return withoutTrailingSlash(url.toString());
+}
+
+export function buildTargetRunOptions(
+ args: string[] = process.argv.slice(2),
+ environment: NodeJS.ProcessEnv = process.env
+): TargetRunOptions {
+ const parsedHeadless = parseHeadlessArgs(args, defaultHeadlessFromEnv(environment));
+ const parsedDebug = parseDebugArgs(parsedHeadless.args, false);
+ const remainingArgs = parsedDebug.args;
+
+ if (remainingArgs.length === 0) {
+ throw new Error("Usage: npm run target -- ");
+ }
+ if (remainingArgs.length > 1) {
+ throw new Error("target runner only accepts one address plus optional --headless/--headed/--debug");
+ }
+
+ const frontendUrl = normalizeTargetAddress(remainingArgs[0]);
+ const backendUrl = `${frontendUrl}/api/v1`;
+ const slowMoMs = parsedDebug.debug ? DEBUG_SLOW_MO_MS : 0;
+
+ return {
+ frontendUrl,
+ backendUrl,
+ headless: parsedHeadless.headless,
+ debug: parsedDebug.debug,
+ slowMoMs,
+ playwrightArgs: ["playwright", "test", ...suiteArgs("p0"), "--grep-invert", "@local-only", "--workers=1"],
+ env: {
+ ...environment,
+ E2E_FRONTEND_URL: frontendUrl,
+ E2E_BACKEND_URL: backendUrl,
+ E2E_CI: "true",
+ E2E_HEADLESS: String(parsedHeadless.headless),
+ E2E_DEBUG: String(parsedDebug.debug),
+ E2E_SLOW_MO_MS: String(slowMoMs),
+ E2E_IGNORE_HTTPS_ERRORS: "true",
+ E2E_REMOTE_TARGET: "true"
+ }
+ };
+}
+
+async function main() {
+ const options = buildTargetRunOptions();
+ const logger = createLogger("target-run.log");
+
+ try {
+ logger.log(`target frontend: ${options.frontendUrl}`);
+ logger.log(`target backend: ${options.backendUrl}`);
+ logger.log(`target headless: ${options.headless}`);
+ logger.log(`target debug: ${options.debug} slowMoMs=${options.slowMoMs}`);
+ logger.log("running remote Playwright P0 suite");
+ try {
+ await runCommand(
+ "npx",
+ options.playwrightArgs,
+ {
+ cwd: e2eRoot,
+ label: "playwright",
+ env: options.env
+ },
+ logger.write
+ );
+ } catch (error) {
+ writeFailureScreenshotSummary(logger.write);
+ throw error;
+ }
+ logger.log("target e2e run completed successfully");
+ } finally {
+ await logger.close();
+ }
+}
+
+if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) {
+ main().catch((error: unknown) => {
+ console.error(error instanceof Error ? error.message : String(error));
+ process.exit(1);
+ });
+}
diff --git a/e2e/scripts/run-with-log.test.ts b/e2e/scripts/run-with-log.test.ts
new file mode 100644
index 0000000..2e9fbdc
--- /dev/null
+++ b/e2e/scripts/run-with-log.test.ts
@@ -0,0 +1,56 @@
+import assert from "node:assert/strict";
+import { parseArgs } from "./run-with-log.js";
+
+assert.deepEqual(parseArgs(["p0"], {}), {
+ suite: "p0",
+ ci: false,
+ headless: false,
+ debug: false,
+ slowMoMs: 0,
+ extraArgs: []
+});
+
+assert.deepEqual(parseArgs(["p1", "--headless", "--workers=1"], {}), {
+ suite: "p1",
+ ci: false,
+ headless: true,
+ debug: false,
+ slowMoMs: 0,
+ extraArgs: ["--workers=1"]
+});
+
+assert.deepEqual(parseArgs(["p2", "--headed", "--workers=1"], { CI: "true" }), {
+ suite: "p2",
+ ci: false,
+ headless: false,
+ debug: false,
+ slowMoMs: 0,
+ extraArgs: ["--workers=1"]
+});
+
+assert.deepEqual(parseArgs(["all", "--ci"], {}), {
+ suite: "all",
+ ci: true,
+ headless: true,
+ debug: false,
+ slowMoMs: 0,
+ extraArgs: []
+});
+
+assert.deepEqual(parseArgs(["p0", "--ci", "--headless=false"], {}), {
+ suite: "p0",
+ ci: true,
+ headless: false,
+ debug: false,
+ slowMoMs: 0,
+ extraArgs: []
+});
+
+assert.deepEqual(parseArgs(["p0", "--debug", "--workers=1"], {}), {
+ suite: "p0",
+ ci: false,
+ headless: false,
+ debug: true,
+ slowMoMs: 500,
+ extraArgs: ["--workers=1"]
+});
diff --git a/e2e/scripts/run-with-log.ts b/e2e/scripts/run-with-log.ts
new file mode 100644
index 0000000..e9f0359
--- /dev/null
+++ b/e2e/scripts/run-with-log.ts
@@ -0,0 +1,181 @@
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import type { ChildProcessWithoutNullStreams } from "node:child_process";
+import { env, e2eRoot, repoRoot } from "../fixtures/env.js";
+import { waitForHttpStatus, waitForTcpPort } from "./wait-for.js";
+import {
+ createLogger,
+ DEBUG_SLOW_MO_MS,
+ defaultHeadlessFromEnv,
+ parseDebugArgs,
+ parseHeadlessArgs,
+ runCommand,
+ startService,
+ stopService,
+ suiteArgs,
+ type Suite,
+ writeFailureScreenshotSummary
+} from "./runner-utils.js";
+
+const npmCmd = "npm";
+const npxCmd = "npx";
+const goCmd = "go";
+
+function childEnv(extra: NodeJS.ProcessEnv = {}) {
+ return {
+ ...process.env,
+ E2E_FRONTEND_URL: env.frontendUrl,
+ E2E_BACKEND_URL: env.backendUrl,
+ E2E_DB_HOST: env.db.host,
+ E2E_DB_PORT: String(env.db.port),
+ E2E_DB_USER: env.db.user,
+ E2E_DB_PASSWORD: env.db.password,
+ E2E_DB_NAME: env.db.database,
+ E2E_RUNTIME_AGENT_TOKEN: env.runtimeAgentToken,
+ E2E_WORKSPACE_ROOT: env.workspaceRoot,
+ E2E_OBJECT_STORAGE_FALLBACK: env.objectStorageFallback,
+ ...extra
+ };
+}
+
+export function parseArgs(
+ args: string[] = process.argv.slice(2),
+ environment: NodeJS.ProcessEnv = process.env
+): { suite: Suite; ci: boolean; headless: boolean; debug: boolean; slowMoMs: number; extraArgs: string[] } {
+ const [suiteArg = "p0", ...rest] = args;
+ if (!["p0", "p1", "p2", "all"].includes(suiteArg)) {
+ throw new Error(`unknown e2e suite: ${suiteArg}`);
+ }
+ const ci = rest.includes("--ci");
+ const filteredArgs = rest.filter((value) => value !== "--ci" && value !== "--");
+ const parsedHeadless = parseHeadlessArgs(
+ filteredArgs,
+ ci || defaultHeadlessFromEnv(environment)
+ );
+ const parsedDebug = parseDebugArgs(parsedHeadless.args, false);
+ return {
+ suite: suiteArg as Suite,
+ ci,
+ headless: parsedHeadless.headless,
+ debug: parsedDebug.debug,
+ slowMoMs: parsedDebug.debug ? DEBUG_SLOW_MO_MS : 0,
+ extraArgs: parsedDebug.args
+ };
+}
+
+function frontendHostAndPort() {
+ const url = new URL(env.frontendUrl);
+ return {
+ host: url.hostname,
+ port: url.port || (url.protocol === "https:" ? "443" : "80")
+ };
+}
+
+async function main() {
+ const { suite, ci, headless, debug, slowMoMs, extraArgs } = parseArgs();
+ const logger = createLogger();
+ let backend: ChildProcessWithoutNullStreams | undefined;
+ let frontend: ChildProcessWithoutNullStreams | undefined;
+
+ try {
+ logger.log(`starting e2e suite=${suite} ci=${ci} headless=${headless} debug=${debug} slowMoMs=${slowMoMs}`);
+ logger.log(`run log: ${logger.logPath}`);
+
+ logger.log("starting MySQL");
+ await runCommand(
+ "docker",
+ ["compose", "-f", "docker/docker-compose.e2e.yml", "up", "-d", "mysql"],
+ { cwd: e2eRoot, label: "docker" },
+ logger.write
+ );
+ await waitForTcpPort(env.db.host, env.db.port, 120_000);
+
+ logger.log("resetting e2e database");
+ await runCommand(npxCmd, ["tsx", "scripts/reset-db.ts"], { cwd: e2eRoot, label: "db" }, logger.write);
+
+ logger.log("starting backend");
+ backend = startService(
+ goCmd,
+ ["run", "./cmd/server"],
+ {
+ cwd: path.join(repoRoot, "backend"),
+ label: "backend",
+ env: childEnv({
+ SERVER_ADDRESS: ":9001",
+ DB_HOST: env.db.host,
+ DB_PORT: String(env.db.port),
+ DB_USER: env.db.user,
+ DB_PASSWORD: env.db.password,
+ DB_NAME: env.db.database,
+ JWT_SECRET: "clawmanager-e2e-secret",
+ RUNTIME_SCHEDULER_ENABLED: "false",
+ RUNTIME_AGENT_REPORT_TOKEN: env.runtimeAgentToken,
+ RUNTIME_WORKSPACE_ROOT: env.workspaceRoot,
+ OBJECT_STORAGE_LOCAL_FALLBACK: env.objectStorageFallback,
+ SKILL_SCANNER_ENABLED: "false"
+ })
+ },
+ logger.write
+ );
+ await waitForTcpPort("127.0.0.1", 9001, 120_000);
+
+ logger.log("starting frontend");
+ const frontendTarget = frontendHostAndPort();
+ frontend = startService(
+ npmCmd,
+ ["run", "dev", "--", "--host", frontendTarget.host, "--port", frontendTarget.port, "--strictPort"],
+ { cwd: path.join(repoRoot, "frontend"), label: "frontend", env: childEnv() },
+ logger.write
+ );
+ await waitForHttpStatus(env.frontendUrl, 200, 120_000);
+
+ logger.log("running Playwright");
+ const args = ["playwright", "test", ...suiteArgs(suite)];
+ if (ci && !extraArgs.some((value) => value.startsWith("--workers"))) {
+ args.push("--workers=1");
+ }
+ args.push(...extraArgs);
+ try {
+ await runCommand(
+ npxCmd,
+ args,
+ {
+ cwd: e2eRoot,
+ label: "playwright",
+ env: childEnv({
+ CI: ci ? "true" : process.env.CI,
+ E2E_HEADLESS: String(headless),
+ E2E_DEBUG: String(debug),
+ E2E_SLOW_MO_MS: String(slowMoMs)
+ })
+ },
+ logger.write
+ );
+ } catch (error) {
+ writeFailureScreenshotSummary(logger.write);
+ throw error;
+ }
+
+ logger.log("e2e run completed successfully");
+ } finally {
+ logger.log("cleaning up e2e services");
+ await stopService(frontend, "frontend");
+ await stopService(backend, "backend");
+ await runCommand(
+ "docker",
+ ["compose", "-f", "docker/docker-compose.e2e.yml", "down", "--remove-orphans"],
+ { cwd: e2eRoot, label: "docker" },
+ logger.write
+ ).catch((error: unknown) => {
+ logger.log(`docker cleanup failed: ${error instanceof Error ? error.message : String(error)}`);
+ });
+ await logger.close();
+ }
+}
+
+if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) {
+ main().catch((error: unknown) => {
+ console.error(error instanceof Error ? error.message : String(error));
+ process.exit(1);
+ });
+}
diff --git a/e2e/scripts/runner-utils.test.ts b/e2e/scripts/runner-utils.test.ts
new file mode 100644
index 0000000..68bd885
--- /dev/null
+++ b/e2e/scripts/runner-utils.test.ts
@@ -0,0 +1,78 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import assert from "node:assert/strict";
+import { findFailureScreenshots, parseDebugArgs, parseHeadlessArgs } from "./runner-utils.js";
+
+assert.deepEqual(parseHeadlessArgs([], false), {
+ headless: false,
+ args: []
+});
+
+assert.deepEqual(parseHeadlessArgs([], true), {
+ headless: true,
+ args: []
+});
+
+assert.deepEqual(parseHeadlessArgs(["--headless", "--workers=1"], false), {
+ headless: true,
+ args: ["--workers=1"]
+});
+
+assert.deepEqual(parseHeadlessArgs(["--headless=false", "--workers=1"], true), {
+ headless: false,
+ args: ["--workers=1"]
+});
+
+assert.deepEqual(parseHeadlessArgs(["--headless", "false", "--workers=1"], true), {
+ headless: false,
+ args: ["--workers=1"]
+});
+
+assert.deepEqual(parseHeadlessArgs(["--headed", "--workers=1"], true), {
+ headless: false,
+ args: ["--workers=1"]
+});
+
+assert.throws(() => parseHeadlessArgs(["--headless=maybe"], false), /headless/);
+
+assert.deepEqual(parseDebugArgs([], false), {
+ debug: false,
+ args: []
+});
+
+assert.deepEqual(parseDebugArgs(["--debug", "--workers=1"], false), {
+ debug: true,
+ args: ["--workers=1"]
+});
+
+assert.deepEqual(parseDebugArgs(["--debug=false", "--workers=1"], true), {
+ debug: false,
+ args: ["--workers=1"]
+});
+
+assert.deepEqual(parseDebugArgs(["--debug", "false", "--workers=1"], true), {
+ debug: false,
+ args: ["--workers=1"]
+});
+
+assert.deepEqual(parseDebugArgs(["--no-debug", "--workers=1"], true), {
+ debug: false,
+ args: ["--workers=1"]
+});
+
+assert.throws(() => parseDebugArgs(["--debug=maybe"], false), /debug/);
+
+const screenshotRoot = fs.mkdtempSync(path.join(os.tmpdir(), "clawmanager-e2e-screenshots-"));
+const nestedDir = path.join(screenshotRoot, "failed-test");
+fs.mkdirSync(nestedDir, { recursive: true });
+fs.writeFileSync(path.join(nestedDir, "test-failed-1.png"), "");
+fs.writeFileSync(path.join(nestedDir, "trace.zip"), "");
+fs.writeFileSync(path.join(screenshotRoot, "other.png"), "");
+
+assert.deepEqual(
+ findFailureScreenshots(screenshotRoot)
+ .map((filePath) => path.basename(filePath))
+ .sort(),
+ ["other.png", "test-failed-1.png"]
+);
diff --git a/e2e/scripts/runner-utils.ts b/e2e/scripts/runner-utils.ts
new file mode 100644
index 0000000..7239087
--- /dev/null
+++ b/e2e/scripts/runner-utils.ts
@@ -0,0 +1,278 @@
+import fs from "node:fs";
+import path from "node:path";
+import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
+import { reportsDir, testResultsDir } from "../fixtures/env.js";
+
+export type Suite = "p0" | "p1" | "p2" | "all";
+
+export const DEBUG_SLOW_MO_MS = 500;
+
+const isWindows = process.platform === "win32";
+const truthyValues = new Set(["1", "true", "yes", "on"]);
+const falsyValues = new Set(["0", "false", "no", "off"]);
+
+export function defaultHeadlessFromEnv(environment: NodeJS.ProcessEnv = process.env) {
+ return environment.CI === "true" || environment.E2E_CI === "true";
+}
+
+function parseBooleanFlag(flag: string, value: string) {
+ const normalized = value.trim().toLowerCase();
+ if (truthyValues.has(normalized)) {
+ return true;
+ }
+ if (falsyValues.has(normalized)) {
+ return false;
+ }
+ throw new Error(`${flag} must be true or false`);
+}
+
+export function parseHeadlessArgs(args: string[], defaultHeadless: boolean) {
+ let headless = defaultHeadless;
+ const remainingArgs: string[] = [];
+
+ for (let index = 0; index < args.length; index += 1) {
+ const arg = args[index];
+ if (arg === "--headless") {
+ const nextArg = args[index + 1];
+ const nextValue = nextArg?.toLowerCase();
+ if (nextValue && (truthyValues.has(nextValue) || falsyValues.has(nextValue))) {
+ headless = parseBooleanFlag("--headless", nextArg);
+ index += 1;
+ } else {
+ headless = true;
+ }
+ continue;
+ }
+ if (arg === "--headed" || arg === "--no-headless") {
+ headless = false;
+ continue;
+ }
+ if (arg.startsWith("--headless=")) {
+ headless = parseBooleanFlag("--headless", arg.slice("--headless=".length));
+ continue;
+ }
+ remainingArgs.push(arg);
+ }
+
+ return {
+ headless,
+ args: remainingArgs
+ };
+}
+
+export function parseDebugArgs(args: string[], defaultDebug: boolean) {
+ let debug = defaultDebug;
+ const remainingArgs: string[] = [];
+
+ for (let index = 0; index < args.length; index += 1) {
+ const arg = args[index];
+ if (arg === "--debug") {
+ const nextArg = args[index + 1];
+ const nextValue = nextArg?.toLowerCase();
+ if (nextValue && (truthyValues.has(nextValue) || falsyValues.has(nextValue))) {
+ debug = parseBooleanFlag("--debug", nextArg);
+ index += 1;
+ } else {
+ debug = true;
+ }
+ continue;
+ }
+ if (arg === "--no-debug") {
+ debug = false;
+ continue;
+ }
+ if (arg.startsWith("--debug=")) {
+ debug = parseBooleanFlag("--debug", arg.slice("--debug=".length));
+ continue;
+ }
+ remainingArgs.push(arg);
+ }
+
+ return {
+ debug,
+ args: remainingArgs
+ };
+}
+
+export function findFailureScreenshots(rootDir = testResultsDir) {
+ if (!fs.existsSync(rootDir)) {
+ return [];
+ }
+
+ const screenshots: string[] = [];
+ const visit = (dir: string) => {
+ for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+ const entryPath = path.join(dir, entry.name);
+ if (entry.isDirectory()) {
+ visit(entryPath);
+ } else if (entry.isFile() && entry.name.toLowerCase().endsWith(".png")) {
+ screenshots.push(entryPath);
+ }
+ }
+ };
+
+ visit(rootDir);
+ return screenshots.sort((a, b) => a.localeCompare(b));
+}
+
+export function writeFailureScreenshotSummary(write: (line: string) => void) {
+ const screenshots = findFailureScreenshots();
+ if (screenshots.length === 0) {
+ write(`[e2e] no failure screenshots found in ${testResultsDir}\n`);
+ return;
+ }
+
+ write(`[e2e] failure screenshot evidence:\n`);
+ for (const screenshot of screenshots) {
+ write(`[e2e] ${screenshot}\n`);
+ }
+}
+
+function timestamp() {
+ return new Date().toISOString();
+}
+
+export function createLogger(fileName = "run.log") {
+ fs.mkdirSync(reportsDir, { recursive: true });
+ const logPath = path.join(reportsDir, fileName);
+ const stream = fs.createWriteStream(logPath, { flags: "w" });
+
+ const write = (line: string) => {
+ stream.write(line);
+ process.stdout.write(line);
+ };
+
+ const log = (message: string) => {
+ write(`[${timestamp()}] ${message}\n`);
+ };
+
+ return {
+ logPath,
+ log,
+ write,
+ close: () =>
+ new Promise((resolve) => {
+ stream.end(resolve);
+ })
+ };
+}
+
+function pipeChild(
+ child: ChildProcessWithoutNullStreams,
+ label: string,
+ write: (line: string) => void
+) {
+ child.stdout.on("data", (chunk: Buffer) => {
+ write(chunk.toString().replace(/^/gm, `[${label}] `));
+ });
+ child.stderr.on("data", (chunk: Buffer) => {
+ write(chunk.toString().replace(/^/gm, `[${label}] `));
+ });
+}
+
+function quoteWindowsArg(value: string) {
+ if (/^[A-Za-z0-9_./:=@-]+$/.test(value)) {
+ return value;
+ }
+ return `"${value.replace(/"/g, '\\"')}"`;
+}
+
+function spawnPortable(
+ command: string,
+ args: string[],
+ options: { cwd: string; env?: NodeJS.ProcessEnv }
+): ChildProcessWithoutNullStreams {
+ if (!isWindows) {
+ return spawn(command, args, {
+ cwd: options.cwd,
+ env: options.env,
+ windowsHide: true
+ });
+ }
+
+ const commandLine = [command, ...args].map(quoteWindowsArg).join(" ");
+ return spawn("cmd.exe", ["/d", "/s", "/c", commandLine], {
+ cwd: options.cwd,
+ env: options.env,
+ windowsHide: true
+ });
+}
+
+export function runCommand(
+ command: string,
+ args: string[],
+ options: { cwd: string; env?: NodeJS.ProcessEnv; label: string },
+ write: (line: string) => void
+): Promise {
+ return new Promise((resolve, reject) => {
+ const child = spawnPortable(command, args, options);
+ pipeChild(child, options.label, write);
+ child.on("error", reject);
+ child.on("exit", (code) => {
+ if (code === 0) {
+ resolve();
+ } else {
+ reject(new Error(`${options.label} exited with code ${code}`));
+ }
+ });
+ });
+}
+
+export function startService(
+ command: string,
+ args: string[],
+ options: { cwd: string; env?: NodeJS.ProcessEnv; label: string },
+ write: (line: string) => void
+) {
+ const child = spawnPortable(command, args, options);
+ pipeChild(child, options.label, write);
+ return child;
+}
+
+async function killWindowsProcessTree(pid: number) {
+ await new Promise((resolve) => {
+ const child = spawn("taskkill.exe", ["/pid", String(pid), "/t", "/f"], {
+ windowsHide: true
+ });
+ child.on("exit", () => resolve());
+ child.on("error", () => resolve());
+ });
+}
+
+export async function stopService(
+ child: ChildProcessWithoutNullStreams | undefined,
+ label: string
+) {
+ if (!child || child.killed) {
+ return;
+ }
+
+ await new Promise((resolve) => {
+ child.once("exit", () => resolve());
+ if (isWindows && child.pid) {
+ void killWindowsProcessTree(child.pid).then(resolve);
+ } else {
+ child.kill();
+ }
+ setTimeout(() => {
+ if (!child.killed) {
+ child.kill("SIGKILL");
+ }
+ resolve();
+ }, 5000).unref();
+ });
+ console.log(`[e2e] stopped ${label}`);
+}
+
+export function suiteArgs(suite: Suite) {
+ switch (suite) {
+ case "p0":
+ return ["--grep", "@p0"];
+ case "p1":
+ return ["--grep", "@p1"];
+ case "p2":
+ return ["--grep", "@p2"];
+ case "all":
+ return [];
+ }
+}
diff --git a/e2e/scripts/wait-for.ts b/e2e/scripts/wait-for.ts
new file mode 100644
index 0000000..4a88b8d
--- /dev/null
+++ b/e2e/scripts/wait-for.ts
@@ -0,0 +1,70 @@
+import net from "node:net";
+
+export function sleep(ms: number) {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export async function waitForTcpPort(
+ host: string,
+ port: number,
+ timeoutMs = 60_000
+): Promise {
+ const deadline = Date.now() + timeoutMs;
+ let lastError: unknown;
+
+ while (Date.now() < deadline) {
+ try {
+ await new Promise((resolve, reject) => {
+ const socket = net.createConnection({ host, port });
+ socket.setTimeout(2000);
+ socket.once("connect", () => {
+ socket.end();
+ resolve();
+ });
+ socket.once("timeout", () => {
+ socket.destroy();
+ reject(new Error(`timeout waiting for ${host}:${port}`));
+ });
+ socket.once("error", reject);
+ });
+ return;
+ } catch (error) {
+ lastError = error;
+ await sleep(1000);
+ }
+ }
+
+ throw new Error(
+ `timed out waiting for ${host}:${port}: ${
+ lastError instanceof Error ? lastError.message : String(lastError)
+ }`
+ );
+}
+
+export async function waitForHttpStatus(
+ url: string,
+ expectedStatus = 200,
+ timeoutMs = 60_000
+): Promise {
+ const deadline = Date.now() + timeoutMs;
+ let lastStatus = 0;
+ let lastError = "";
+
+ while (Date.now() < deadline) {
+ try {
+ const response = await fetch(url);
+ lastStatus = response.status;
+ if (response.status === expectedStatus) {
+ return;
+ }
+ } catch (error) {
+ lastError = error instanceof Error ? error.message : String(error);
+ }
+ await sleep(1000);
+ }
+
+ throw new Error(
+ `timed out waiting for ${url}; last status=${lastStatus}; last error=${lastError}`
+ );
+}
+
diff --git a/e2e/setup/auth.setup.ts b/e2e/setup/auth.setup.ts
new file mode 100644
index 0000000..e9f57e1
--- /dev/null
+++ b/e2e/setup/auth.setup.ts
@@ -0,0 +1,28 @@
+import fs from "node:fs";
+import path from "node:path";
+import { e2eRoot } from "../fixtures/env.js";
+import { test, expect } from "../fixtures/test.js";
+import { users } from "../fixtures/users.js";
+import { LoginPage } from "../pages/LoginPage.js";
+
+test("@p0 auth setup writes admin storage state", async ({ page }) => {
+ const loginPage = new LoginPage(page);
+ await test.step("open login page", async () => {
+ await loginPage.goto();
+ await loginPage.expectVisible();
+ });
+
+ await test.step("login as seeded admin", async () => {
+ await loginPage.login(users.admin.username, users.admin.password);
+ await expect
+ .poll(() => page.evaluate(() => window.localStorage.getItem("access_token")))
+ .toBeTruthy();
+ });
+
+ await test.step("write storage state", async () => {
+ const statePath = path.join(e2eRoot, users.admin.storageState);
+ fs.mkdirSync(path.dirname(statePath), { recursive: true });
+ await page.context().storageState({ path: statePath });
+ });
+});
+
diff --git a/e2e/setup/db.setup.ts b/e2e/setup/db.setup.ts
new file mode 100644
index 0000000..088c4ce
--- /dev/null
+++ b/e2e/setup/db.setup.ts
@@ -0,0 +1,11 @@
+import { test, expect } from "../fixtures/test.js";
+import { defaultRuntimePod, registerRuntimePod } from "../fixtures/apiClient.js";
+
+test("@p0 @local-only db setup registers runtime pod", async ({ request }) => {
+ await test.step("register deterministic runtime pod", async () => {
+ const pod = await registerRuntimePod(request);
+ expect(pod.pod_name).toBe(defaultRuntimePod.pod_name);
+ expect(pod.runtime_type).toBe("openclaw");
+ expect(pod.state).toBe("ready");
+ });
+});
diff --git a/e2e/tests/admin/access-control.spec.ts b/e2e/tests/admin/access-control.spec.ts
new file mode 100644
index 0000000..65a3cbc
--- /dev/null
+++ b/e2e/tests/admin/access-control.spec.ts
@@ -0,0 +1,37 @@
+import { login, registerUser } from "../../fixtures/apiClient.js";
+import { env } from "../../fixtures/env.js";
+import { test, expect } from "../../fixtures/test.js";
+import { users } from "../../fixtures/users.js";
+
+test("@p0 unauthenticated admin route redirects to login", async ({ page }) => {
+ await page.goto("/admin/runtime-pods");
+
+ await expect(page).toHaveURL(/\/login$/);
+});
+
+test("@p0 @local-only non-admin user cannot open admin routes", async ({ page, request }) => {
+ await registerUser(request, users.user);
+ const tokens = await login(request, users.user);
+
+ await page.addInitScript(
+ ({ accessToken, refreshToken }) => {
+ window.localStorage.setItem("access_token", accessToken);
+ window.localStorage.setItem("refresh_token", refreshToken);
+ },
+ { accessToken: tokens.access_token, refreshToken: tokens.refresh_token }
+ );
+
+ await page.goto(`${env.frontendUrl}/admin/runtime-pods`);
+
+ await expect(page).toHaveURL(/\/dashboard$/);
+});
+
+test.describe("admin access", () => {
+ test.use({ storageState: users.admin.storageState });
+
+ test("@p0 admin can open runtime pods route", async ({ page }) => {
+ await page.goto("/admin/runtime-pods");
+
+ await expect(page.getByRole("heading", { name: /^runtime$/i })).toBeVisible();
+ });
+});
diff --git a/e2e/tests/admin/runtime-pods.spec.ts b/e2e/tests/admin/runtime-pods.spec.ts
new file mode 100644
index 0000000..d922ad1
--- /dev/null
+++ b/e2e/tests/admin/runtime-pods.spec.ts
@@ -0,0 +1,23 @@
+import { defaultRuntimePod } from "../../fixtures/apiClient.js";
+import { test } from "../../fixtures/test.js";
+import { users } from "../../fixtures/users.js";
+import { AdminRuntimePodsPage } from "../../pages/AdminRuntimePodsPage.js";
+
+test.use({ storageState: users.admin.storageState });
+
+test("@p0 @local-only admin runtime pods page renders no runtime pods for an empty filter", async ({ page }) => {
+ const runtimePodsPage = new AdminRuntimePodsPage(page);
+
+ await runtimePodsPage.goto();
+ await runtimePodsPage.selectRuntimeFilter("Hermes");
+ await runtimePodsPage.expectEmptyState();
+});
+
+test("@p0 @local-only admin runtime pods page renders seeded pod status and capacity", async ({ page }) => {
+ const runtimePodsPage = new AdminRuntimePodsPage(page);
+
+ await runtimePodsPage.goto();
+ await runtimePodsPage.selectRuntimeFilter("OpenClaw");
+ await runtimePodsPage.expectPodStatusAndCapacity(defaultRuntimePod);
+ await runtimePodsPage.expectPodMetricsVisible(defaultRuntimePod.pod_name);
+});
diff --git a/e2e/tests/api/auth.spec.ts b/e2e/tests/api/auth.spec.ts
new file mode 100644
index 0000000..88eb393
--- /dev/null
+++ b/e2e/tests/api/auth.spec.ts
@@ -0,0 +1,26 @@
+import { getMe, getMeRaw, login } from "../../fixtures/apiClient.js";
+import { test, expect } from "../../fixtures/test.js";
+import { users } from "../../fixtures/users.js";
+
+test("@p0 auth login returns tokens", async ({ request }) => {
+ const tokens = await login(request);
+
+ expect(tokens.access_token).toBeTruthy();
+ expect(tokens.refresh_token).toBeTruthy();
+ expect(tokens.expires_in).toBeGreaterThan(0);
+});
+
+test("@p0 auth me returns current user with valid token", async ({ request }) => {
+ const tokens = await login(request);
+ const user = await getMe(request, tokens.access_token);
+
+ expect(user.username).toBe(users.admin.username);
+ expect(user.role).toBe(users.admin.role);
+});
+
+test("@p0 auth me rejects unauthenticated requests", async ({ request }) => {
+ const response = await getMeRaw(request);
+
+ expect(response.status()).toBe(401);
+});
+
diff --git a/e2e/tests/api/runtime-agent.spec.ts b/e2e/tests/api/runtime-agent.spec.ts
new file mode 100644
index 0000000..147e39b
--- /dev/null
+++ b/e2e/tests/api/runtime-agent.spec.ts
@@ -0,0 +1,49 @@
+import {
+ defaultRuntimePod,
+ listRuntimePods,
+ login,
+ postRuntimePodRegisterRaw,
+ registerRuntimePod,
+ sendRuntimeHeartbeat
+} from "../../fixtures/apiClient.js";
+import { test, expect } from "../../fixtures/test.js";
+
+test("@p0 @local-only runtime-agent register persists an openclaw runtime pod", async ({ request }) => {
+ const pod = await registerRuntimePod(request, {
+ pod_name: "e2e-openclaw-runtime-api-register"
+ });
+ const tokens = await login(request);
+ const pods = await listRuntimePods(request, tokens.access_token);
+
+ expect(pod.runtime_type).toBe("openclaw");
+ expect(pod.capacity).toBe(defaultRuntimePod.capacity);
+ expect(pod.used_slots).toBe(defaultRuntimePod.used_slots);
+ expect(pod.draining).toBe(defaultRuntimePod.draining);
+ expect(pods.some((item) => item.pod_name === "e2e-openclaw-runtime-api-register")).toBe(true);
+});
+
+test("@p0 @local-only runtime-agent heartbeat updates pod health state", async ({ request }) => {
+ const pod = await registerRuntimePod(request, {
+ pod_name: "e2e-openclaw-runtime-api-heartbeat"
+ });
+
+ await sendRuntimeHeartbeat(request, { pod_id: pod.id }, { state: "draining", used_slots: 3, draining: true });
+
+ const tokens = await login(request);
+ const pods = await listRuntimePods(request, tokens.access_token);
+ const updated = pods.find((item) => item.id === pod.id);
+
+ expect(updated?.state).toBe("draining");
+ expect(updated?.used_slots).toBe(3);
+ expect(updated?.draining).toBe(true);
+});
+
+test("@p0 @local-only runtime-agent register rejects unsupported runtime type", async ({ request }) => {
+ const response = await postRuntimePodRegisterRaw(request, {
+ ...defaultRuntimePod,
+ runtime_type: "ubuntu",
+ pod_name: "e2e-unsupported-runtime-api"
+ });
+
+ expect(response.status()).toBe(400);
+});
diff --git a/e2e/tests/instances/lite-pro-modes.spec.ts b/e2e/tests/instances/lite-pro-modes.spec.ts
new file mode 100644
index 0000000..5cb1a55
--- /dev/null
+++ b/e2e/tests/instances/lite-pro-modes.spec.ts
@@ -0,0 +1,405 @@
+import { expect, test } from "../../fixtures/test.js";
+import type { Page } from "@playwright/test";
+import type { InstanceRecord } from "../../fixtures/apiClient.js";
+import { users } from "../../fixtures/users.js";
+
+test.use({ storageState: users.admin.storageState });
+
+type InstanceMode = "lite" | "pro";
+
+interface ApiEnvelope {
+ success: boolean;
+ data: T;
+ error?: string;
+}
+
+interface InstanceListResponse {
+ instances: InstanceRecord[];
+ total: number;
+ page: number;
+ limit: number;
+}
+
+interface PasswordExternalAccessResult {
+ access: {
+ id: number;
+ instance_id: number;
+ enabled: boolean;
+ auth_mode: string;
+ };
+ password: string;
+ share_url?: string;
+}
+
+function modeOf(instance: InstanceRecord): InstanceMode {
+ if (instance.instance_mode === "lite" || instance.instance_mode === "pro") {
+ return instance.instance_mode;
+ }
+ return instance.runtime_type === "gateway" ? "lite" : "pro";
+}
+
+function activeInstances(instances: InstanceRecord[]) {
+ return instances.filter((instance) => instance.status !== "deleting");
+}
+
+function escapeRegExp(value: string) {
+ return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+function firstMode(instances: InstanceRecord[], mode: InstanceMode) {
+ return activeInstances(instances).find((instance) => modeOf(instance) === mode);
+}
+
+function runningOpenClawLiteInstances(instances: InstanceRecord[]) {
+ return activeInstances(instances).filter(
+ (instance) =>
+ instance.type === "openclaw" &&
+ modeOf(instance) === "lite" &&
+ instance.status === "running"
+ );
+}
+
+async function apiFromPage(
+ page: Page,
+ path: string,
+ options: { method?: string; data?: unknown } = {}
+): Promise {
+ const result = await page.evaluate(
+ async ({ requestPath, method, data }) => {
+ const accessToken = window.localStorage.getItem("access_token");
+ const headers: Record = {};
+ if (accessToken) {
+ headers.Authorization = `Bearer ${accessToken}`;
+ }
+ if (data !== undefined) {
+ headers["Content-Type"] = "application/json";
+ }
+
+ const response = await fetch(requestPath, {
+ method,
+ headers,
+ body: data === undefined ? undefined : JSON.stringify(data)
+ });
+ const text = await response.text();
+ let payload: ApiEnvelope | null = null;
+ if (text) {
+ payload = JSON.parse(text) as ApiEnvelope;
+ }
+ return { ok: response.ok, status: response.status, payload, text };
+ },
+ {
+ requestPath: path,
+ method: options.method ?? "GET",
+ data: options.data
+ }
+ );
+
+ if (!result.ok || !result.payload?.success) {
+ throw new Error(result.payload?.error || result.text || `request failed with status ${result.status}`);
+ }
+ return result.payload.data as T;
+}
+
+async function adminInstances(page: Page) {
+ await page.goto("/dashboard");
+ const response = await apiFromPage(page, "/api/v1/admin/instances?page=1&limit=1000");
+ return response.instances;
+}
+
+async function userInstances(page: Page) {
+ await page.goto("/dashboard");
+ const response = await apiFromPage(page, "/api/v1/instances?page=1&limit=1000");
+ return response.instances;
+}
+
+async function primeOpenClawGatewayStorage(page: Page, instanceId: number) {
+ await page.goto("/dashboard");
+ await page.evaluate((staleInstanceId) => {
+ for (const key of Object.keys(window.localStorage)) {
+ if (/openclaw|clawmanager\.openclaw/i.test(key)) {
+ window.localStorage.removeItem(key);
+ }
+ }
+
+ const gatewayURL = new URL(`/api/v1/instances/${staleInstanceId}/proxy`, window.location.origin);
+ gatewayURL.protocol = gatewayURL.protocol === "https:" ? "wss:" : "ws:";
+ gatewayURL.search = "";
+ gatewayURL.hash = "";
+ gatewayURL.pathname = gatewayURL.pathname.replace(/\/+$/, "");
+
+ const settings = {
+ gatewayUrl: gatewayURL.toString(),
+ sessionKey: "main",
+ lastActiveSessionKey: "main",
+ theme: "claw",
+ themeMode: "system",
+ chatFocusMode: false,
+ chatShowThinking: true,
+ chatShowToolCalls: true,
+ splitRatio: 0.6,
+ navCollapsed: false,
+ navWidth: 220,
+ navGroupsCollapsed: {},
+ borderRadius: 50,
+ sessionsByGateway: {
+ [gatewayURL.toString()]: {
+ sessionKey: "main",
+ lastActiveSessionKey: "main"
+ }
+ }
+ };
+ const serialized = JSON.stringify(settings);
+ window.localStorage.setItem("openclaw.control.settings.v1", serialized);
+ window.localStorage.setItem(`openclaw.control.settings.v1:${gatewayURL.toString()}`, serialized);
+ window.localStorage.setItem("clawmanager.openclaw.instanceId", String(staleInstanceId));
+ window.localStorage.setItem("clawmanager.openclaw.gatewayUrl", gatewayURL.toString());
+ }, instanceId);
+}
+
+async function expectedGatewayURL(page: Page, instanceId: number) {
+ return page.evaluate((targetInstanceId) => {
+ const gatewayURL = new URL(`/api/v1/instances/${targetInstanceId}/proxy`, window.location.origin);
+ gatewayURL.protocol = gatewayURL.protocol === "https:" ? "wss:" : "ws:";
+ gatewayURL.search = "";
+ gatewayURL.hash = "";
+ gatewayURL.pathname = gatewayURL.pathname.replace(/\/+$/, "");
+ return gatewayURL.toString();
+ }, instanceId);
+}
+
+async function reachConfigurationStep(page: Page, mode: InstanceMode) {
+ await page.goto("/instances/new");
+ await expect(page.getByRole("heading", { name: /create instance/i })).toBeVisible();
+
+ await page.locator("main input#name").nth(1).fill(`e2e-${mode}-mode-check`);
+ await page.getByRole("button", { name: new RegExp(`^${mode === "lite" ? "Lite" : "Pro"}`, "i") }).click();
+ const basicNextButton = page.getByRole("button", { name: /^Next$/i });
+ await expect(basicNextButton).toBeEnabled();
+ await basicNextButton.click();
+
+ await expect(page.getByRole("heading", { name: /select instance type/i })).toBeVisible();
+ const nextButton = page.getByRole("button", { name: /^Next$/i });
+ await expect(nextButton).toBeEnabled();
+ await nextButton.click();
+
+ await expect(page.getByRole("heading", { name: /summary/i })).toBeVisible();
+}
+
+async function visiblePageText(page: Page) {
+ return page.evaluate(() => document.body.innerText);
+}
+
+test.describe("lite/pro instance modes", () => {
+ test("@p0 create wizard exposes the lite/pro mode contract", async ({ page }) => {
+ await test.step("lite derives gateway runtime and skips dedicated resources", async () => {
+ await reachConfigurationStep(page, "lite");
+ const summary = page.locator(".app-panel").filter({
+ has: page.getByRole("heading", { name: /summary/i })
+ });
+
+ await expect(summary.getByText(/^Lite$/)).toBeVisible();
+ await expect(summary.getByText(/^Gateway$/)).toBeVisible();
+ await expect(page.getByRole("heading", { name: /quick configuration/i })).toHaveCount(0);
+ await expect(page.getByRole("button", { name: /enable gpu/i })).toHaveCount(0);
+ await expect(summary.getByText(/^CPU$/)).toHaveCount(0);
+ await expect(summary.getByText(/^Memory$/)).toHaveCount(0);
+ await expect(summary.getByText(/^Storage$/)).toHaveCount(0);
+ await expect(summary.getByText(/^GPU$/)).toHaveCount(0);
+ });
+
+ await test.step("pro derives desktop runtime and shows dedicated resources", async () => {
+ await reachConfigurationStep(page, "pro");
+ const summary = page.locator(".app-panel").filter({
+ has: page.getByRole("heading", { name: /summary/i })
+ });
+
+ await expect(summary.getByText(/^Pro$/)).toBeVisible();
+ await expect(summary.getByText(/^Desktop$/)).toBeVisible();
+ await expect(page.getByRole("heading", { name: /quick configuration/i })).toBeVisible();
+ await expect(page.getByRole("button", { name: /enable gpu/i })).toBeVisible();
+ await expect(summary.getByText(/^CPU$/)).toBeVisible();
+ await expect(summary.getByText(/^Memory$/)).toBeVisible();
+ await expect(summary.getByText(/^Storage$/)).toBeVisible();
+ await expect(summary.getByText(/^GPU$/)).toBeVisible();
+ });
+ });
+
+ test("@p0 lite detail keeps gateway workspace layout", async ({ page }) => {
+ const instances = await adminInstances(page);
+ const lite = firstMode(instances, "lite");
+ if (!lite) {
+ test.skip(true, "No lite/gateway instance exists in this target environment.");
+ return;
+ }
+
+ await page.goto(`/instances/${lite.id}`);
+ await expect(page.getByRole("heading", { name: lite.name }).first()).toBeVisible();
+ await expect.poll(() => visiblePageText(page)).toContain("Mode Lite");
+ await expect.poll(() => visiblePageText(page)).toContain("Runtime gateway");
+ await page.getByRole("button", { name: /share link/i }).click();
+ await expect(page.locator('[data-panel="share-link-popover"]').last()).toBeVisible();
+ await expect(page.getByRole("button", { name: /create password/i })).toBeVisible();
+
+ await expect(page.getByRole("heading", { name: /runtime overview/i })).toHaveCount(0);
+ await expect(page.getByRole("heading", { name: /runtime events/i })).toHaveCount(0);
+ await expect(page.getByRole("heading", { name: /instance skills/i })).toHaveCount(0);
+ });
+
+ test("@p0 pro detail keeps desktop management layout", async ({ page }) => {
+ const instances = await adminInstances(page);
+ const pro = firstMode(instances, "pro");
+ if (!pro) {
+ test.skip(true, "No pro/desktop instance exists in this target environment.");
+ return;
+ }
+
+ await page.goto(`/instances/${pro.id}`);
+ await expect(page.getByRole("heading", { name: pro.name }).first()).toBeVisible();
+ await expect.poll(() => visiblePageText(page)).toContain("Mode Pro");
+ await expect.poll(() => visiblePageText(page)).toContain("Runtime desktop");
+ await page.getByRole("button", { name: /share link/i }).click();
+ await expect(page.locator('[data-panel="share-link-popover"]').last()).toBeVisible();
+ await expect(page.getByRole("button", { name: /create password/i })).toBeVisible();
+
+ await expect(page.getByRole("heading", { name: /runtime overview/i })).toBeVisible();
+ await expect(page.getByRole("heading", { name: /runtime events/i })).toBeVisible();
+ await expect(page.getByRole("heading", { name: /instance skills/i })).toBeVisible();
+ const proDetailText = await visiblePageText(page);
+ expect(proDetailText.indexOf("Instance Skills")).toBeLessThan(proDetailText.indexOf("Runtime Overview"));
+ expect(proDetailText).not.toContain("Resource Monitor");
+ expect(proDetailText).not.toContain("Runtime Status");
+
+ const workspace = await apiFromPage<{ entries: Array<{ path: string }> }>(
+ page,
+ `/api/v1/instances/${pro.id}/workspace/files?path=`
+ );
+ expect(workspace.entries.length).toBeGreaterThan(0);
+ expect(workspace.entries.some((entry) => [".config", ".openclaw", "Desktop"].includes(entry.path))).toBeTruthy();
+ });
+
+ test("@p0 admin instances page surfaces mode and backend management controls", async ({ page }) => {
+ const instances = await adminInstances(page);
+ const lite = firstMode(instances, "lite");
+ const pro = firstMode(instances, "pro");
+
+ await page.goto("/admin/instances");
+ await expect(page.getByRole("heading", { name: /global instances/i })).toBeVisible();
+ await expect.poll(() => visiblePageText(page)).toContain("Gateway runtime");
+ await expect.poll(() => visiblePageText(page)).toContain("Desktop deployment");
+
+ const filters = page.locator("main select");
+ const modeFilter = filters.nth(6);
+ const runtimeFilter = filters.nth(7);
+ await expect(modeFilter).toContainText("Lite");
+ await expect(modeFilter).toContainText("Pro");
+ await expect(runtimeFilter).toContainText("Gateway");
+ await expect(runtimeFilter).toContainText("Desktop");
+
+ if (lite) {
+ await modeFilter.selectOption("lite");
+ await expect.poll(() => visiblePageText(page)).toContain(lite.name);
+ await expect.poll(() => visiblePageText(page)).toContain("Gateway binding");
+ await expect.poll(() => visiblePageText(page)).toContain("Gateway");
+ }
+
+ if (pro) {
+ await modeFilter.selectOption("pro");
+ await expect.poll(() => visiblePageText(page)).toContain(pro.name);
+ await expect.poll(() => visiblePageText(page)).toContain("Deployment");
+ await expect.poll(() => visiblePageText(page)).toContain("Desktop");
+ }
+ });
+
+ test("@p0 portal refreshes OpenClaw Lite gateway settings before embedding", async ({ page }) => {
+ const instances = await userInstances(page);
+ const liteInstances = runningOpenClawLiteInstances(instances);
+ if (liteInstances.length < 2) {
+ test.skip(true, "Need at least two running OpenClaw Lite instances for stale gateway coverage.");
+ return;
+ }
+
+ const [target, stale] = liteInstances;
+ await primeOpenClawGatewayStorage(page, stale.id);
+ await page.goto("/portal");
+ await page.locator("button").filter({ hasText: target.name }).first().dispatchEvent("click");
+
+ const gatewayURL = await expectedGatewayURL(page, target.id);
+ await expect
+ .poll(() => page.evaluate(() => window.localStorage.getItem("clawmanager.openclaw.instanceId")))
+ .toBe(String(target.id));
+ await expect
+ .poll(() => page.evaluate(() => window.localStorage.getItem("clawmanager.openclaw.gatewayUrl")))
+ .toBe(gatewayURL);
+
+ const settings = await page.evaluate(() => window.localStorage.getItem("openclaw.control.settings.v1"));
+ expect(settings).toBeTruthy();
+ expect(JSON.parse(settings!).gatewayUrl).toBe(gatewayURL);
+ });
+
+ test("@p1 password share link opens a password gate", async ({ page }) => {
+ const instances = await adminInstances(page);
+ const target = activeInstances(instances).find((instance) => instance.runtime_type !== "shell");
+ if (!target) {
+ test.skip(true, "No non-shell instance exists for share link coverage.");
+ return;
+ }
+
+ let passwordCreated = false;
+ try {
+ const result = await apiFromPage(
+ page,
+ `/api/v1/instances/${target.id}/external-access/password`,
+ {
+ method: "POST",
+ data: {
+ expires_mode: "preset",
+ expires_preset: "1h"
+ }
+ }
+ );
+ passwordCreated = true;
+
+ expect(result.password).toMatch(/^pwd_/);
+ expect(result.share_url).toBeTruthy();
+ expect(result.share_url).toMatch(/^\/s\/[^/?#]+\/$/);
+ expect(result.share_url).not.toContain("token=");
+ expect(result.share_url).not.toContain("/api/v1/");
+ expect(result.share_url).not.toContain("/proxy");
+
+ await page.goto(`/instances/${target.id}`);
+ await page.reload({ waitUntil: "domcontentloaded" });
+ await expect(page.locator('button[data-share-auth="password"]', { hasText: /share link/i }).last()).toBeVisible();
+ await page.getByRole("button", { name: /share link/i }).click();
+ const sharePanel = page.locator('[data-panel="share-link-popover"]').last();
+ const shareHeader = sharePanel.locator(".mb-3").first();
+ await expect(shareHeader).not.toContainText("Password");
+ await expect(shareHeader).not.toContainText(result.password.slice(0, 12));
+ await expect(sharePanel.getByLabel(/share link url/i)).toHaveValue(
+ new RegExp(`${escapeRegExp(result.share_url!)}$`)
+ );
+ await expect(sharePanel).not.toContainText("Password is shown only");
+ await expect(sharePanel).not.toContainText("created before link recovery");
+ await expect(sharePanel.getByRole("button", { name: /copy share link/i })).toBeVisible();
+ await expect(sharePanel.getByRole("button", { name: /copy password/i })).toBeVisible();
+
+ const passwordInput = sharePanel.getByLabel(/share link password/i);
+ await expect(passwordInput).toHaveValue(result.password);
+ await expect(passwordInput).toHaveAttribute("type", "password");
+ await sharePanel.getByRole("button", { name: /show password/i }).click();
+ await expect(passwordInput).toHaveAttribute("type", "text");
+ await expect(sharePanel.getByRole("button", { name: /hide password/i })).toBeVisible();
+
+ await page.goto(result.share_url!);
+ await expect(page.getByRole("heading", { name: /share link password/i })).toBeVisible();
+ await expect(page.locator('input[name="password"]')).toBeVisible();
+ await expect(page.getByRole("button", { name: /open|continue/i })).toBeVisible();
+ } finally {
+ if (passwordCreated) {
+ await page.goto("/dashboard");
+ await apiFromPage(page, `/api/v1/instances/${target.id}/external-access`, {
+ method: "DELETE"
+ });
+ }
+ }
+ });
+});
diff --git a/e2e/tests/smoke/login.spec.ts b/e2e/tests/smoke/login.spec.ts
new file mode 100644
index 0000000..a624f1f
--- /dev/null
+++ b/e2e/tests/smoke/login.spec.ts
@@ -0,0 +1,34 @@
+import { LoginPage } from "../../pages/LoginPage.js";
+import { test, expect } from "../../fixtures/test.js";
+import { users } from "../../fixtures/users.js";
+
+test.use({ storageState: undefined });
+
+test("@p0 login page renders", async ({ page }) => {
+ const loginPage = new LoginPage(page);
+
+ await loginPage.goto();
+ await loginPage.expectVisible();
+});
+
+test("@p0 admin can log in", async ({ page }) => {
+ const loginPage = new LoginPage(page);
+
+ await loginPage.goto();
+ await loginPage.login(users.admin.username, users.admin.password);
+
+ await expect
+ .poll(() => page.evaluate(() => window.localStorage.getItem("access_token")))
+ .toBeTruthy();
+});
+
+test("@p0 failed login shows an error", async ({ page }) => {
+ const loginPage = new LoginPage(page);
+
+ await loginPage.goto();
+ await loginPage.login(users.admin.username, "wrong-password");
+
+ await expect(page.getByText(/invalid username or password|login failed/i)).toBeVisible();
+ await expect(page).toHaveURL(/\/login$/);
+});
+
diff --git a/e2e/tests/smoke/navigation.spec.ts b/e2e/tests/smoke/navigation.spec.ts
new file mode 100644
index 0000000..637607b
--- /dev/null
+++ b/e2e/tests/smoke/navigation.spec.ts
@@ -0,0 +1,48 @@
+import { test, expect } from "../../fixtures/test.js";
+import { users } from "../../fixtures/users.js";
+
+test.describe("authenticated admin navigation", () => {
+ test.use({ storageState: users.admin.storageState });
+
+ test("@p0 authenticated admin reaches workspace dashboard", async ({ page }) => {
+ await page.goto("/dashboard");
+
+ await expect(page.getByRole("heading", { name: /workspace/i })).toBeVisible();
+ });
+
+ test("@p0 authenticated admin reaches admin console", async ({ page }) => {
+ await page.goto("/admin");
+
+ await expect(page.getByRole("heading", { name: /console/i })).toBeVisible();
+ });
+
+ test("@p0 authenticated admin can reach instances, settings, and admin entry routes", async ({ page }) => {
+ await page.goto("/instances");
+ await expect(page.getByRole("heading", { name: /my instances/i })).toBeVisible();
+
+ await page.goto("/settings");
+ await expect(page.getByRole("heading", { name: /settings/i })).toBeVisible();
+ await expect(page.getByRole("heading", { name: /change password/i })).toBeVisible();
+
+ await page.goto("/admin");
+ await expect(page.getByRole("link", { name: /^runtime$/i }).first()).toBeVisible();
+ await expect(page.getByRole("link", { name: /^settings$/i }).first()).toBeVisible();
+
+ await page.goto("/admin/runtime-pods");
+ await expect(page.getByRole("heading", { name: /^runtime$/i })).toBeVisible();
+
+ await page.goto("/admin/settings");
+ await expect(page.getByRole("heading", { name: /system settings/i })).toBeVisible();
+ });
+});
+
+test.describe("unauthenticated navigation", () => {
+ test.use({ storageState: undefined });
+
+ test("@p0 unauthenticated users are redirected to login", async ({ page }) => {
+ await page.goto("/instances");
+
+ await expect(page).toHaveURL(/\/login$/);
+ await expect(page.getByRole("heading", { name: /sign in to clawmanager/i })).toBeVisible();
+ });
+});
diff --git a/e2e/tests/teams/team-lite-mode.spec.ts b/e2e/tests/teams/team-lite-mode.spec.ts
new file mode 100644
index 0000000..772a26d
--- /dev/null
+++ b/e2e/tests/teams/team-lite-mode.spec.ts
@@ -0,0 +1,79 @@
+import { expect, test } from "../../fixtures/test.js";
+import {
+ createTeam,
+ deleteTeam,
+ getTeam,
+ listInstances,
+ login
+} from "../../fixtures/apiClient.js";
+import { users } from "../../fixtures/users.js";
+
+test.describe("Team lite member mode", () => {
+ test("@p0 creates Team members as Lite gateway instances", async ({ request }) => {
+ const tokens = await login(request, users.admin);
+ const teamName = `e2e-lite-team-${Date.now().toString(36)}`;
+ let teamId: number | undefined;
+
+ try {
+ const created = await createTeam(request, tokens.access_token, {
+ name: teamName,
+ description: "E2E coverage for Team Lite member creation",
+ communication_mode: "leader_mediated",
+ shared_storage_gb: 1,
+ members: [
+ {
+ member_id: "leader",
+ name: "lite-leader",
+ role: "leader",
+ mode: "lite",
+ instance_mode: "lite",
+ runtime_type: "openclaw",
+ cpu_cores: 0.1,
+ memory_gb: 1,
+ disk_gb: 10,
+ is_leader: true
+ },
+ {
+ member_id: "worker",
+ name: "lite-worker",
+ role: "developer",
+ mode: "lite",
+ instance_mode: "lite",
+ runtime_type: "openclaw",
+ cpu_cores: 0.1,
+ memory_gb: 1,
+ disk_gb: 10
+ }
+ ]
+ });
+ teamId = created.team.id;
+
+ expect(created.team.name).toBe(teamName);
+ expect(created.leader_member_id).toBe("leader");
+ expect(created.members).toHaveLength(2);
+ expect(created.members.every((member) => member.instance_mode === "lite")).toBeTruthy();
+ expect(created.members.every((member) => member.runtime_type === "openclaw")).toBeTruthy();
+ expect(created.members.every((member) => typeof member.instance_id === "number")).toBeTruthy();
+
+ const details = await getTeam(request, tokens.access_token, teamId);
+ expect(details.team.status).toBe("running");
+ expect(details.members.map((member) => member.member_key).sort()).toEqual(["leader", "worker"]);
+ expect(details.members.every((member) => member.instance_mode === "lite")).toBeTruthy();
+
+ const instanceIds = new Set(
+ details.members
+ .map((member) => member.instance_id)
+ .filter((id): id is number => typeof id === "number")
+ );
+ const instances = await listInstances(request, tokens.access_token, { admin: true, limit: 1000 });
+ const memberInstances = instances.instances.filter((instance) => instanceIds.has(instance.id));
+ expect(memberInstances).toHaveLength(2);
+ expect(memberInstances.every((instance) => instance.instance_mode === "lite")).toBeTruthy();
+ expect(memberInstances.every((instance) => instance.runtime_type === "gateway")).toBeTruthy();
+ } finally {
+ if (teamId !== undefined) {
+ await deleteTeam(request, tokens.access_token, teamId);
+ }
+ }
+ });
+});
diff --git a/e2e/tsconfig.json b/e2e/tsconfig.json
new file mode 100644
index 0000000..4236e38
--- /dev/null
+++ b/e2e/tsconfig.json
@@ -0,0 +1,15 @@
+{
+ "compilerOptions": {
+ "target": "ES2022",
+ "module": "NodeNext",
+ "moduleResolution": "NodeNext",
+ "strict": true,
+ "types": ["node", "@playwright/test"],
+ "esModuleInterop": true,
+ "forceConsistentCasingInFileNames": true,
+ "skipLibCheck": true,
+ "resolveJsonModule": true,
+ "noEmit": true
+ },
+ "include": ["**/*.ts"]
+}
diff --git a/frontend/package-lock.json b/frontend/package-lock.json
index f7d6fe7..2eb1659 100644
--- a/frontend/package-lock.json
+++ b/frontend/package-lock.json
@@ -13,6 +13,7 @@
"@xterm/addon-unicode11": "^0.9.0",
"@xterm/xterm": "^6.0.0",
"axios": "^1.13.6",
+ "lucide-react": "^1.17.0",
"react": "^19.2.4",
"react-dom": "^19.2.4",
"react-router-dom": "^7.13.1",
@@ -3024,6 +3025,15 @@
"yallist": "^3.0.2"
}
},
+ "node_modules/lucide-react": {
+ "version": "1.17.0",
+ "resolved": "https://registry.npmjs.org/lucide-react/-/lucide-react-1.17.0.tgz",
+ "integrity": "sha512-9FA9evdox/JQL5PT57fdA1x/yg8T7knJ98+zjTL3UfKza6pflQUUh3XtaQIHKvnsJw1lmsEyHVlt5jchYxOQ5w==",
+ "license": "ISC",
+ "peerDependencies": {
+ "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0"
+ }
+ },
"node_modules/math-intrinsics": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
diff --git a/frontend/package.json b/frontend/package.json
index c90b4ae..1ad39df 100644
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -15,6 +15,7 @@
"@xterm/addon-unicode11": "^0.9.0",
"@xterm/xterm": "^6.0.0",
"axios": "^1.13.6",
+ "lucide-react": "^1.17.0",
"react": "^19.2.4",
"react-dom": "^19.2.4",
"react-router-dom": "^7.13.1",
diff --git a/frontend/public/lobster_logo.png b/frontend/public/lobster_logo.png
new file mode 100644
index 0000000..7592055
Binary files /dev/null and b/frontend/public/lobster_logo.png differ
diff --git a/frontend/scripts/create-instance-page-mode.test.mjs b/frontend/scripts/create-instance-page-mode.test.mjs
new file mode 100644
index 0000000..cea99eb
--- /dev/null
+++ b/frontend/scripts/create-instance-page-mode.test.mjs
@@ -0,0 +1,74 @@
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import path from "node:path";
+
+const scriptDir = path.dirname(fileURLToPath(import.meta.url));
+const sourcePath = path.resolve(
+ scriptDir,
+ "../src/pages/instances/CreateInstancePage.tsx",
+);
+const source = readFileSync(sourcePath, "utf8");
+
+function sectionBetween(startMarker, endMarker) {
+ const start = source.indexOf(startMarker);
+ const end = source.indexOf(endMarker);
+ if (start === -1 || end === -1 || end <= start) {
+ throw new Error(`Unable to locate section: ${startMarker}`);
+ }
+ return source.slice(start, end);
+}
+
+function assert(condition, message) {
+ if (!condition) {
+ throw new Error(message);
+ }
+}
+
+const stepOne = sectionBetween(
+ "{/* Step 1: Basic Information */}",
+ "{/* Step 2: Select Type */}",
+);
+const stepTwo = sectionBetween(
+ "{/* Step 2: Select Type */}",
+ "{/* Step 3: Configuration */}",
+);
+const stepThree = source.slice(source.indexOf("{/* Step 3: Configuration */}"));
+
+assert(
+ /id:\s*"lite"/.test(source) && /id:\s*"pro"/.test(source),
+ "Create instance page must define Lite and Pro mode options.",
+);
+assert(
+ stepOne.includes("renderInstanceModeSelector()"),
+ "Lite/Pro selector must be visible in step 1.",
+);
+assert(
+ !stepTwo.includes("renderInstanceModeSelector()"),
+ "Lite/Pro selector should not be hidden behind step 2.",
+);
+assert(
+ source.includes('const usesDedicatedResources = selectedMode === "pro";'),
+ "Create page must distinguish Pro-only dedicated resource controls.",
+);
+assert(
+ source.includes('const showRuntimeImageSelector = selectedMode === "pro";'),
+ "Runtime image selector visibility must be controlled by Pro mode only.",
+);
+assert(
+ stepTwo.includes("{showRuntimeImageSelector && ("),
+ "Step 2 runtime image selector must be hidden for Lite mode.",
+);
+assert(
+ stepThree.includes("{usesDedicatedResources && ("),
+ "Step 3 resource controls must be hidden for Lite mode.",
+);
+assert(
+ stepThree.includes("{showRuntimeImageSelector && ("),
+ "Step 3 summary must hide runtime image details for Lite mode.",
+);
+assert(
+ source.includes("...(usesDedicatedResources"),
+ "Quota validation must include CPU/memory/storage/GPU only for Pro mode.",
+);
+
+console.log("Create instance mode selector placement is valid.");
diff --git a/frontend/scripts/instance-detail-pro-layout.test.mjs b/frontend/scripts/instance-detail-pro-layout.test.mjs
new file mode 100644
index 0000000..ef9e5f3
--- /dev/null
+++ b/frontend/scripts/instance-detail-pro-layout.test.mjs
@@ -0,0 +1,111 @@
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import path from "node:path";
+
+const scriptDir = path.dirname(fileURLToPath(import.meta.url));
+const detailSource = readFileSync(
+ path.resolve(scriptDir, "../src/pages/instances/InstanceDetailPage.tsx"),
+ "utf8",
+);
+const frameSource = readFileSync(
+ path.resolve(scriptDir, "../src/components/InstanceServiceFrame.tsx"),
+ "utf8",
+);
+
+function assert(condition, message) {
+ if (!condition) {
+ throw new Error(message);
+ }
+}
+
+assert(
+ detailSource.includes("isDedicatedInstance") &&
+ detailSource.includes('instance.instance_mode === "pro"') &&
+ detailSource.includes('instance.runtime_type !== "gateway"'),
+ "Instance detail page must branch pro/desktop instances away from the lite workspace layout.",
+);
+
+for (const label of ["Runtime Overview", "Runtime Events", "Instance Skills"]) {
+ assert(detailSource.includes(label), `Pro instance detail section missing: ${label}`);
+}
+
+for (const api of ["getRuntimeDetails", "listInstanceSkills", "attachSkillToInstance", "removeSkillFromInstance"]) {
+ assert(detailSource.includes(api), `Pro instance detail page must use ${api}.`);
+}
+
+assert(
+ detailSource.includes("renderLiteWorkspace") && detailSource.includes("renderProWorkspace"),
+ "Instance detail page must keep separate lite and pro render paths.",
+);
+
+function sliceBetween(source, startMarker, endMarker) {
+ const start = source.indexOf(startMarker);
+ assert(start >= 0, `Missing marker: ${startMarker}`);
+ const end = source.indexOf(endMarker, start);
+ assert(end > start, `Missing end marker after: ${startMarker}`);
+ return source.slice(start, end);
+}
+
+const proRender = sliceBetween(
+ detailSource,
+ "const renderProWorkspace = () => (",
+ "\n return (",
+);
+
+assert(
+ proRender.includes("shareLinkControl"),
+ "Pro instance detail must place Share Link with the primary instance action buttons.",
+);
+
+assert(
+ !detailSource.includes("externalAccessSection") && !detailSource.includes("externalAccessCompact"),
+ "Instance detail must not render the old full-width or inline external access controls.",
+);
+
+assert(
+ detailSource.includes('data-panel="share-link-popover"') &&
+ detailSource.includes("Share link expiration") &&
+ detailSource.includes("Create Password"),
+ "Share link expiration and password controls must live inside the Share Link popover.",
+);
+
+assert(
+ proRender.includes('data-section="runtime-overview"') &&
+ proRender.indexOf("Instance Skills") < proRender.indexOf("Runtime Overview"),
+ "Instance Skills must render before the compact Runtime Overview card.",
+);
+
+assert(
+ !proRender.includes("Resource Monitor") && !proRender.includes("Runtime Status"),
+ "Runtime Overview must use a compact summary instead of large nested Resource Monitor and Runtime Status sections.",
+);
+
+assert(
+ proRender.includes('initialPath="/config"'),
+ "Pro instance workspace must open at /config.",
+);
+
+assert(
+ proRender.includes("pro-desktop-workspace"),
+ "Pro instance workspace must be placed in the desktop workspace area.",
+);
+
+assert(
+ !proRender.includes("h-[560px]") &&
+ (proRender.includes("aspect-video") || proRender.includes("aspect-[16/9]")),
+ "Pro desktop frame must use a stable 16:9 desktop ratio instead of the old fixed height.",
+);
+
+assert(
+ proRender.indexOf("Runtime Events") > proRender.indexOf("Runtime Overview"),
+ "Runtime Events must be lower priority and render after Instance Skills.",
+);
+
+assert(
+ frameSource.includes("requestFullscreen") &&
+ frameSource.includes("Maximize2") &&
+ frameSource.includes("Minimize2"),
+ "Instance service frame must expose a fullscreen control.",
+);
+
+console.log("Instance detail pro layout contract is valid.");
diff --git a/frontend/scripts/instance-external-access.test.mjs b/frontend/scripts/instance-external-access.test.mjs
new file mode 100644
index 0000000..8bedbca
--- /dev/null
+++ b/frontend/scripts/instance-external-access.test.mjs
@@ -0,0 +1,57 @@
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import path from "node:path";
+
+const scriptDir = path.dirname(fileURLToPath(import.meta.url));
+const detailSource = readFileSync(
+ path.resolve(scriptDir, "../src/pages/instances/InstanceDetailPage.tsx"),
+ "utf8",
+);
+const serviceSource = readFileSync(
+ path.resolve(scriptDir, "../src/services/instanceService.ts"),
+ "utf8",
+);
+const typeSource = readFileSync(
+ path.resolve(scriptDir, "../src/types/instance.ts"),
+ "utf8",
+);
+
+function assert(condition, message) {
+ if (!condition) {
+ throw new Error(message);
+ }
+}
+
+for (const label of ["1 hour", "24 hours", "7 days", "30 days", "Custom", "Permanent"]) {
+ assert(detailSource.includes(label), `External access expiration option missing: ${label}`);
+}
+
+assert(
+ detailSource.includes('useState("preset")') &&
+ detailSource.includes('useState("24h")'),
+ "External access UI must default to a 24 hour preset.",
+);
+assert(
+ serviceSource.includes("ExternalAccessRequest") &&
+ serviceSource.includes("external-access/share-link") &&
+ serviceSource.includes("external-access/password"),
+ "Instance service must send expiration payloads to share link and password endpoints.",
+);
+assert(
+ detailSource.includes("Share Link") &&
+ detailSource.includes("Password") &&
+ !detailSource.includes("Public Link") &&
+ !detailSource.includes("API Key"),
+ "Instance external access UI must use Share Link and Password labels.",
+);
+assert(
+ !serviceSource.includes("external-access/public-link") &&
+ !serviceSource.includes("external-access/api-key"),
+ "Instance service must not call legacy public-link or api-key endpoints.",
+);
+assert(
+ !typeSource.includes("public_slug:") && !typeSource.includes("token?:"),
+ "Frontend external access types must not expose legacy long-link slug or token fields.",
+);
+
+console.log("Instance external access short-link UI contract is valid.");
diff --git a/frontend/scripts/instance-list-websocket.test.mjs b/frontend/scripts/instance-list-websocket.test.mjs
new file mode 100644
index 0000000..a45e423
--- /dev/null
+++ b/frontend/scripts/instance-list-websocket.test.mjs
@@ -0,0 +1,39 @@
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import path from "node:path";
+
+const scriptDir = path.dirname(fileURLToPath(import.meta.url));
+const listSource = readFileSync(
+ path.resolve(scriptDir, "../src/pages/instances/InstanceListPage.tsx"),
+ "utf8",
+);
+const detailSource = readFileSync(
+ path.resolve(scriptDir, "../src/pages/instances/InstanceDetailPage.tsx"),
+ "utf8",
+);
+
+function assert(condition, message) {
+ if (!condition) {
+ throw new Error(message);
+ }
+}
+
+assert(
+ !listSource.includes("useInstanceStatusWebSocket") &&
+ !listSource.includes("InstanceStatusUpdate"),
+ "Instance list must not keep a long-lived instance status WebSocket connection.",
+);
+
+assert(
+ listSource.includes("window.setInterval") &&
+ listSource.includes('instance.status === "creating"') &&
+ listSource.includes('instance.status === "deleting"'),
+ "Instance list must keep transition-state polling after removing WebSocket updates.",
+);
+
+assert(
+ detailSource.includes("useInstanceStatusWebSocket"),
+ "Instance detail should keep WebSocket updates for focused real-time status.",
+);
+
+console.log("Instance list WebSocket contract is valid.");
diff --git a/frontend/scripts/instance-portal-mode-layout.test.mjs b/frontend/scripts/instance-portal-mode-layout.test.mjs
new file mode 100644
index 0000000..78f8c91
--- /dev/null
+++ b/frontend/scripts/instance-portal-mode-layout.test.mjs
@@ -0,0 +1,104 @@
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import path from "node:path";
+
+const scriptDir = path.dirname(fileURLToPath(import.meta.url));
+const portalSource = readFileSync(
+ path.resolve(scriptDir, "../src/pages/instances/InstancePortalPage.tsx"),
+ "utf8",
+);
+const desktopAccessHookSource = readFileSync(
+ path.resolve(scriptDir, "../src/hooks/useInstanceDesktopAccess.ts"),
+ "utf8",
+);
+
+function assert(condition, message) {
+ if (!condition) {
+ throw new Error(message);
+ }
+}
+
+assert(
+ portalSource.includes("WorkspaceFileManager"),
+ "Portal must reuse the instance detail workspace file manager.",
+);
+
+assert(
+ portalSource.includes('selectedInstance.instance_mode === "pro"'),
+ "Portal must branch rendering by instance_mode so Lite instances are not shown as Pro.",
+);
+
+assert(
+ portalSource.includes('initialPath="/config"'),
+ "Portal Pro workspace file manager must match detail page config-root behavior.",
+);
+
+assert(
+ portalSource.includes('data-portal-mode={isProPortal ? "pro" : "lite"}'),
+ "Portal must expose separate Pro and Lite layout markers.",
+);
+
+assert(
+ portalSource.includes("xl:grid-rows-[minmax(0,1fr)]") &&
+ portalSource.includes("xl:overflow-hidden") &&
+ !portalSource.includes("content-start overflow-y-auto") &&
+ !portalSource.includes("aspect-video"),
+ "Portal Pro desktop layout must stretch to fill the available height without leaving a lower blank area.",
+);
+
+assert(
+ portalSource.includes("canShowRuntimeOverlay") &&
+ portalSource.includes("isProPortal"),
+ "Portal runtime controls must be limited to Pro layout.",
+);
+
+assert(
+ !portalSource.includes("__openclaw__/canvas"),
+ "Portal Pro desktop instances must use the backend proxy entry instead of the OpenClaw gateway canvas route.",
+);
+
+assert(
+ portalSource.includes("portalEmbedUrl") &&
+ portalSource.includes("portalEmbedUrlForInstance") &&
+ portalSource.includes("preparedPortalFrame.embedUrl === portalEmbedUrl") &&
+ portalSource.includes("src={portalFrameSrc}"),
+ "Portal iframe must use a mode-aware embed URL instead of the raw access URL.",
+);
+
+assert(
+ portalSource.includes("instanceIdFromProxyUrl") &&
+ portalSource.includes("embedUrlInstanceId !== instance.id"),
+ "Portal iframe must not reuse an access URL from a previously selected instance.",
+);
+
+assert(
+ portalSource.includes("setShouldConnect(false)") &&
+ portalSource.includes("instance.id !== selectedId"),
+ "Portal must reset the connection intent before switching to another instance.",
+);
+
+assert(
+ portalSource.includes("pendingConnectInstanceIdRef") &&
+ portalSource.includes("pendingConnectInstanceIdRef.current === nextInstanceId") &&
+ portalSource.includes("shouldConnect && portalEmbedUrl"),
+ "Portal access generation must target the currently selected instance after switching.",
+);
+
+assert(
+ portalSource.includes('selectedInstanceStatus !== "running"') &&
+ portalSource.includes("requestAccess();"),
+ "Portal must auto-connect running instances when switching in the portal.",
+);
+
+assert(
+ /modeLabel\(\s*instance\.instance_mode\s*\)/.test(portalSource) &&
+ /modeClass\(\s*instance\.instance_mode\s*,?\s*\)/.test(portalSource),
+ "Portal left instance list must display the Lite/Pro instance type.",
+);
+
+assert(
+ desktopAccessHookSource.includes("refreshAccess({ forceReload: true })"),
+ "Desktop access hook must force a fresh access URL when connecting a selected instance.",
+);
+
+console.log("Instance portal mode layout contract is valid.");
diff --git a/frontend/scripts/runtime-pods-page-contract.test.mjs b/frontend/scripts/runtime-pods-page-contract.test.mjs
new file mode 100644
index 0000000..07f04a0
--- /dev/null
+++ b/frontend/scripts/runtime-pods-page-contract.test.mjs
@@ -0,0 +1,49 @@
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import path from "node:path";
+
+const scriptDir = path.dirname(fileURLToPath(import.meta.url));
+const layoutPath = path.resolve(scriptDir, "../src/components/AdminLayout.tsx");
+const pagePath = path.resolve(scriptDir, "../src/pages/admin/RuntimePodsPage.tsx");
+const i18nPath = path.resolve(scriptDir, "../src/lib/i18n.ts");
+
+const layoutSource = readFileSync(layoutPath, "utf8");
+const pageSource = readFileSync(pagePath, "utf8");
+const i18nSource = readFileSync(i18nPath, "utf8");
+
+function assert(condition, message) {
+ if (!condition) {
+ throw new Error(message);
+ }
+}
+
+assert(
+ layoutSource.includes("t('nav.runtime')") || layoutSource.includes('t("nav.runtime")'),
+ "Admin sidebar must label the runtime pod page through nav.runtime.",
+);
+assert(
+ !layoutSource.includes("label: 'Runtime Pods'") && !layoutSource.includes('label: "Runtime Pods"'),
+ "Admin sidebar must not hard-code Runtime Pods.",
+);
+assert(
+ pageSource.includes("t('runtimePods.title')") || pageSource.includes('t("runtimePods.title")'),
+ "Runtime pods page title must use runtimePods.title.",
+);
+assert(
+ pageSource.includes("selectRuntimePod(pod)") &&
+ pageSource.includes('role="button"') &&
+ pageSource.includes("onKeyDown={(event) => handleRuntimePodCardKeyDown(event, pod)}"),
+ "Runtime pod cards must be selectable by clicking or using the keyboard.",
+);
+assert(
+ pageSource.includes("event.stopPropagation()"),
+ "Runtime pod card action buttons must not trigger card selection accidentally.",
+);
+assert(
+ i18nSource.includes('runtime: "Runtime"') &&
+ i18nSource.includes('runtime: "运行时"') &&
+ i18nSource.includes('title: "运行时"'),
+ "Runtime page translations must include English and Chinese runtime labels.",
+);
+
+console.log("Runtime pods page source contract is valid.");
diff --git a/frontend/scripts/system-settings-runtime-groups.test.mjs b/frontend/scripts/system-settings-runtime-groups.test.mjs
new file mode 100644
index 0000000..3675980
--- /dev/null
+++ b/frontend/scripts/system-settings-runtime-groups.test.mjs
@@ -0,0 +1,70 @@
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import path from "node:path";
+
+const scriptDir = path.dirname(fileURLToPath(import.meta.url));
+const pagePath = path.resolve(
+ scriptDir,
+ "../src/pages/admin/SystemSettingsPage.tsx",
+);
+const servicePath = path.resolve(
+ scriptDir,
+ "../src/services/systemSettingsService.ts",
+);
+const i18nPath = path.resolve(scriptDir, "../src/lib/i18n.ts");
+
+const pageSource = readFileSync(pagePath, "utf8");
+const serviceSource = readFileSync(servicePath, "utf8");
+const i18nSource = readFileSync(i18nPath, "utf8");
+
+function assert(condition, message) {
+ if (!condition) {
+ throw new Error(message);
+ }
+}
+
+assert(
+ serviceSource.includes('"desktop" | "gateway"'),
+ "System image settings must expose desktop/gateway runtime types.",
+);
+assert(
+ !serviceSource.includes('"desktop" | "shell"'),
+ "System image settings must not expose shell as the Lite runtime type.",
+);
+assert(
+ pageSource.includes("LITE_RUNTIME_CARDS") &&
+ pageSource.includes("PRO_BASE_RUNTIME_CARDS"),
+ "System settings page must define separate Lite and Pro runtime groups.",
+);
+assert(
+ pageSource.includes("runtime_type: 'gateway'") &&
+ pageSource.includes("runtime_type: 'desktop'"),
+ "System settings page must map Lite to gateway and Pro to desktop.",
+);
+assert(
+ pageSource.includes("ghcr.io/yuan-lab-llm/agentsruntime/openclaw-lite:latest") &&
+ pageSource.includes("ghcr.io/yuan-lab-llm/agentsruntime/hermes-lite:latest"),
+ "System settings page must use the Lite default runtime images.",
+);
+assert(
+ pageSource.includes("addProCustomCard"),
+ "System settings page must keep custom Pro runtime card creation.",
+);
+assert(
+ pageSource.includes("systemSettingsPage.liteRolloutTitle") &&
+ pageSource.includes("systemSettingsPage.proRuntimeTitle") &&
+ i18nSource.includes("Lite runtime rolling upgrade") &&
+ i18nSource.includes("Pro runtime"),
+ "System settings page must render separate rollout, Lite, and Pro sections.",
+);
+assert(
+ pageSource.includes("runtimePoolService.listPods(rolloutRuntimeType)") &&
+ pageSource.includes("rolloutCurrentImage"),
+ "System settings rollout current image must come from live runtime pods, not only saved card settings.",
+);
+assert(
+ !pageSource.includes("RUNTIME_TYPE_OPTIONS"),
+ "System settings page must not expose the legacy Shell/Desktop selector.",
+);
+
+console.log("System settings runtime grouping source contract is valid.");
diff --git a/frontend/scripts/user-portal-entry.test.mjs b/frontend/scripts/user-portal-entry.test.mjs
new file mode 100644
index 0000000..cbc0705
--- /dev/null
+++ b/frontend/scripts/user-portal-entry.test.mjs
@@ -0,0 +1,39 @@
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import path from "node:path";
+
+const scriptDir = path.dirname(fileURLToPath(import.meta.url));
+const layoutSource = readFileSync(
+ path.resolve(scriptDir, "../src/components/UserLayout.tsx"),
+ "utf8",
+);
+const instanceListSource = readFileSync(
+ path.resolve(scriptDir, "../src/pages/instances/InstanceListPage.tsx"),
+ "utf8",
+);
+const routerSource = readFileSync(
+ path.resolve(scriptDir, "../src/router/index.tsx"),
+ "utf8",
+);
+
+function assert(condition, message) {
+ if (!condition) {
+ throw new Error(message);
+ }
+}
+
+assert(
+ !layoutSource.includes("path: '/portal'") &&
+ !layoutSource.includes('path: "/portal"'),
+ "User sidebar must not expose Portal as a top-level navigation entry.",
+);
+assert(
+ instanceListSource.includes('to="/portal"'),
+ "Instance list must keep the Portal entry point.",
+);
+assert(
+ routerSource.includes('path="/portal"'),
+ "Portal route must remain available for instance-page entry points.",
+);
+
+console.log("User portal entry placement is valid.");
diff --git a/frontend/scripts/workspace-folder-upload.test.mjs b/frontend/scripts/workspace-folder-upload.test.mjs
new file mode 100644
index 0000000..c964df7
--- /dev/null
+++ b/frontend/scripts/workspace-folder-upload.test.mjs
@@ -0,0 +1,66 @@
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import path from "node:path";
+
+const scriptDir = path.dirname(fileURLToPath(import.meta.url));
+const source = readFileSync(
+ path.resolve(scriptDir, "../src/components/WorkspaceFileManager.tsx"),
+ "utf8",
+);
+
+function assert(condition, message) {
+ if (!condition) {
+ throw new Error(message);
+ }
+}
+
+assert(
+ source.includes("folderUploadInputRef"),
+ "Workspace file manager must include a dedicated folder upload input.",
+);
+
+assert(
+ /
-
-
-
+
+
+
+
+
);
}
diff --git a/frontend/src/components/AdminLayout.tsx b/frontend/src/components/AdminLayout.tsx
index ad1d3f5..1f9ced1 100644
--- a/frontend/src/components/AdminLayout.tsx
+++ b/frontend/src/components/AdminLayout.tsx
@@ -1,5 +1,18 @@
import React, { useState } from 'react';
import { Link, useLocation } from 'react-router-dom';
+import {
+ ArrowLeft,
+ Bot,
+ ChevronDown,
+ Gauge,
+ Home,
+ LogOut,
+ Monitor,
+ Server,
+ Settings,
+ Shield,
+ Users,
+} from 'lucide-react';
import { useAuth } from '../contexts/AuthContext';
import { useI18n } from '../contexts/I18nContext';
import LanguageSwitcher from './LanguageSwitcher';
@@ -12,18 +25,40 @@ interface AdminLayoutProps {
interface NavItem {
path: string;
label: string;
- icon: string;
+ icon: React.ComponentType<{ className?: string }>;
matchPaths?: string[];
exact?: boolean;
}
+const shellContainerClass = 'w-full px-3 sm:px-4 lg:px-5 2xl:px-6';
+const appLogoSrc = '/lobster_logo.png';
+
const AdminLayout: React.FC = ({ children, title }) => {
const location = useLocation();
const { user, logout } = useAuth();
const { t } = useI18n();
- const shellContainerClass = 'mx-auto w-full max-w-[1800px] px-4 sm:px-6 lg:px-8 xl:px-10 2xl:px-12';
const [profileExpanded, setProfileExpanded] = useState(false);
+ const navItems: NavItem[] = [
+ { path: '/admin', label: t('nav.adminDashboard'), icon: Home, exact: true },
+ { path: '/admin/users', label: t('nav.users'), icon: Users },
+ { path: '/admin/instances', label: t('nav.instances'), icon: Monitor },
+ { path: '/admin/runtime-pods', label: t('nav.runtime'), icon: Server },
+ {
+ path: '/admin/security',
+ label: t('nav.securityCenter'),
+ icon: Shield,
+ matchPaths: ['/admin/assets', '/admin/skills'],
+ },
+ {
+ path: '/admin/ai-gateway',
+ label: t('nav.aiGateway'),
+ icon: Bot,
+ matchPaths: ['/admin/models', '/admin/ai-audit', '/admin/costs', '/admin/risk-rules'],
+ },
+ { path: '/admin/settings', label: t('nav.settings'), icon: Settings },
+ ];
+
const isActive = (item: NavItem) => {
if (item.exact) {
return location.pathname === item.path;
@@ -37,19 +72,19 @@ const AdminLayout: React.FC = ({ children, title }) => {
window.location.href = '/login';
};
- const navItems: NavItem[] = [
- { path: '/admin', label: t('nav.adminDashboard'), icon: 'M3 12l2-2m0 0l7-7 7 7M5 10v10a1 1 0 001 1h3m10-11l2 2m-2-2v10a1 1 0 01-1 1h-3m-6 0a1 1 0 001-1v-4a1 1 0 011-1h2a1 1 0 011 1v4a1 1 0 001 1m-6 0h6', exact: true },
- { path: '/admin/users', label: t('nav.users'), icon: 'M12 4.354a4 4 0 110 5.292M15 21H3v-1a6 6 0 0112 0v1zm0 0h6v-1a6 6 0 00-9-5.197M13 7a4 4 0 11-8 0 4 4 0 018 0z' },
- { path: '/admin/instances', label: t('nav.instances'), icon: 'M5 12h14M5 12a2 2 0 01-2-2V6a2 2 0 012-2h14a2 2 0 012 2v4a2 2 0 01-2 2M5 12a2 2 0 00-2 2v4a2 2 0 002 2h14a2 2 0 002-2v-4a2 2 0 00-2-2m-2-4h.01M17 16h.01' },
- { path: '/admin/security', label: t('nav.securityCenter'), icon: 'M12 2l8 4.5v5c0 5.8-3.6 10.8-8 12.5-4.4-1.7-8-6.7-8-12.5v-5L12 2z', matchPaths: ['/admin/assets', '/admin/skills'] },
- {
- path: '/admin/ai-gateway',
- label: t('nav.aiGateway'),
- icon: 'M12 2l8 4.5v5c0 5.8-3.6 10.8-8 12.5-4.4-1.7-8-6.7-8-12.5v-5L12 2zm0 6.5v3m0 4h.01M9 12h6',
- matchPaths: ['/admin/models', '/admin/ai-audit', '/admin/costs', '/admin/risk-rules'],
- },
- { path: '/admin/settings', label: t('nav.settings'), icon: 'M10.325 4.317c.426-1.756 2.924-1.756 3.35 0a1.724 1.724 0 002.573 1.066c1.543-.94 3.31.826 2.37 2.37a1.724 1.724 0 001.065 2.572c1.756.426 1.756 2.924 0 3.35a1.724 1.724 0 00-1.066 2.573c.94 1.543-.826 3.31-2.37 2.37a1.724 1.724 0 00-2.572 1.065c-.426 1.756-2.924 1.756-3.35 0a1.724 1.724 0 00-2.573-1.066c-1.543.94-3.31-.826-2.37-2.37a1.724 1.724 0 00-1.065-2.572c-1.756-.426-1.756-2.924 0-3.35a1.724 1.724 0 001.066-2.573c-.94-1.543.826-3.31 2.37-2.37.996.608 2.296.07 2.572-1.065z M15 12a3 3 0 11-6 0 3 3 0 016 0z' },
- ];
+ const renderNavItem = (item: NavItem) => {
+ const Icon = item.icon;
+ return (
+
+
+ {item.label}
+
+ );
+ };
return (
@@ -57,235 +92,112 @@ const AdminLayout: React.FC
= ({ children, title }) => {
-
+

-
{t('app.name')}
+
{t('app.name')}
-
-
-
- {navItems.map((item) => (
-
-
- {item.label}
-
- ))}
-
-
+
-
- {children}
-
+
{children}