github-featured/yuan-lab-llm--clawmanager
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

feat: Skill Hub hardening with session usage tracking and egress governance (#163)

Browse Source 
* feat(skill-hub): add Skill Hub catalog, publish flow, and lite materialize pipeline

Introduce Skill Hub for browsing, importing, publishing, and installing skills, with lite instance package materialization and runtime sync support.

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(skill-hub): remove token-governance hooks from Skill Hub PR

Strip validateManagedRuntimeEnvironmentOverrides, network lock policy sync,
and egress proxy audit wiring that belong to the upcoming token-usage work,
so the Skill Hub branch compiles independently.

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(skill-hub): update migration number in materialize docs

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(skill-hub): make RuntimeAgentClient test stub and hub tests compile-safe

Add ResyncInstanceSkills to the runtime pool handler fake client, and harden
skill hub payload helpers/tests against nil storage/instance repos so go test passes.

Co-authored-by: Cursor <cursoragent@cursor.com>

* feat: add Skill Hub hardening with session usage tracking and egress governance

Unify Skill Hub runtime sync improvements with session-token observability,
egress network policy, and admin/instance usage reporting for reopenable PR.

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(skill-hub): repair CI tests and nested skill install

* fix(ci): restore release deployment configuration

---------

Co-authored-by: heshengran <heshengran@ieisystem.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
heranran
2026-07-17 12:37:01 +08:00
committed by GitHub
parent 58c19856fd
commit 71c7672545
120 changed files with 15204 additions and 883 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build.yml
+6
View File
@@ -166,6 +166,12 @@ jobs:
text,
)
text = re.sub(
r"(\n\s+- name: workspaces\s+)persistentVolumeClaim:\s+claimName: clawmanager-workspaces",
r"\1emptyDir: {}",
text,
)
path.write_text(text, encoding="utf-8")
PY
.gitignore
+9
View File
@@ -6,6 +6,7 @@
*.dylib
/backend/bin/
/backend/tmp/
/runtimeagent/
# Test binary, built with `go test -c`
*.test
@@ -61,6 +62,14 @@ Thumbs.db
/fix-install-plan.md
/docs/deployment-storage-remediation-plan.zh-CN.md
# Local API / debug artifacts (do not commit)
/_*.json
/_*.txt
/_test_*.js
/_pr*.md
.clawmanager-last-image
/pr138.patch
# Team runtime config snapshots
/team-*.json
/agency-agents-main/
backend/cmd/server/main.go
+77 -7
View File
@@ -1,4 +1,4 @@
package main
package main
import (
"context"
@@ -104,7 +104,7 @@ func main() {
log.Fatalf("Failed to initialize object storage: %v", err)
}
skillScannerClient := services.NewSkillScannerClient(cfg.SkillScanner)
aiObservabilityService := services.NewAIObservabilityService(modelInvocationRepo, auditEventRepo, costRecordRepo, riskHitRepo, chatMessageRepo, llmModelRepo, instanceRepo, userRepo)
aiObservabilityService := services.NewAIObservabilityService(modelInvocationRepo, auditEventRepo, costRecordRepo, riskHitRepo, chatMessageRepo, chatSessionRepo, llmModelRepo, instanceRepo, userRepo, instanceRuntimeStatusRepo)
clusterResourceService := services.NewClusterResourceService(instanceRepo)
services.SetRuntimeImageSettingsProvider(systemImageSettingService)
services.SetOpenClawTransferRuntimeRepositories(instanceRepo, bindingRepo, runtimePodRepo)
@@ -140,7 +140,19 @@ func main() {
runtimeEvents := services.NewRuntimeEventService(platformRedis)
workspaceFileService := services.NewWorkspaceFileService(workspaceFileAuditRepo)
runtimeWorkspaceFileService := services.NewRuntimeWorkspaceFileService(workspaceFileAuditRepo)
skillService := services.NewSkillService(skillRepo, instanceRepo, instanceCommandService, objectStorageService, skillScannerClient)
skillService := services.NewSkillService(skillRepo, instanceRepo, userRepo, instanceCommandService, instanceCommandRepo, objectStorageService, skillScannerClient)
materializeJobRepo := repository.NewSkillPackageMaterializeJobRepository(database)
materializeService := services.NewSkillPackageMaterializeService(materializeJobRepo, skillRepo, services.SkillServiceAsMaterializer(skillService))
services.ConfigureSkillPackageMaterialize(skillService, materializeService)
materializeWorker := services.NewSkillPackageMaterializeWorker(
materializeService,
time.Duration(cfg.SkillMaterialize.TickMS)*time.Millisecond,
cfg.SkillMaterialize.BatchSize,
cfg.SkillMaterialize.Concurrency,
cfg.SkillMaterialize.PerInstanceConcurrency,
cfg.SkillMaterialize.Enabled,
)
services.ConfigureSkillRuntimeSync(skillService, bindingRepo, runtimePodRepo, runtimeAgentClient)
securityScanService := services.NewSecurityScanService(securityScanRepo, skillRepo, objectStorageService, skillScannerClient)
externalAccessService := services.NewInstanceExternalAccessService(instanceExternalAccessRepo)
aiGatewayService := aigateway.NewService(llmModelRepo, modelInvocationService, auditEventService, costRecordService, riskDetectionService, riskHitService, chatSessionService, chatMessageService)
@@ -157,6 +169,7 @@ func main() {
openClawConfigService,
skillService,
externalAccessService,
aiObservabilityService,
services.WithInstanceProxyRuntimeRepositories(instanceRepo, runtimePodRepo, bindingRepo),
)
systemSettingsHandler := handlers.NewSystemSettingsHandler(systemImageSettingService)
@@ -165,15 +178,16 @@ func main() {
aiObservabilityHandler := handlers.NewAIObservabilityHandler(aiObservabilityService)
riskRuleHandler := handlers.NewRiskRuleHandler(riskRuleService)
clusterResourceHandler := handlers.NewClusterResourceHandler(clusterResourceService)
egressProxyHandler := handlers.NewEgressProxyHandler()
egressProxyHandler := handlers.NewEgressProxyHandler(auditEventService)
openClawConfigHandler := handlers.NewOpenClawConfigHandler(openClawConfigService)
skillHandler := handlers.NewSkillHandler(skillService, instanceService)
skillHubHandler := handlers.NewSkillHubHandler(skillService, instanceService)
securityHandler := handlers.NewSecurityHandler(securityScanService)
agentHandler := handlers.NewAgentHandler(instanceAgentService, instanceCommandService, instanceRuntimeStatusService, instanceConfigRevisionService, skillService)
teamHandler := handlers.NewTeamHandler(teamService)
workspaceFileHandler := handlers.NewWorkspaceFileHandler(instanceService, workspaceFileService, runtimeWorkspaceFileService)
workspaceFileHandler.SetSkillRepository(skillRepo)
runtimeAgentHandler := handlers.NewRuntimeAgentHandler(cfg.Runtime, runtimePodRepo, bindingRepo, instanceRepo, runtimeEvents)
runtimeAgentHandler := handlers.NewRuntimeAgentHandler(cfg.Runtime, runtimePodRepo, bindingRepo, instanceRepo, runtimeEvents, skillService)
// Initialize WebSocket hub and handler
wsHub := services.GetHub()
@@ -238,6 +252,7 @@ func main() {
startBackground := func(ctx context.Context) {
log.Printf("Starting leader-only background loops (identity=%s)", cfg.LeaderElection.Identity)
syncService.Start()
materializeWorker.Start()
teamService.StartBackground(ctx)
if runtimeScheduler != nil {
runtimeSchedulerMu.Lock()
@@ -252,6 +267,7 @@ func main() {
}
stopBackground := func() {
log.Printf("Stopping leader-only background loops (identity=%s)", cfg.LeaderElection.Identity)
materializeWorker.Stop()
runtimeSchedulerMu.Lock()
if runtimeSchedulerCancel != nil {
runtimeSchedulerCancel()
@@ -354,6 +370,8 @@ func main() {
instances.POST("/:id/restart", instanceHandler.RestartInstance)
instances.GET("/:id/status", instanceHandler.GetInstanceStatus)
instances.GET("/:id/runtime", instanceHandler.GetRuntimeDetails)
instances.GET("/:id/session-usage", instanceHandler.GetInstanceSessionUsage)
instances.GET("/:id/session-usage/detail", instanceHandler.GetInstanceSessionUsageDetail)
instances.POST("/:id/runtime/:command", instanceHandler.CreateRuntimeCommand)
instances.GET("/:id/config/revisions", instanceHandler.ListConfigRevisions)
instances.POST("/:id/config/revisions/publish", instanceHandler.PublishConfigRevision)
@@ -379,11 +397,15 @@ func main() {
instances.GET("/:id/skills", skillHandler.ListInstanceSkills)
instances.GET("/:id/skills/available", skillHandler.ListAvailableInstanceSkills)
instances.POST("/:id/skills", skillHandler.AttachSkillToInstance)
instances.POST("/:id/skills/sync", instanceHandler.RefreshInstanceSkills)
instances.POST("/:id/skills/:skillId/import-to-library", instanceHandler.ImportInstanceSkillToLibrary)
instances.POST("/:id/skills/:skillId/retry-package-collect", instanceHandler.RetrySkillPackageCollect)
instances.POST("/:id/skills/:skillId/publish-to-hub", instanceHandler.PublishInstanceSkillToHub)
instances.DELETE("/:id/skills/:skillId", skillHandler.RemoveSkillFromInstance)
}
// Admin console: cross-user instance listing. Gated by admin
// middleware — non-admin callers get 403. The workspace
// middleware 鈥?non-admin callers get 403. The workspace
// /instances endpoint above stays caller-scoped regardless of
// role; admin status only unlocks this dedicated surface.
adminInstances := api.Group("/admin/instances")
@@ -464,6 +486,38 @@ func main() {
skills.GET("/:id/scan-results", skillHandler.ListScanResults)
}
skillHub := api.Group("/skill-hub")
skillHub.Use(middleware.Auth())
skillHub.Use(middleware.SetUserInfo(userRepo))
{
skillHub.GET("/catalog", skillHubHandler.ListCatalog)
skillHub.GET("/tags", skillHubHandler.ListTags)
skillHub.GET("/mine", skillHubHandler.ListMine)
skillHub.GET("/attachable", skillHubHandler.ListAttachable)
skillHub.POST("/skills/import/preview", skillHubHandler.PreviewImportSkills)
skillHub.POST("/skills/import", skillHubHandler.ImportSkills)
skillHub.GET("/skills/:id", skillHubHandler.GetSkill)
skillHub.POST("/skills/:id/publish", skillHubHandler.PublishSkill)
skillHub.POST("/skills/:id/unpublish", skillHubHandler.UnpublishSkill)
skillHub.PUT("/skills/:id/tags", skillHubHandler.UpdateTags)
skillHub.DELETE("/skills/:id", skillHubHandler.DeleteSkill)
skillHub.GET("/skills/:id/download", skillHubHandler.DownloadSkill)
skillHub.POST("/skills/:id/install", skillHubHandler.InstallSkill)
}
adminSkillHub := api.Group("/admin/skill-hub")
adminSkillHub.Use(middleware.Auth())
adminSkillHub.Use(middleware.SetUserInfo(userRepo))
adminSkillHub.Use(middleware.NewAdminAuth(userRepo))
{
adminSkillHub.GET("/skills", skillHubHandler.ListAdminSkills)
adminSkillHub.POST("/skills/:id/publish", skillHubHandler.PublishSkill)
adminSkillHub.POST("/skills/:id/unpublish", skillHubHandler.UnpublishSkill)
adminSkillHub.PUT("/skills/:id/tags", skillHubHandler.UpdateTags)
adminSkillHub.DELETE("/skills/:id", skillHubHandler.DeleteSkill)
adminSkillHub.POST("/skills/:id/install", skillHubHandler.InstallSkill)
}
systemSettings := api.Group("/system-settings")
systemSettings.Use(middleware.Auth())
systemSettings.Use(middleware.SetUserInfo(userRepo))
@@ -509,6 +563,22 @@ func main() {
adminCosts.GET("", aiObservabilityHandler.GetCostOverview)
}
adminLLMGovernance := api.Group("/admin/llm-governance")
adminLLMGovernance.Use(middleware.Auth())
adminLLMGovernance.Use(middleware.SetUserInfo(userRepo))
adminLLMGovernance.Use(middleware.NewAdminAuth(userRepo))
{
adminLLMGovernance.GET("/overview", aiObservabilityHandler.GetLLMGovernanceOverview)
}
adminSessionUsage := api.Group("/admin/session-usage")
adminSessionUsage.Use(middleware.Auth())
adminSessionUsage.Use(middleware.SetUserInfo(userRepo))
adminSessionUsage.Use(middleware.NewAdminAuth(userRepo))
{
adminSessionUsage.GET("/overview", aiObservabilityHandler.GetSessionUsageOverview)
}
adminRiskRules := api.Group("/admin/risk-rules")
adminRiskRules.Use(middleware.Auth())
adminRiskRules.Use(middleware.SetUserInfo(userRepo))
@@ -559,7 +629,7 @@ func main() {
agent.POST("/state/report", agentHandler.ReportState)
agent.POST("/skills/inventory", agentHandler.ReportSkillInventory)
agent.POST("/skills/upload", agentHandler.UploadSkillPackage)
agent.GET("/skills/versions/:skillVersion/download", skillHandler.DownloadSkillVersionForAgent)
agent.GET("/skills/versions/:skillVersion/download", agentHandler.DownloadSkillVersion)
agent.GET("/config/revisions/:id", agentHandler.GetConfigRevision)
}
backend/internal/aigateway/service.go
+44 -4
View File
@@ -23,6 +23,7 @@ import (
"clawreef/internal/models"
"clawreef/internal/repository"
"clawreef/internal/services"
"clawreef/internal/utils"
)
// ToolCallFunction represents a tool/function call payload.
@@ -71,6 +72,8 @@ type ChatCompletionRequest struct {
StreamOptions json.RawMessage `json:"stream_options,omitempty"`
User *string `json:"user,omitempty"`
SessionID *string `json:"session_id,omitempty"`
OpenClawSessionKey *string `json:"-"`
ManagedAgentType *string `json:"-"`
InstanceID *int `json:"instance_id,omitempty"`
InstanceMode *string `json:"instance_mode,omitempty"`
RuntimeType *string `json:"runtime_type,omitempty"`
@@ -429,16 +432,43 @@ func (s *service) prepareChatRequest(userID int, req ChatCompletionRequest) (*pr
selectedModel: selectedModel,
req: req,
}
prepared.sessionID = resolveSessionID(req)
sessionSource, sessionID := resolveSessionIdentity(req)
prepared.sessionID = sessionID
if prepared.sessionID != "" && sessionSource != "openclaw_header" {
if runtimeType := strings.TrimSpace(sessionNormalizationRuntimeType(req)); runtimeType != "" {
prepared.sessionID = utils.NormalizeOpenClawSessionID(prepared.sessionID, runtimeType)
}
}
prepared.traceID = s.resolveTraceID(userID, req, prepared.sessionID)
if prepared.sessionID == "" {
prepared.sessionID = normalizeSessionID(nil, prepared.traceID)
sessionSource = "trace_fallback"
}
prepared.sessionIDPtr = stringPtr(prepared.sessionID)
prepared.req.SessionID = prepared.sessionIDPtr
prepared.requestID = normalizeOrCreateID(req.RequestID, "req")
prepared.requestIDPtr = stringPtr(prepared.requestID)
if sessionSource == "trace_fallback" && req.InstanceID != nil {
if err := s.auditEventService.RecordEvent(&models.AuditEvent{
TraceID: prepared.traceID,
SessionID: prepared.sessionIDPtr,
RequestID: prepared.requestIDPtr,
UserID: prepared.userIDPtr,
InstanceID: req.InstanceID,
InstanceMode: runtimeAttributionString(req.InstanceMode),
RuntimeType: runtimeAttributionString(req.RuntimeType),
GatewayID: runtimeAttributionString(req.GatewayID),
RuntimePodID: runtimeAttributionInt64(req.RuntimePodID),
EventType: "gateway.session.fallback",
TrafficClass: models.TrafficClassLLM,
Severity: models.AuditSeverityWarn,
Message: fmt.Sprintf("LLM request missing stable session key for instance %d", *req.InstanceID),
}); err != nil {
logPersistenceError("record gateway.session.fallback", prepared.traceID, err)
}
}
sessionTitle := deriveSessionTitle(prepared.req.Messages)
if _, err := s.chatSessionService.EnsureSession(prepared.sessionID, prepared.userIDPtr, prepared.req.InstanceID, stringPtr(prepared.traceID), sessionTitle); err != nil {
logPersistenceError("ensure chat session", prepared.traceID, err)
@@ -2231,13 +2261,23 @@ func fallbackCurrency(currency string) string {
}
func resolveSessionID(req ChatCompletionRequest) string {
_, sessionID := resolveSessionIdentity(req)
return sessionID
}
func resolveSessionIdentity(req ChatCompletionRequest) (source string, sessionID string) {
if req.OpenClawSessionKey != nil {
if key := strings.TrimSpace(*req.OpenClawSessionKey); key != "" {
return "openclaw_header", utils.NormalizeOpenClawSessionID(key, sessionNormalizationRuntimeType(req))
}
}
if normalized := normalizeOptionalString(req.SessionID); normalized != "" {
return normalizeExistingIdentifier(normalized, "sess")
return "explicit", normalizeExistingIdentifier(normalized, "sess")
}
if normalized := normalizeOptionalString(req.User); normalized != "" {
return normalizeExistingIdentifier(normalized, "sess")
return "openai_user", normalizeExistingIdentifier(normalized, "sess")
}
return ""
return "", ""
}
func normalizeSessionID(value *string, traceID string) string {
backend/internal/aigateway/session_identity.go
+63
View File
@@ -0,0 +1,63 @@
package aigateway
import "strings"
const defaultManagedSessionKey = "main"
// IsManagedInstanceType reports whether an instance type participates in managed
// runtime LLM governance (OpenClaw / Hermes).
func IsManagedInstanceType(instanceType string) bool {
switch strings.ToLower(strings.TrimSpace(instanceType)) {
case "openclaw", "hermes":
return true
default:
return false
}
}
// HasExplicitSessionIdentity returns true when the caller supplied a stable
// session identifier via header or request body fields.
func HasExplicitSessionIdentity(req ChatCompletionRequest) bool {
if req.OpenClawSessionKey != nil && strings.TrimSpace(*req.OpenClawSessionKey) != "" {
return true
}
if normalizeOptionalString(req.SessionID) != "" {
return true
}
if normalizeOptionalString(req.User) != "" {
return true
}
return false
}
// ApplyManagedInstanceSessionDefaults fills in the default OpenClaw/Hermes session
// key for instance gateway token calls that omitted explicit session identity.
func ApplyManagedInstanceSessionDefaults(req *ChatCompletionRequest, gatewayAuthType, instanceType string) {
if req == nil {
return
}
if strings.TrimSpace(gatewayAuthType) != "instance" {
return
}
instanceType = strings.TrimSpace(instanceType)
if !IsManagedInstanceType(instanceType) {
return
}
req.ManagedAgentType = stringPtr(instanceType)
if HasExplicitSessionIdentity(*req) {
return
}
req.OpenClawSessionKey = stringPtr(defaultManagedSessionKey)
}
func sessionNormalizationRuntimeType(req ChatCompletionRequest) string {
if req.ManagedAgentType != nil {
if runtimeType := strings.TrimSpace(*req.ManagedAgentType); runtimeType != "" {
return runtimeType
}
}
if req.RuntimeType != nil {
return strings.TrimSpace(*req.RuntimeType)
}
return ""
}
backend/internal/aigateway/session_identity_test.go
+61
View File
@@ -0,0 +1,61 @@
package aigateway
import "testing"
func TestApplyManagedInstanceSessionDefaultsUsesMainForOpenClawInstanceToken(t *testing.T) {
req := ChatCompletionRequest{
Messages: []ChatMessage{{Role: "user", Content: "hello"}},
}
ApplyManagedInstanceSessionDefaults(&req, "instance", "openclaw")
if req.OpenClawSessionKey == nil || *req.OpenClawSessionKey != "main" {
t.Fatalf("expected default session key main, got %+v", req.OpenClawSessionKey)
}
if req.ManagedAgentType == nil || *req.ManagedAgentType != "openclaw" {
t.Fatalf("expected managed agent type openclaw, got %+v", req.ManagedAgentType)
}
if got := resolveSessionID(req); got != "agent:openclaw:main" {
t.Fatalf("expected normalized session id agent:openclaw:main, got %q", got)
}
}
func TestApplyManagedInstanceSessionDefaultsUsesMainForHermesInstanceToken(t *testing.T) {
req := ChatCompletionRequest{}
ApplyManagedInstanceSessionDefaults(&req, "instance", "hermes")
if got := resolveSessionID(req); got != "agent:hermes:main" {
t.Fatalf("expected normalized session id agent:hermes:main, got %q", got)
}
}
func TestApplyManagedInstanceSessionDefaultsSkipsUserJWTCalls(t *testing.T) {
req := ChatCompletionRequest{}
ApplyManagedInstanceSessionDefaults(&req, "user", "openclaw")
if req.OpenClawSessionKey != nil {
t.Fatalf("expected no default session key for user auth, got %+v", req.OpenClawSessionKey)
}
}
func TestApplyManagedInstanceSessionDefaultsRespectsExplicitHeader(t *testing.T) {
explicit := "work"
req := ChatCompletionRequest{
OpenClawSessionKey: &explicit,
RuntimeType: stringPtr("desktop"),
}
ApplyManagedInstanceSessionDefaults(&req, "instance", "openclaw")
if got := resolveSessionID(req); got != "agent:openclaw:work" {
t.Fatalf("expected explicit session key to win, got %q", got)
}
}
func TestApplyManagedInstanceSessionDefaultsRespectsExplicitSessionID(t *testing.T) {
explicit := "agent:openclaw:custom"
req := ChatCompletionRequest{
SessionID: &explicit,
}
ApplyManagedInstanceSessionDefaults(&req, "instance", "openclaw")
if req.OpenClawSessionKey != nil {
t.Fatalf("expected body session id to prevent default header injection, got %+v", req.OpenClawSessionKey)
}
if got := resolveSessionID(req); got != explicit {
t.Fatalf("expected explicit session id %q, got %q", explicit, got)
}
}
backend/internal/config/config.go
+31
View File
@@ -20,6 +20,7 @@ type Config struct {
Runtime RuntimePoolConfig `yaml:"runtime"`
ObjectStorage ObjectStorageConfig `yaml:"objectStorage"`
SkillScanner SkillScannerConfig `yaml:"skillScanner"`
SkillMaterialize SkillMaterializeConfig `yaml:"skillMaterialize"`
LeaderElection LeaderElectionConfig `yaml:"leaderElection"`
}
@@ -200,6 +201,14 @@ type SkillScannerConfig struct {
Enabled bool `yaml:"enabled"`
}
type SkillMaterializeConfig struct {
Enabled bool `yaml:"enabled"`
TickMS int `yaml:"tickMs"`
BatchSize int `yaml:"batchSize"`
Concurrency int `yaml:"concurrency"`
PerInstanceConcurrency int `yaml:"perInstanceConcurrency"`
}
// Load loads configuration from file and environment variables
func Load() (*Config, error) {
runtimeNamespace := getEnv("RUNTIME_NAMESPACE", getEnv("K8S_NAMESPACE", "clawmanager-system"))
@@ -305,6 +314,13 @@ func Load() (*Config, error) {
TimeoutSeconds: 30,
Enabled: strings.EqualFold(getEnv("SKILL_SCANNER_ENABLED", "false"), "true"),
},
SkillMaterialize: SkillMaterializeConfig{
Enabled: strings.EqualFold(getEnv("SKILL_MATERIALIZE_WORKER_ENABLED", "true"), "true"),
TickMS: 2000,
BatchSize: 5,
Concurrency: 5,
PerInstanceConcurrency: 2,
},
LeaderElection: LeaderElectionConfig{
Enabled: strings.EqualFold(getEnv("CLAWMANAGER_LEADER_ELECTION", "true"), "true"),
Namespace: getEnv("POD_NAMESPACE", "clawmanager-system"),
@@ -497,6 +513,21 @@ func applyEnvOverrides(config *Config) {
if timeoutSeconds := os.Getenv("SKILL_SCANNER_TIMEOUT_SECONDS"); timeoutSeconds != "" {
fmt.Sscanf(timeoutSeconds, "%d", &config.SkillScanner.TimeoutSeconds)
}
if enabled := os.Getenv("SKILL_MATERIALIZE_WORKER_ENABLED"); enabled != "" {
config.SkillMaterialize.Enabled = strings.EqualFold(enabled, "true")
}
if tickMS := os.Getenv("SKILL_MATERIALIZE_TICK_MS"); tickMS != "" {
fmt.Sscanf(tickMS, "%d", &config.SkillMaterialize.TickMS)
}
if batchSize := os.Getenv("SKILL_MATERIALIZE_BATCH_SIZE"); batchSize != "" {
fmt.Sscanf(batchSize, "%d", &config.SkillMaterialize.BatchSize)
}
if concurrency := os.Getenv("SKILL_MATERIALIZE_CONCURRENCY"); concurrency != "" {
fmt.Sscanf(concurrency, "%d", &config.SkillMaterialize.Concurrency)
}
if perInstance := os.Getenv("SKILL_MATERIALIZE_PER_INSTANCE_CONCURRENCY"); perInstance != "" {
fmt.Sscanf(perInstance, "%d", &config.SkillMaterialize.PerInstanceConcurrency)
}
}
func normalizeStorageConfig(config *Config) {
backend/internal/db/migrations/023_add_skill_hub.sql
+46
View File
@@ -0,0 +1,46 @@
ALTER TABLE skills
ADD COLUMN visibility ENUM('private', 'public') NOT NULL DEFAULT 'private' AFTER status,
ADD COLUMN published_at TIMESTAMP NULL AFTER visibility,
ADD COLUMN published_by INT NULL AFTER published_at,
ADD INDEX idx_skills_visibility (visibility, status, source_type);
ALTER TABLE skills
ADD CONSTRAINT fk_skills_published_by FOREIGN KEY (published_by) REFERENCES users(id) ON DELETE SET NULL;
CREATE TABLE IF NOT EXISTS skill_hub_tags (
id INT AUTO_INCREMENT PRIMARY KEY,
tag_key VARCHAR(64) NOT NULL,
name VARCHAR(120) NOT NULL,
description VARCHAR(255) NULL,
sort_order INT NOT NULL DEFAULT 0,
admin_only BOOLEAN NOT NULL DEFAULT FALSE,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
UNIQUE KEY uk_skill_hub_tags_tag_key (tag_key),
INDEX idx_skill_hub_tags_sort (sort_order, id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE IF NOT EXISTS skill_hub_tag_assignments (
id INT AUTO_INCREMENT PRIMARY KEY,
skill_id INT NOT NULL,
tag_id INT NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (skill_id) REFERENCES skills(id) ON DELETE CASCADE,
FOREIGN KEY (tag_id) REFERENCES skill_hub_tags(id) ON DELETE CASCADE,
UNIQUE KEY uk_skill_hub_tag_assignments (skill_id, tag_id),
INDEX idx_skill_hub_tag_assignments_tag (tag_id, skill_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
INSERT INTO skill_hub_tags (tag_key, name, description, sort_order, admin_only) VALUES
('productivity', 'Productivity', 'Efficiency and workflow skills', 10, FALSE),
('coding', 'Coding', 'Software development skills', 20, FALSE),
('browser', 'Browser', 'Browser automation skills', 30, FALSE),
('data', 'Data', 'Data processing and analytics skills', 40, FALSE),
('communication', 'Communication', 'Messaging and collaboration skills', 50, FALSE),
('automation', 'Automation', 'Task automation skills', 60, FALSE),
('research', 'Research', 'Research and information gathering skills', 70, FALSE),
('community', 'Community', 'Community shared skills', 80, FALSE),
('admin-curated', 'Admin Curated', 'Curated by platform administrators', 90, TRUE),
('featured', 'Featured', 'Featured on the Skill Hub', 100, TRUE);
UPDATE skills SET visibility = 'private';
backend/internal/db/migrations/039_add_skill_package_materialize.sql
+71
View File
@@ -0,0 +1,71 @@
SET @instance_skills_workspace_dir_exists = (
SELECT COUNT(*)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'instance_skills'
AND COLUMN_NAME = 'workspace_dir'
);
SET @instance_skills_workspace_dir_sql = IF(
@instance_skills_workspace_dir_exists = 0,
'ALTER TABLE instance_skills ADD COLUMN workspace_dir VARCHAR(120) NULL AFTER install_path',
'SELECT 1'
);
PREPARE instance_skills_workspace_dir_stmt FROM @instance_skills_workspace_dir_sql;
EXECUTE instance_skills_workspace_dir_stmt;
DEALLOCATE PREPARE instance_skills_workspace_dir_stmt;
CREATE TABLE IF NOT EXISTS skill_package_materialize_jobs (
id INT AUTO_INCREMENT PRIMARY KEY,
instance_id INT NOT NULL,
skill_id INT NOT NULL,
blob_id INT NOT NULL,
workspace_dir VARCHAR(120) NOT NULL,
content_hash VARCHAR(128) NOT NULL,
status VARCHAR(30) NOT NULL DEFAULT 'pending',
attempt_count INT NOT NULL DEFAULT 0,
max_attempts INT NOT NULL DEFAULT 5,
last_error TEXT NULL,
idempotency_key VARCHAR(255) NOT NULL,
trigger_source VARCHAR(50) NOT NULL DEFAULT 'sync',
started_at TIMESTAMP NULL,
finished_at TIMESTAMP NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
FOREIGN KEY (instance_id) REFERENCES instances(id) ON DELETE CASCADE,
FOREIGN KEY (skill_id) REFERENCES skills(id) ON DELETE CASCADE,
FOREIGN KEY (blob_id) REFERENCES skill_blobs(id) ON DELETE CASCADE,
UNIQUE KEY uk_sp_materialize_idempotency (idempotency_key),
INDEX idx_sp_materialize_status_created (status, created_at),
INDEX idx_sp_materialize_instance_status (instance_id, status),
INDEX idx_sp_materialize_blob (blob_id, status)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
UPDATE instance_skills
SET workspace_dir = SUBSTRING_INDEX(REPLACE(install_path, '\\', '/'), '/', -1)
WHERE workspace_dir IS NULL
AND install_path IS NOT NULL
AND TRIM(install_path) <> '';
UPDATE instance_commands ic
JOIN instances i ON i.id = ic.instance_id
SET ic.status = 'cancelled',
ic.error_message = 'superseded by skill_package_materialize_jobs'
WHERE ic.command_type = 'collect_skill_package'
AND ic.status IN ('pending', 'dispatched', 'running')
AND (LOWER(TRIM(i.instance_mode)) = 'lite' OR LOWER(TRIM(i.runtime_type)) = 'gateway');
SET @instance_skills_workspace_dir_index_exists = (
SELECT COUNT(*)
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'instance_skills'
AND INDEX_NAME = 'idx_instance_skills_workspace_dir'
);
SET @instance_skills_workspace_dir_index_sql = IF(
@instance_skills_workspace_dir_index_exists = 0,
'ALTER TABLE instance_skills ADD INDEX idx_instance_skills_workspace_dir (workspace_dir)',
'SELECT 1'
);
PREPARE instance_skills_workspace_dir_index_stmt FROM @instance_skills_workspace_dir_index_sql;
EXECUTE instance_skills_workspace_dir_index_stmt;
DEALLOCATE PREPARE instance_skills_workspace_dir_index_stmt;
backend/internal/db/migrations/040_fix_injected_skill_provenance.sql
+7
View File
@@ -0,0 +1,7 @@
UPDATE instance_skills isk
JOIN skills s ON s.id = isk.skill_id
SET isk.source_type = 'injected_by_clawmanager'
WHERE isk.source_type = 'discovered_in_instance'
AND s.source_type = 'uploaded'
AND s.visibility = 'public'
AND isk.status = 'active';
backend/internal/db/migrations/041_add_session_usage_indexes.sql
+52
View File
@@ -0,0 +1,52 @@
SET @dbname = DATABASE();
SET @indexname = 'idx_cost_records_instance_id';
SET @preparedStatement = (
SELECT IF(
EXISTS(
SELECT 1 FROM information_schema.statistics
WHERE table_schema = @dbname
AND table_name = 'cost_records'
AND index_name = @indexname
),
'SELECT 1',
'ALTER TABLE cost_records ADD INDEX idx_cost_records_instance_id (instance_id)'
)
);
PREPARE stmt FROM @preparedStatement;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
SET @indexname = 'idx_cost_records_session_id';
SET @preparedStatement = (
SELECT IF(
EXISTS(
SELECT 1 FROM information_schema.statistics
WHERE table_schema = @dbname
AND table_name = 'cost_records'
AND index_name = @indexname
),
'SELECT 1',
'ALTER TABLE cost_records ADD INDEX idx_cost_records_session_id (session_id)'
)
);
PREPARE stmt FROM @preparedStatement;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
SET @indexname = 'idx_model_invocations_instance_session';
SET @preparedStatement = (
SELECT IF(
EXISTS(
SELECT 1 FROM information_schema.statistics
WHERE table_schema = @dbname
AND table_name = 'model_invocations'
AND index_name = @indexname
),
'SELECT 1',
'ALTER TABLE model_invocations ADD INDEX idx_model_invocations_instance_session (instance_id, session_id, created_at)'
)
);
PREPARE stmt FROM @preparedStatement;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
backend/internal/db/migrations_test.go
+17
View File
@@ -183,3 +183,20 @@ func TestMigration038AddsGatewayTokenAliases(t *testing.T) {
t.Fatalf("migration 038 must not store raw access tokens")
}
}
func TestMigration041AddsSessionUsageIndexes(t *testing.T) {
raw, err := embeddedMigrations.ReadFile("migrations/041_add_session_usage_indexes.sql")
if err != nil {
t.Fatalf("read migration 041: %v", err)
}
sql := string(raw)
for _, required := range []string{
"idx_cost_records_instance_id",
"idx_cost_records_session_id",
"idx_model_invocations_instance_session",
} {
if !strings.Contains(sql, required) {
t.Fatalf("migration 041 must contain %s", required)
}
}
}
backend/internal/egresspolicy/policy.go
+118
View File
@@ -0,0 +1,118 @@
package egresspolicy
import (
"os"
"strings"
)
type Mode string
const (
ModeOpen Mode = "open"
ModeDenylist Mode = "denylist"
ModeAllowlist Mode = "allowlist"
)
type Policy struct {
Mode Mode
DeniedHostSuffixes []string
AllowedHostSuffixes []string
}
func LoadFromEnv() Policy {
mode := Mode(strings.ToLower(strings.TrimSpace(os.Getenv("CLAWMANAGER_EGRESS_LLM_POLICY"))))
if mode == "" {
mode = ModeDenylist
}
policy := Policy{
Mode: mode,
DeniedHostSuffixes: append(defaultDeniedHostSuffixes(),
splitCSV(os.Getenv("CLAWMANAGER_EGRESS_DENIED_SUFFIXES"))...),
AllowedHostSuffixes: append(defaultAllowedHostSuffixes(),
splitCSV(os.Getenv("CLAWMANAGER_EGRESS_ALLOWED_SUFFIXES"))...),
}
return policy
}
func defaultDeniedHostSuffixes() []string {
return []string{
"api.openai.com",
"openai.azure.com",
"api.anthropic.com",
"generativelanguage.googleapis.com",
"api.deepseek.com",
"api.moonshot.cn",
"open.bigmodel.cn",
"dashscope.aliyuncs.com",
}
}
func defaultAllowedHostSuffixes() []string {
return []string{
"github.com",
"registry-1.docker.io",
"pypi.org",
"npmjs.org",
"clawmanager-gateway",
"clawmanager-egress-proxy",
}
}
func (p Policy) AllowHost(host string) (bool, string) {
host = normalizeHost(host)
if host == "" {
return false, "empty host"
}
switch p.Mode {
case ModeOpen, "":
return true, ""
case ModeAllowlist:
if matchesAnySuffix(host, p.AllowedHostSuffixes) {
return true, ""
}
return false, "host not in allowlist"
default:
if matchesAnySuffix(host, p.DeniedHostSuffixes) {
return false, "llm provider host blocked"
}
return true, ""
}
}
func normalizeHost(host string) string {
host = strings.TrimSpace(strings.ToLower(host))
if host == "" {
return ""
}
if idx := strings.Index(host, ":"); idx >= 0 {
host = host[:idx]
}
return strings.Trim(host, ".")
}
func matchesAnySuffix(host string, suffixes []string) bool {
for _, suffix := range suffixes {
suffix = strings.TrimSpace(strings.ToLower(suffix))
if suffix == "" {
continue
}
if host == suffix || strings.HasSuffix(host, "."+suffix) {
return true
}
}
return false
}
func splitCSV(raw string) []string {
parts := strings.Split(raw, ",")
result := make([]string, 0, len(parts))
for _, part := range parts {
part = strings.TrimSpace(part)
if part != "" {
result = append(result, part)
}
}
return result
}
backend/internal/egresspolicy/policy_test.go
+33
View File
@@ -0,0 +1,33 @@
package egresspolicy
import "testing"
func TestPolicyDenylistBlocksOpenAI(t *testing.T) {
policy := Policy{
Mode: ModeDenylist,
DeniedHostSuffixes: defaultDeniedHostSuffixes(),
}
allowed, reason := policy.AllowHost("api.openai.com")
if allowed || reason == "" {
t.Fatalf("expected openai host to be blocked, allowed=%v reason=%q", allowed, reason)
}
}
func TestPolicyDenylistAllowsGitHub(t *testing.T) {
policy := Policy{
Mode: ModeDenylist,
DeniedHostSuffixes: defaultDeniedHostSuffixes(),
}
allowed, reason := policy.AllowHost("github.com")
if !allowed || reason != "" {
t.Fatalf("expected github host to be allowed, allowed=%v reason=%q", allowed, reason)
}
}
func TestPolicyOpenAllowsEverything(t *testing.T) {
policy := Policy{Mode: ModeOpen}
allowed, reason := policy.AllowHost("api.openai.com")
if !allowed || reason != "" {
t.Fatalf("expected open mode to allow host, allowed=%v reason=%q", allowed, reason)
}
}
backend/internal/handlers/agent_handler.go
+17 -1
View File
@@ -1,6 +1,7 @@
package handlers
import (
"fmt"
"net/http"
"strconv"
"strings"
@@ -198,6 +199,7 @@ func (h *AgentHandler) ReportSkillInventory(c *gin.Context) {
utils.HandleError(c, err)
return
}
h.skillService.CompletePendingSkillInventorySync(session.Instance.ID)
utils.Success(c, http.StatusOK, "Agent skill inventory reported successfully", nil)
}
@@ -225,12 +227,26 @@ func (h *AgentHandler) UploadSkillPackage(c *gin.Context) {
}
item, err := h.skillService.UploadAgentSkillPackage(c.Request.Context(), session.Instance.ID, req, fileHeader)
if err != nil {
utils.HandleError(c, err)
utils.HandleHubError(c, err)
return
}
utils.Success(c, http.StatusCreated, "Agent skill package uploaded successfully", item)
}
func (h *AgentHandler) DownloadSkillVersion(c *gin.Context) {
if _, ok := h.authenticateAgentSession(c); !ok {
return
}
content, fileName, err := h.skillService.DownloadSkillVersionByExternalID(c.Param("skillVersion"))
if err != nil {
utils.HandleError(c, err)
return
}
c.Header("Content-Type", "application/octet-stream")
c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", fileName))
c.Data(http.StatusOK, "application/octet-stream", content)
}
func (h *AgentHandler) authenticateAgentSession(c *gin.Context) (*services.AgentSession, bool) {
sessionToken := extractBearerToken(c.GetHeader("Authorization"))
if sessionToken == "" {
backend/internal/handlers/agent_handler_test.go
+25
View File
@@ -0,0 +1,25 @@
package handlers
import (
"net/http"
"net/http/httptest"
"testing"
"github.com/gin-gonic/gin"
)
func TestDownloadSkillVersionRequiresAgentSession(t *testing.T) {
gin.SetMode(gin.TestMode)
handler := NewAgentHandler(nil, nil, nil, nil, nil)
recorder := httptest.NewRecorder()
c, _ := gin.CreateTestContext(recorder)
c.Request = httptest.NewRequest(http.MethodGet, "/api/agent/skills/versions/skill-version-1/download", nil)
c.Params = gin.Params{{Key: "skillVersion", Value: "skill-version-1"}}
handler.DownloadSkillVersion(c)
if recorder.Code != http.StatusUnauthorized {
t.Fatalf("status = %d, want %d", recorder.Code, http.StatusUnauthorized)
}
}
backend/internal/handlers/ai_gateway_handler.go
+18 -2
View File
@@ -49,8 +49,9 @@ func (h *AIGatewayHandler) ChatCompletions(c *gin.Context) {
return
}
req.RawBody = rawBody
if req.SessionID == nil {
if sessionKey := strings.TrimSpace(c.GetHeader("x-openclaw-session-key")); sessionKey != "" {
if sessionKey := strings.TrimSpace(c.GetHeader("x-openclaw-session-key")); sessionKey != "" {
req.OpenClawSessionKey = &sessionKey
if req.SessionID == nil {
req.SessionID = &sessionKey
}
}
@@ -59,6 +60,13 @@ func (h *AIGatewayHandler) ChatCompletions(c *gin.Context) {
req.TraceID = &runID
}
}
gatewayAuthType, _ := c.Get("gatewayAuthType")
instanceType, _ := c.Get("instanceType")
aigateway.ApplyManagedInstanceSessionDefaults(
&req,
stringValue(gatewayAuthType),
stringValue(instanceType),
)
userID, exists := c.Get("userID")
if !exists {
@@ -158,3 +166,11 @@ func setInt64Metadata(c *gin.Context, field **int64, key string) bool {
}
return **field == value
}
func stringValue(raw interface{}) string {
value, ok := raw.(string)
if !ok {
return ""
}
return strings.TrimSpace(value)
}
backend/internal/handlers/ai_observability_handler.go
+57
View File
@@ -30,6 +30,15 @@ type CostQueryRequest struct {
Search string `form:"search"`
}
// SessionUsageOverviewQueryRequest binds admin session usage overview filters.
type SessionUsageOverviewQueryRequest struct {
Page int `form:"page,default=1"`
Limit int `form:"limit,default=20"`
Search string `form:"search"`
Since string `form:"since"`
Until string `form:"until"`
}
// NewAIObservabilityHandler creates a new observability handler.
func NewAIObservabilityHandler(service services.AIObservabilityService) *AIObservabilityHandler {
return &AIObservabilityHandler{service: service}
@@ -90,3 +99,51 @@ func (h *AIObservabilityHandler) GetCostOverview(c *gin.Context) {
utils.Success(c, http.StatusOK, "AI cost overview retrieved successfully", overview)
}
// GetLLMGovernanceOverview returns managed-runtime LLM governance summary for admins.
func (h *AIObservabilityHandler) GetLLMGovernanceOverview(c *gin.Context) {
overview, err := h.service.GetLLMGovernanceOverview()
if err != nil {
utils.HandleError(c, err)
return
}
utils.Success(c, http.StatusOK, "LLM governance overview retrieved successfully", overview)
}
// GetSessionUsageOverview returns cross-instance session token usage for admins.
func (h *AIObservabilityHandler) GetSessionUsageOverview(c *gin.Context) {
var req SessionUsageOverviewQueryRequest
if err := c.ShouldBindQuery(&req); err != nil {
utils.ValidationError(c, err)
return
}
since, err := parseOptionalRFC3339(req.Since)
if err != nil {
utils.Error(c, http.StatusBadRequest, "Invalid since timestamp")
return
}
until, parseUntilErr := parseOptionalRFC3339(req.Until)
if parseUntilErr != nil {
utils.Error(c, http.StatusBadRequest, "Invalid until timestamp")
return
}
if err := validateSessionUsageTimeRange(since, until); err != nil {
utils.Error(c, http.StatusBadRequest, err.Error())
return
}
overview, err := h.service.GetAdminSessionUsageOverview(services.InstanceSessionUsageOverviewQuery{
Page: req.Page,
Limit: req.Limit,
Search: req.Search,
Since: since,
Until: until,
})
if err != nil {
utils.HandleError(c, err)
return
}
utils.Success(c, http.StatusOK, "Session usage overview retrieved successfully", overview)
}
backend/internal/handlers/ai_observability_handler_test.go
+105
View File
@@ -0,0 +1,105 @@
package handlers
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"clawreef/internal/repository"
"clawreef/internal/services"
"github.com/gin-gonic/gin"
)
type stubAIObservabilityHandlerService struct {
overview *services.InstanceSessionUsageOverview
err error
lastQuery services.InstanceSessionUsageOverviewQuery
}
func (s *stubAIObservabilityHandlerService) ListAuditItems(services.AuditQuery) (*services.AuditListResult, error) {
return nil, nil
}
func (s *stubAIObservabilityHandlerService) GetTraceDetail(string) (*services.AuditTraceDetail, error) {
return nil, nil
}
func (s *stubAIObservabilityHandlerService) GetCostOverview(services.CostQuery) (*services.CostOverview, error) {
return nil, nil
}
func (s *stubAIObservabilityHandlerService) GetInstanceSessionUsage(int, services.InstanceSessionUsageQuery) (*services.InstanceSessionUsageResult, error) {
return nil, nil
}
func (s *stubAIObservabilityHandlerService) GetInstanceSessionUsageDetail(int, string, repository.SessionUsageFilter) (*services.InstanceSessionUsageDetail, error) {
return nil, nil
}
func (s *stubAIObservabilityHandlerService) GetInstanceLLMGovernanceStatus(int, map[string]interface{}) (*services.InstanceLLMGovernanceStatus, error) {
return nil, nil
}
func (s *stubAIObservabilityHandlerService) GetLLMGovernanceOverview() (*services.LLMGovernanceOverview, error) {
return nil, nil
}
func (s *stubAIObservabilityHandlerService) GetAdminSessionUsageOverview(query services.InstanceSessionUsageOverviewQuery) (*services.InstanceSessionUsageOverview, error) {
s.lastQuery = query
if s.err != nil {
return nil, s.err
}
return s.overview, nil
}
func TestGetSessionUsageOverviewReturns200(t *testing.T) {
gin.SetMode(gin.TestMode)
recorder := httptest.NewRecorder()
c, _ := gin.CreateTestContext(recorder)
c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/admin/session-usage/overview?page=1&limit=10&search=oc", nil)
service := &stubAIObservabilityHandlerService{
overview: &services.InstanceSessionUsageOverview{
Summary: services.InstanceSessionUsageSummary{Currency: "USD"},
Items: []services.InstanceSessionUsageOverviewItem{},
Total: 0,
Page: 1,
Limit: 10,
},
}
handler := NewAIObservabilityHandler(service)
handler.GetSessionUsageOverview(c)
if recorder.Code != http.StatusOK {
t.Fatalf("status = %d, want %d, body = %s", recorder.Code, http.StatusOK, recorder.Body.String())
}
if service.lastQuery.Search != "oc" {
t.Fatalf("expected search=oc, got %q", service.lastQuery.Search)
}
if !strings.Contains(recorder.Body.String(), "Session usage overview retrieved successfully") {
t.Fatalf("unexpected body: %s", recorder.Body.String())
}
}
func TestGetSessionUsageOverviewInvalidSince(t *testing.T) {
gin.SetMode(gin.TestMode)
recorder := httptest.NewRecorder()
c, _ := gin.CreateTestContext(recorder)
c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/admin/session-usage/overview?since=not-a-date", nil)
handler := NewAIObservabilityHandler(&stubAIObservabilityHandlerService{})
handler.GetSessionUsageOverview(c)
if recorder.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want %d, body = %s", recorder.Code, http.StatusBadRequest, recorder.Body.String())
}
}
func TestGetSessionUsageOverviewRejectsUntilBeforeSince(t *testing.T) {
gin.SetMode(gin.TestMode)
recorder := httptest.NewRecorder()
c, _ := gin.CreateTestContext(recorder)
c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/admin/session-usage/overview?since=2026-07-10T00:00:00Z&until=2026-07-01T00:00:00Z", nil)
handler := NewAIObservabilityHandler(&stubAIObservabilityHandlerService{})
handler.GetSessionUsageOverview(c)
if recorder.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want %d, body = %s", recorder.Code, http.StatusBadRequest, recorder.Body.String())
}
}
backend/internal/handlers/egress_proxy_handler.go
+59 -1
View File
@@ -1,22 +1,31 @@
package handlers
import (
"fmt"
"io"
"log"
"net"
"net/http"
"strconv"
"strings"
"time"
"clawreef/internal/egresspolicy"
"clawreef/internal/models"
"clawreef/internal/services"
"github.com/gin-gonic/gin"
)
// EgressProxyHandler provides a minimal forward proxy for ordinary HTTP/HTTPS traffic.
type EgressProxyHandler struct {
transport *http.Transport
policy egresspolicy.Policy
audit services.AuditEventService
}
// NewEgressProxyHandler creates a new egress proxy handler.
func NewEgressProxyHandler() *EgressProxyHandler {
func NewEgressProxyHandler(audit services.AuditEventService) *EgressProxyHandler {
return &EgressProxyHandler{
transport: &http.Transport{
Proxy: nil,
@@ -27,6 +36,8 @@ func NewEgressProxyHandler() *EgressProxyHandler {
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
},
policy: egresspolicy.LoadFromEnv(),
audit: audit,
}
}
@@ -42,6 +53,12 @@ func (h *EgressProxyHandler) Handle(c *gin.Context) {
return
}
if allowed, reason := h.policy.AllowHost(c.Request.URL.Host); !allowed {
h.recordBlockedEgress(c, c.Request.URL.Host, reason)
c.String(http.StatusForbidden, "egress blocked: %s (%s)", c.Request.URL.Host, reason)
return
}
outReq := c.Request.Clone(c.Request.Context())
outReq.RequestURI = ""
removeHopHeaders(outReq.Header)
@@ -66,6 +83,12 @@ func (h *EgressProxyHandler) handleConnect(c *gin.Context) {
return
}
if allowed, reason := h.policy.AllowHost(target); !allowed {
h.recordBlockedEgress(c, target, reason)
c.String(http.StatusForbidden, "egress blocked: %s (%s)", target, reason)
return
}
upstreamConn, err := net.DialTimeout("tcp", target, 30*time.Second)
if err != nil {
c.String(http.StatusBadGateway, "proxy connect error: %v", err)
@@ -91,6 +114,41 @@ func (h *EgressProxyHandler) handleConnect(c *gin.Context) {
go tunnelConns(clientConn, upstreamConn)
}
func (h *EgressProxyHandler) recordBlockedEgress(c *gin.Context, host, reason string) {
if h.audit == nil {
return
}
instanceID := resolveEgressInstanceID(c)
remoteAddr := strings.TrimSpace(c.Request.RemoteAddr)
message := fmt.Sprintf("Blocked egress to %s (%s) from %s", host, reason, remoteAddr)
if err := h.audit.RecordEvent(&models.AuditEvent{
TraceID: fmt.Sprintf("egress_%d", time.Now().UnixNano()),
InstanceID: instanceID,
EventType: "egress.llm.blocked",
TrafficClass: models.TrafficClassGenericEgress,
Severity: models.AuditSeverityWarn,
Message: message,
}); err != nil {
log.Printf("failed to record egress block audit event: %v", err)
}
}
func resolveEgressInstanceID(c *gin.Context) *int {
for _, headerName := range []string{
"X-ClawManager-Instance-Id",
"X-ClawManager-Egress-Instance-Id",
} {
raw := strings.TrimSpace(c.GetHeader(headerName))
if raw == "" {
continue
}
if parsed, err := strconv.Atoi(raw); err == nil && parsed > 0 {
return &parsed
}
}
return nil
}
func tunnelConns(dst net.Conn, src net.Conn) {
defer dst.Close()
defer src.Close()
backend/internal/handlers/egress_proxy_handler_test.go
+82
View File
@@ -0,0 +1,82 @@
package handlers
import (
"net/http"
"net/http/httptest"
"testing"
"clawreef/internal/egresspolicy"
"clawreef/internal/models"
"github.com/gin-gonic/gin"
)
type stubEgressAuditService struct {
events []*models.AuditEvent
}
func (s *stubEgressAuditService) RecordEvent(event *models.AuditEvent) error {
s.events = append(s.events, event)
return nil
}
func (s *stubEgressAuditService) ListEventsByTraceID(string) ([]models.AuditEvent, error) {
return nil, nil
}
func TestEgressProxyHandlerBlocksDeniedConnectHost(t *testing.T) {
gin.SetMode(gin.TestMode)
audit := &stubEgressAuditService{}
handler := &EgressProxyHandler{
policy: egresspolicy.Policy{
Mode: egresspolicy.ModeDenylist,
DeniedHostSuffixes: []string{"api.openai.com"},
},
audit: audit,
}
recorder := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(recorder)
ctx.Request = httptest.NewRequest(http.MethodConnect, "https://api.openai.com:443", nil)
ctx.Request.Host = "api.openai.com:443"
ctx.Request.Header.Set("X-ClawManager-Instance-Id", "42")
handler.handleConnect(ctx)
if recorder.Code != http.StatusForbidden {
t.Fatalf("expected 403, got %d", recorder.Code)
}
if len(audit.events) != 1 || audit.events[0].EventType != "egress.llm.blocked" {
t.Fatalf("expected egress audit event, got %+v", audit.events)
}
if audit.events[0].InstanceID == nil || *audit.events[0].InstanceID != 42 {
t.Fatalf("expected instance id 42 on egress audit event, got %+v", audit.events[0].InstanceID)
}
}
func TestEgressProxyHandlerAcceptsEgressInstanceHeaderAlias(t *testing.T) {
gin.SetMode(gin.TestMode)
audit := &stubEgressAuditService{}
handler := &EgressProxyHandler{
policy: egresspolicy.Policy{
Mode: egresspolicy.ModeDenylist,
DeniedHostSuffixes: []string{"api.openai.com"},
},
audit: audit,
}
recorder := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(recorder)
ctx.Request = httptest.NewRequest(http.MethodConnect, "https://api.openai.com:443", nil)
ctx.Request.Host = "api.openai.com:443"
ctx.Request.Header.Set("X-ClawManager-Egress-Instance-Id", "77")
handler.handleConnect(ctx)
if recorder.Code != http.StatusForbidden {
t.Fatalf("expected 403, got %d", recorder.Code)
}
if audit.events[0].InstanceID == nil || *audit.events[0].InstanceID != 77 {
t.Fatalf("expected instance id 77 on egress audit event, got %+v", audit.events[0].InstanceID)
}
}
backend/internal/handlers/instance_handler.go
+253 -14
View File
@@ -13,6 +13,7 @@ import (
"time"
"clawreef/internal/models"
"clawreef/internal/repository"
"clawreef/internal/services"
"clawreef/internal/utils"
@@ -135,10 +136,11 @@ type InstanceHandler struct {
openClawConfigService services.OpenClawConfigService
skillService services.SkillService
externalAccessService services.InstanceExternalAccessService
aiObservabilityService services.AIObservabilityService
}
// NewInstanceHandler creates a new instance handler
func NewInstanceHandler(instanceService services.InstanceService, instanceAgentService services.InstanceAgentService, runtimeStatusService services.InstanceRuntimeStatusService, instanceCommandService services.InstanceCommandService, instanceConfigRevisionService services.InstanceConfigRevisionService, openClawConfigService services.OpenClawConfigService, skillService services.SkillService, externalAccessService services.InstanceExternalAccessService, proxyOptions ...services.InstanceProxyServiceOption) *InstanceHandler {
func NewInstanceHandler(instanceService services.InstanceService, instanceAgentService services.InstanceAgentService, runtimeStatusService services.InstanceRuntimeStatusService, instanceCommandService services.InstanceCommandService, instanceConfigRevisionService services.InstanceConfigRevisionService, openClawConfigService services.OpenClawConfigService, skillService services.SkillService, externalAccessService services.InstanceExternalAccessService, aiObservabilityService services.AIObservabilityService, proxyOptions ...services.InstanceProxyServiceOption) *InstanceHandler {
accessService := services.NewInstanceAccessService()
return &InstanceHandler{
instanceService: instanceService,
@@ -153,6 +155,7 @@ func NewInstanceHandler(instanceService services.InstanceService, instanceAgentS
openClawConfigService: openClawConfigService,
skillService: skillService,
externalAccessService: externalAccessService,
aiObservabilityService: aiObservabilityService,
}
}
@@ -164,9 +167,10 @@ func (h *InstanceHandler) Shutdown() {
}
type InstanceRuntimeDetailsResponse struct {
Runtime *services.InstanceRuntimeStatusPayload `json:"runtime,omitempty"`
Agent *services.InstanceAgentPayload `json:"agent,omitempty"`
Commands []services.InstanceCommandPayload `json:"commands,omitempty"`
Runtime *services.InstanceRuntimeStatusPayload `json:"runtime,omitempty"`
Agent *services.InstanceAgentPayload `json:"agent,omitempty"`
Commands []services.InstanceCommandPayload `json:"commands,omitempty"`
LLMGovernance *services.InstanceLLMGovernanceStatus `json:"llm_governance,omitempty"`
}
type CreateRuntimeCommandRequest struct {
@@ -364,9 +368,10 @@ func (h *InstanceHandler) CreateInstance(c *gin.Context) {
return
}
userRole, _ := c.Get("userRole")
for _, skillID := range skillIDs {
if _, err := h.skillService.AttachSkillToInstance(instance.ID, skillID); err != nil {
utils.HandleError(c, err)
if _, err := h.skillService.AttachSkillToInstance(userID.(int), userRole.(string), instance.ID, skillID); err != nil {
utils.HandleHubError(c, err)
return
}
}
@@ -400,6 +405,7 @@ func instanceCreateRequestToService(req CreateInstanceRequest) services.CreateIn
func (h *InstanceHandler) BatchCreateLiteInstances(c *gin.Context) {
userID, _ := c.Get("userID")
userRole, _ := c.Get("userRole")
var req BatchCreateLiteInstancesRequest
if err := c.ShouldBindJSON(&req); err != nil {
@@ -449,7 +455,7 @@ func (h *InstanceHandler) BatchCreateLiteInstances(c *gin.Context) {
}
attachFailed := false
for _, skillID := range skillIDs {
if _, err := h.skillService.AttachSkillToInstance(instance.ID, skillID); err != nil {
if _, err := h.skillService.AttachSkillToInstance(userID.(int), userRole.(string), instance.ID, skillID); err != nil {
result.Status = "failed"
result.Error = err.Error()
response.Failed++
@@ -986,7 +992,7 @@ func availabilityForInstanceStatus(status string) string {
}
func (h *InstanceHandler) GetRuntimeDetails(c *gin.Context) {
id, _, ok := h.resolveOwnedInstance(c)
id, instance, ok := h.resolveOwnedInstance(c)
if !ok {
return
}
@@ -1007,11 +1013,128 @@ func (h *InstanceHandler) GetRuntimeDetails(c *gin.Context) {
return
}
utils.Success(c, http.StatusOK, "Instance runtime details retrieved successfully", InstanceRuntimeDetailsResponse{
response := InstanceRuntimeDetailsResponse{
Runtime: runtime,
Agent: agent,
Commands: commands,
}
if h.aiObservabilityService != nil && instance != nil &&
(instance.Type == "openclaw" || instance.Type == "hermes") {
var systemInfo map[string]interface{}
if runtime != nil {
systemInfo = runtime.SystemInfo
}
if governance, govErr := h.aiObservabilityService.GetInstanceLLMGovernanceStatus(id, systemInfo); govErr == nil {
response.LLMGovernance = governance
}
}
utils.Success(c, http.StatusOK, "Instance runtime details retrieved successfully", response)
}
type SessionUsageQueryRequest struct {
Page int `form:"page,default=1"`
Limit int `form:"limit,default=20"`
Search string `form:"search"`
Since string `form:"since"`
Until string `form:"until"`
}
type SessionUsageDetailQueryRequest struct {
SessionID string `form:"session_id" binding:"required"`
Since string `form:"since"`
Until string `form:"until"`
}
func (h *InstanceHandler) GetInstanceSessionUsage(c *gin.Context) {
id, _, ok := h.resolveOwnedInstance(c)
if !ok {
return
}
if h.aiObservabilityService == nil {
utils.Error(c, http.StatusInternalServerError, "Session usage service is not configured")
return
}
var req SessionUsageQueryRequest
if err := c.ShouldBindQuery(&req); err != nil {
utils.ValidationError(c, err)
return
}
since, err := parseOptionalRFC3339(req.Since)
if err != nil {
utils.Error(c, http.StatusBadRequest, "Invalid since timestamp")
return
}
until, parseUntilErr := parseOptionalRFC3339(req.Until)
if parseUntilErr != nil {
utils.Error(c, http.StatusBadRequest, "Invalid until timestamp")
return
}
if err := validateSessionUsageTimeRange(since, until); err != nil {
utils.Error(c, http.StatusBadRequest, err.Error())
return
}
result, err := h.aiObservabilityService.GetInstanceSessionUsage(id, services.InstanceSessionUsageQuery{
Page: req.Page,
Limit: req.Limit,
Search: req.Search,
Since: since,
Until: until,
})
if err != nil {
utils.HandleError(c, err)
return
}
utils.Success(c, http.StatusOK, "Instance session usage retrieved successfully", result)
}
func (h *InstanceHandler) GetInstanceSessionUsageDetail(c *gin.Context) {
id, _, ok := h.resolveOwnedInstance(c)
if !ok {
return
}
if h.aiObservabilityService == nil {
utils.Error(c, http.StatusInternalServerError, "Session usage service is not configured")
return
}
var req SessionUsageDetailQueryRequest
if err := c.ShouldBindQuery(&req); err != nil {
utils.ValidationError(c, err)
return
}
since, err := parseOptionalRFC3339(req.Since)
if err != nil {
utils.Error(c, http.StatusBadRequest, "Invalid since timestamp")
return
}
until, parseUntilErr := parseOptionalRFC3339(req.Until)
if parseUntilErr != nil {
utils.Error(c, http.StatusBadRequest, "Invalid until timestamp")
return
}
if err := validateSessionUsageTimeRange(since, until); err != nil {
utils.Error(c, http.StatusBadRequest, err.Error())
return
}
detail, err := h.aiObservabilityService.GetInstanceSessionUsageDetail(id, req.SessionID, repository.SessionUsageFilter{
Since: since,
Until: until,
})
if err != nil {
if strings.Contains(strings.ToLower(err.Error()), "not found") {
utils.Error(c, http.StatusNotFound, "Session usage not found")
return
}
utils.HandleError(c, err)
return
}
utils.Success(c, http.StatusOK, "Instance session usage detail retrieved successfully", detail)
}
func (h *InstanceHandler) CreateRuntimeCommand(c *gin.Context) {
@@ -1143,25 +1266,37 @@ func (h *InstanceHandler) resolveOwnedInstance(c *gin.Context) (int, *models.Ins
utils.Error(c, http.StatusBadRequest, "Invalid instance ID")
return 0, nil, false
}
instance, ok := h.authorizeInstanceAccess(c, id)
if !ok {
return 0, nil, false
}
return id, instance, true
}
instance, err := h.instanceService.GetByID(id)
func (h *InstanceHandler) authorizeInstanceAccess(c *gin.Context, instanceID int) (*models.Instance, bool) {
if instanceID <= 0 {
utils.Error(c, http.StatusBadRequest, "Invalid instance ID")
return nil, false
}
instance, err := h.instanceService.GetByID(instanceID)
if err != nil {
utils.HandleError(c, err)
return 0, nil, false
return nil, false
}
if instance == nil {
utils.Error(c, http.StatusNotFound, "Instance not found")
return 0, nil, false
return nil, false
}
userID, _ := c.Get("userID")
userRole, _ := c.Get("userRole")
if userRole != "admin" && instance.UserID != userID.(int) {
utils.Error(c, http.StatusForbidden, "Access denied")
return 0, nil, false
return nil, false
}
return id, instance, true
return instance, true
}
// GenerateAccessToken generates an access token for an instance
@@ -1634,6 +1769,110 @@ func (h *InstanceHandler) ImportHermes(c *gin.Context) {
utils.Success(c, http.StatusOK, "Hermes workspace imported successfully", nil)
}
func (h *InstanceHandler) RefreshInstanceSkills(c *gin.Context) {
instance, ok := h.requireOwnedInstance(c)
if !ok {
return
}
if instance.Type != "openclaw" && instance.Type != "hermes" {
utils.Error(c, http.StatusBadRequest, "skill inventory sync is only available for openclaw and hermes instances")
return
}
userID, _ := c.Get("userID")
issuedBy := userID.(int)
command, err := h.createSkillInventorySyncCommand(instance, issuedBy)
if err != nil {
utils.HandleError(c, err)
return
}
utils.Success(c, http.StatusOK, "Skill inventory sync requested", command)
}
func (h *InstanceHandler) createSkillInventorySyncCommand(instance *models.Instance, issuedBy int) (*services.InstanceCommandPayload, error) {
command, err := h.instanceCommandService.Create(instance.ID, &issuedBy, services.CreateInstanceCommandRequest{
CommandType: services.InstanceCommandTypeSyncSkillInventory,
Payload: map[string]interface{}{
"trigger": "manual",
"mode": "full",
},
IdempotencyKey: fmt.Sprintf("sync-skill-inventory-%d-%d", instance.ID, time.Now().Unix()),
TimeoutSeconds: 300,
})
if err != nil {
return nil, err
}
if services.IsLiteRuntimeInstance(instance) || services.SupportsServerWorkspaceSkillScan(instance) {
if err := h.skillService.RequestLiteSkillInventorySync(instance.ID); err != nil {
return nil, err
}
}
return command, nil
}
func (h *InstanceHandler) ImportInstanceSkillToLibrary(c *gin.Context) {
instance, ok := h.requireOwnedInstance(c)
if !ok {
return
}
skillID, err := strconv.Atoi(c.Param("skillId"))
if err != nil {
utils.Error(c, http.StatusBadRequest, "invalid skill ID")
return
}
userID, _ := c.Get("userID")
userRole, _ := c.Get("userRole")
item, err := h.skillService.ImportInstanceSkillToLibrary(userID.(int), userRole.(string), instance.ID, skillID)
if err != nil {
utils.HandleHubError(c, err)
return
}
utils.Success(c, http.StatusOK, "Skill imported to library successfully", item)
}
func (h *InstanceHandler) RetrySkillPackageCollect(c *gin.Context) {
instance, ok := h.requireOwnedInstance(c)
if !ok {
return
}
skillID, err := strconv.Atoi(c.Param("skillId"))
if err != nil {
utils.Error(c, http.StatusBadRequest, "invalid skill ID")
return
}
userID, _ := c.Get("userID")
userRole, _ := c.Get("userRole")
if err := h.skillService.RetrySkillPackageCollection(userID.(int), userRole.(string), instance.ID, skillID); err != nil {
utils.HandleHubError(c, err)
return
}
utils.Success(c, http.StatusAccepted, "Skill package collection requested", gin.H{"status": "pending"})
}
func (h *InstanceHandler) PublishInstanceSkillToHub(c *gin.Context) {
instance, ok := h.requireOwnedInstance(c)
if !ok {
return
}
skillID, err := strconv.Atoi(c.Param("skillId"))
if err != nil {
utils.Error(c, http.StatusBadRequest, "invalid skill ID")
return
}
var req services.PublishSkillHubRequest
if err := c.ShouldBindJSON(&req); err != nil {
utils.ValidationError(c, err)
return
}
userID, _ := c.Get("userID")
userRole, _ := c.Get("userRole")
item, err := h.skillService.PublishFromInstance(userID.(int), userRole.(string), instance.ID, skillID, req.TagIDs)
if err != nil {
utils.HandleHubError(c, err)
return
}
utils.Success(c, http.StatusOK, "Skill published to hub successfully", item)
}
func (h *InstanceHandler) requireOwnedInstance(c *gin.Context) (*models.Instance, bool) {
idStr := c.Param("id")
id, err := strconv.Atoi(idStr)
backend/internal/handlers/instance_handler_test.go
+165
View File
@@ -1,6 +1,7 @@
package handlers
import (
"fmt"
"net/http"
"net/http/httptest"
"strings"
@@ -8,6 +9,7 @@ import (
"time"
"clawreef/internal/models"
"clawreef/internal/repository"
"clawreef/internal/services"
"github.com/gin-gonic/gin"
@@ -323,3 +325,166 @@ func TestBatchDeleteLiteInstancesRejectsProInstance(t *testing.T) {
t.Fatalf("response did not explain lite-only rejection: %s", recorder.Body.String())
}
}
type stubSessionUsageObservabilityService struct {
usage *services.InstanceSessionUsageResult
detail *services.InstanceSessionUsageDetail
err error
}
func (s *stubSessionUsageObservabilityService) ListAuditItems(services.AuditQuery) (*services.AuditListResult, error) {
return nil, nil
}
func (s *stubSessionUsageObservabilityService) GetTraceDetail(string) (*services.AuditTraceDetail, error) {
return nil, nil
}
func (s *stubSessionUsageObservabilityService) GetCostOverview(services.CostQuery) (*services.CostOverview, error) {
return nil, nil
}
func (s *stubSessionUsageObservabilityService) GetInstanceSessionUsage(int, services.InstanceSessionUsageQuery) (*services.InstanceSessionUsageResult, error) {
if s.err != nil {
return nil, s.err
}
return s.usage, nil
}
func (s *stubSessionUsageObservabilityService) GetInstanceSessionUsageDetail(int, string, repository.SessionUsageFilter) (*services.InstanceSessionUsageDetail, error) {
if s.err != nil {
return nil, s.err
}
if s.detail == nil {
return nil, fmt.Errorf("session usage not found")
}
return s.detail, nil
}
func (s *stubSessionUsageObservabilityService) GetInstanceLLMGovernanceStatus(int, map[string]interface{}) (*services.InstanceLLMGovernanceStatus, error) {
return nil, nil
}
func (s *stubSessionUsageObservabilityService) GetLLMGovernanceOverview() (*services.LLMGovernanceOverview, error) {
return nil, nil
}
func (s *stubSessionUsageObservabilityService) GetAdminSessionUsageOverview(services.InstanceSessionUsageOverviewQuery) (*services.InstanceSessionUsageOverview, error) {
return nil, nil
}
func TestGetInstanceSessionUsageReturns200(t *testing.T) {
gin.SetMode(gin.TestMode)
recorder := httptest.NewRecorder()
c, _ := gin.CreateTestContext(recorder)
c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/instances/9/session-usage?page=1&limit=10", nil)
c.Params = gin.Params{{Key: "id", Value: "9"}}
c.Set("userID", 7)
c.Set("userRole", "user")
handler := &InstanceHandler{
instanceService: &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{
9: {ID: 9, UserID: 7, Name: "openclaw-lite", Type: "openclaw"},
}},
aiObservabilityService: &stubSessionUsageObservabilityService{
usage: &services.InstanceSessionUsageResult{
Summary: services.InstanceSessionUsageSummary{Currency: "USD"},
Items: []services.InstanceSessionUsageItem{},
},
},
}
handler.GetInstanceSessionUsage(c)
if recorder.Code != http.StatusOK {
t.Fatalf("status = %d, want %d, body = %s", recorder.Code, http.StatusOK, recorder.Body.String())
}
if !strings.Contains(recorder.Body.String(), "Instance session usage retrieved successfully") {
t.Fatalf("unexpected body: %s", recorder.Body.String())
}
}
func TestGetInstanceSessionUsageInvalidSince(t *testing.T) {
gin.SetMode(gin.TestMode)
recorder := httptest.NewRecorder()
c, _ := gin.CreateTestContext(recorder)
c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/instances/9/session-usage?since=bad-timestamp", nil)
c.Params = gin.Params{{Key: "id", Value: "9"}}
c.Set("userID", 7)
c.Set("userRole", "user")
handler := &InstanceHandler{
instanceService: &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{
9: {ID: 9, UserID: 7, Name: "openclaw-lite", Type: "openclaw"},
}},
aiObservabilityService: &stubSessionUsageObservabilityService{},
}
handler.GetInstanceSessionUsage(c)
if recorder.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want %d, body = %s", recorder.Code, http.StatusBadRequest, recorder.Body.String())
}
}
func TestGetInstanceSessionUsageRejectsUntilBeforeSince(t *testing.T) {
gin.SetMode(gin.TestMode)
recorder := httptest.NewRecorder()
c, _ := gin.CreateTestContext(recorder)
c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/instances/9/session-usage?since=2026-07-10T00:00:00Z&until=2026-07-01T00:00:00Z", nil)
c.Params = gin.Params{{Key: "id", Value: "9"}}
c.Set("userID", 7)
c.Set("userRole", "user")
handler := &InstanceHandler{
instanceService: &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{
9: {ID: 9, UserID: 7, Name: "openclaw-lite", Type: "openclaw"},
}},
aiObservabilityService: &stubSessionUsageObservabilityService{},
}
handler.GetInstanceSessionUsage(c)
if recorder.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want %d, body = %s", recorder.Code, http.StatusBadRequest, recorder.Body.String())
}
}
func TestGetInstanceSessionUsageDetailRequiresSessionID(t *testing.T) {
gin.SetMode(gin.TestMode)
recorder := httptest.NewRecorder()
c, _ := gin.CreateTestContext(recorder)
c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/instances/9/session-usage/detail", nil)
c.Params = gin.Params{{Key: "id", Value: "9"}}
c.Set("userID", 7)
c.Set("userRole", "user")
handler := &InstanceHandler{
instanceService: &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{
9: {ID: 9, UserID: 7, Name: "openclaw-lite", Type: "openclaw"},
}},
aiObservabilityService: &stubSessionUsageObservabilityService{},
}
handler.GetInstanceSessionUsageDetail(c)
if recorder.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want %d, body = %s", recorder.Code, http.StatusBadRequest, recorder.Body.String())
}
}
func TestGetInstanceSessionUsageDetailNotFound(t *testing.T) {
gin.SetMode(gin.TestMode)
recorder := httptest.NewRecorder()
c, _ := gin.CreateTestContext(recorder)
c.Request = httptest.NewRequest(http.MethodGet, "/api/v1/instances/9/session-usage/detail?session_id=missing", nil)
c.Params = gin.Params{{Key: "id", Value: "9"}}
c.Set("userID", 7)
c.Set("userRole", "user")
handler := &InstanceHandler{
instanceService: &fakeWorkspaceHandlerInstanceService{instances: map[int]*models.Instance{
9: {ID: 9, UserID: 7, Name: "openclaw-lite", Type: "openclaw"},
}},
aiObservabilityService: &stubSessionUsageObservabilityService{},
}
handler.GetInstanceSessionUsageDetail(c)
if recorder.Code != http.StatusNotFound {
t.Fatalf("status = %d, want %d, body = %s", recorder.Code, http.StatusNotFound, recorder.Body.String())
}
}
backend/internal/handlers/runtime_agent_handler.go
+9 -1
View File
@@ -28,6 +28,7 @@ type RuntimeAgentHandler struct {
bindingRepo repository.InstanceRuntimeBindingRepository
instanceRepo repository.InstanceRepository
events runtimeEventPublisher
skillService services.SkillService
}
type runtimeAgentPodIdentity struct {
@@ -89,13 +90,14 @@ type runtimeAgentGatewayReport struct {
HealthAt *time.Time `json:"health_at,omitempty"`
}
func NewRuntimeAgentHandler(cfg config.RuntimePoolConfig, podRepo repository.RuntimePodRepository, bindingRepo repository.InstanceRuntimeBindingRepository, instanceRepo repository.InstanceRepository, events runtimeEventPublisher) *RuntimeAgentHandler {
func NewRuntimeAgentHandler(cfg config.RuntimePoolConfig, podRepo repository.RuntimePodRepository, bindingRepo repository.InstanceRuntimeBindingRepository, instanceRepo repository.InstanceRepository, events runtimeEventPublisher, skillService services.SkillService) *RuntimeAgentHandler {
return &RuntimeAgentHandler{
cfg: cfg,
podRepo: podRepo,
bindingRepo: bindingRepo,
instanceRepo: instanceRepo,
events: events,
skillService: skillService,
}
}
@@ -351,6 +353,12 @@ func (h *RuntimeAgentHandler) ReportSkills(c *gin.Context) {
utils.ValidationError(c, err)
return
}
if h.skillService != nil {
if err := h.skillService.SyncRuntimeAgentSkillsReport(payload); err != nil {
utils.HandleError(c, err)
return
}
}
h.publish(c.Request.Context(), "runtime_agent_skills_reported", payload)
utils.Success(c, http.StatusOK, "Runtime agent skills report accepted", nil)
}
backend/internal/handlers/runtime_agent_handler_test.go
+8 -8
View File
@@ -19,7 +19,7 @@ import (
func TestRuntimeAgentHandlerRejectsInvalidToken(t *testing.T) {
gin.SetMode(gin.TestMode)
podRepo := &runtimeAgentHandlerPodRepo{}
handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, podRepo, &runtimeAgentHandlerBindingRepo{}, nil, &runtimeAgentHandlerEvents{})
handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, podRepo, &runtimeAgentHandlerBindingRepo{}, nil, &runtimeAgentHandlerEvents{}, nil)
router := gin.New()
router.POST("/api/v1/runtime-agent/metrics/report", handler.ReportMetrics)
@@ -44,7 +44,7 @@ func TestRuntimeAgentHandlerRegisterUsesConfiguredCapacity(t *testing.T) {
handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{
AgentReportToken: "secret",
MaxGatewaysPerPod: 33,
}, podRepo, &runtimeAgentHandlerBindingRepo{}, nil, events)
}, podRepo, &runtimeAgentHandlerBindingRepo{}, nil, events, nil)
router := gin.New()
router.POST("/api/v1/runtime-agent/register", handler.Register)
@@ -95,7 +95,7 @@ func TestRuntimeAgentHandlerHeartbeatUsesConfiguredCapacity(t *testing.T) {
handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{
AgentReportToken: "secret",
MaxGatewaysPerPod: 44,
}, podRepo, &runtimeAgentHandlerBindingRepo{}, nil, events)
}, podRepo, &runtimeAgentHandlerBindingRepo{}, nil, events, nil)
router := gin.New()
router.POST("/api/v1/runtime-agent/heartbeat", handler.Heartbeat)
@@ -134,7 +134,7 @@ func TestRuntimeAgentHandlerMetricsReportUpdatesPodAndPublishesEvent(t *testing.
gin.SetMode(gin.TestMode)
podRepo := &runtimeAgentHandlerPodRepo{}
events := &runtimeAgentHandlerEvents{}
handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, podRepo, &runtimeAgentHandlerBindingRepo{}, nil, events)
handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, podRepo, &runtimeAgentHandlerBindingRepo{}, nil, events, nil)
router := gin.New()
router.POST("/api/v1/runtime-agent/metrics/report", handler.ReportMetrics)
@@ -187,7 +187,7 @@ func TestRuntimeAgentHandlerGatewayReportOnlyUpdatesCurrentPodBinding(t *testing
12: {InstanceID: 12, RuntimePodID: 9, Generation: 3},
},
}
handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, &runtimeAgentHandlerPodRepo{}, bindingRepo, nil, &runtimeAgentHandlerEvents{})
handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, &runtimeAgentHandlerPodRepo{}, bindingRepo, nil, &runtimeAgentHandlerEvents{}, nil)
router := gin.New()
router.POST("/api/v1/runtime-agent/gateways/report", handler.ReportGateways)
@@ -228,7 +228,7 @@ func TestRuntimeAgentHandlerGatewayReportSyncsInstanceRuntimeState(t *testing.T)
},
}
instanceRepo := &runtimeAgentHandlerInstanceRepo{}
handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, &runtimeAgentHandlerPodRepo{}, bindingRepo, instanceRepo, &runtimeAgentHandlerEvents{})
handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret"}, &runtimeAgentHandlerPodRepo{}, bindingRepo, instanceRepo, &runtimeAgentHandlerEvents{}, nil)
router := gin.New()
router.POST("/api/v1/runtime-agent/gateways/report", handler.ReportGateways)
@@ -274,7 +274,7 @@ func TestRuntimeAgentHandlerGatewayReportDeletesMissingCurrentPodBinding(t *test
12: {InstanceID: 12, RuntimePodID: 9, Generation: 2, State: "error"},
},
}
handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret", HeartbeatTimeout: 10 * time.Second}, &runtimeAgentHandlerPodRepo{}, bindingRepo, nil, &runtimeAgentHandlerEvents{})
handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret", HeartbeatTimeout: 10 * time.Second}, &runtimeAgentHandlerPodRepo{}, bindingRepo, nil, &runtimeAgentHandlerEvents{}, nil)
router := gin.New()
router.POST("/api/v1/runtime-agent/gateways/report", handler.ReportGateways)
@@ -304,7 +304,7 @@ func TestRuntimeAgentHandlerGatewayReportDoesNotDeleteFreshBindingFromFirstEmpty
10: {InstanceID: 10, RuntimePodID: 9, Generation: 3, State: "running", LastHealthAt: &lastHealthAt},
},
}
handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret", HeartbeatTimeout: 10 * time.Second}, &runtimeAgentHandlerPodRepo{}, bindingRepo, nil, &runtimeAgentHandlerEvents{})
handler := NewRuntimeAgentHandler(config.RuntimePoolConfig{AgentReportToken: "secret", HeartbeatTimeout: 10 * time.Second}, &runtimeAgentHandlerPodRepo{}, bindingRepo, nil, &runtimeAgentHandlerEvents{}, nil)
router := gin.New()
router.POST("/api/v1/runtime-agent/gateways/report", handler.ReportGateways)
backend/internal/handlers/runtime_pool_handler_test.go
+3
View File
@@ -536,3 +536,6 @@ func (c *runtimePoolHandlerAgentClient) DeleteGateway(ctx context.Context, endpo
return nil
}
func (c *runtimePoolHandlerAgentClient) Drain(ctx context.Context, endpoint string) error { return nil }
func (c *runtimePoolHandlerAgentClient) ResyncInstanceSkills(ctx context.Context, endpoint string, instanceID int, mode string) error {
return nil
}
backend/internal/handlers/session_usage_query.go
+27
View File
@@ -0,0 +1,27 @@
package handlers
import (
"fmt"
"strings"
"time"
)
func parseOptionalRFC3339(value string) (*time.Time, error) {
value = strings.TrimSpace(value)
if value == "" {
return nil, nil
}
parsed, err := time.Parse(time.RFC3339, value)
if err != nil {
return nil, err
}
utc := parsed.UTC()
return &utc, nil
}
func validateSessionUsageTimeRange(since, until *time.Time) error {
if since != nil && until != nil && !until.After(*since) {
return fmt.Errorf("until must be after since")
}
return nil
}
backend/internal/handlers/session_usage_query_test.go
+33
View File
@@ -0,0 +1,33 @@
package handlers
import (
"testing"
"time"
)
func TestValidateSessionUsageTimeRange(t *testing.T) {
since := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
until := time.Date(2026, 7, 2, 0, 0, 0, 0, time.UTC)
if err := validateSessionUsageTimeRange(&since, &until); err != nil {
t.Fatalf("expected valid range, got %v", err)
}
if err := validateSessionUsageTimeRange(&since, &since); err == nil {
t.Fatalf("expected equal since/until to be rejected")
}
if err := validateSessionUsageTimeRange(nil, &until); err != nil {
t.Fatalf("expected open-ended range, got %v", err)
}
}
func TestParseOptionalRFC3339(t *testing.T) {
if _, err := parseOptionalRFC3339(""); err != nil {
t.Fatalf("empty value should be allowed: %v", err)
}
parsed, err := parseOptionalRFC3339("2026-07-01T00:00:00Z")
if err != nil || parsed == nil {
t.Fatalf("expected parsed timestamp, got %v err=%v", parsed, err)
}
if _, err := parseOptionalRFC3339("not-a-date"); err == nil {
t.Fatalf("expected invalid timestamp error")
}
}
backend/internal/handlers/skill_handler.go
+12 -18
View File
@@ -56,12 +56,13 @@ func (h *SkillHandler) ListAllSkills(c *gin.Context) {
func (h *SkillHandler) GetSkill(c *gin.Context) {
userID, _ := c.Get("userID")
userRole, _ := c.Get("userRole")
skillID, err := strconv.Atoi(c.Param("id"))
if err != nil {
utils.Error(c, http.StatusBadRequest, "invalid skill ID")
return
}
item, err := h.service.GetSkill(userID.(int), skillID)
item, err := h.service.GetSkill(userID.(int), userRole.(string), skillID)
if err != nil {
utils.HandleError(c, err)
return
@@ -91,12 +92,13 @@ func (h *SkillHandler) UpdateSkill(c *gin.Context) {
func (h *SkillHandler) DeleteSkill(c *gin.Context) {
userID, _ := c.Get("userID")
userRole, _ := c.Get("userRole")
skillID, err := strconv.Atoi(c.Param("id"))
if err != nil {
utils.Error(c, http.StatusBadRequest, "invalid skill ID")
return
}
if err := h.service.DeleteSkill(userID.(int), skillID); err != nil {
if err := h.service.DeleteSkill(userID.(int), userRole.(string), skillID); err != nil {
utils.HandleError(c, err)
return
}
@@ -105,12 +107,13 @@ func (h *SkillHandler) DeleteSkill(c *gin.Context) {
func (h *SkillHandler) DownloadSkill(c *gin.Context) {
userID, _ := c.Get("userID")
userRole, _ := c.Get("userRole")
skillID, err := strconv.Atoi(c.Param("id"))
if err != nil {
utils.Error(c, http.StatusBadRequest, "invalid skill ID")
return
}
content, fileName, err := h.service.DownloadSkill(userID.(int), skillID)
content, fileName, err := h.service.DownloadSkill(userID.(int), userRole.(string), skillID)
if err != nil {
utils.HandleError(c, err)
return
@@ -120,25 +123,15 @@ func (h *SkillHandler) DownloadSkill(c *gin.Context) {
c.Data(http.StatusOK, "application/zip", content)
}
func (h *SkillHandler) DownloadSkillVersionForAgent(c *gin.Context) {
content, fileName, err := h.service.DownloadSkillVersionByExternalID(c.Param("skillVersion"))
if err != nil {
utils.HandleError(c, err)
return
}
c.Header("Content-Type", "application/octet-stream")
c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", fileName))
c.Data(http.StatusOK, "application/octet-stream", content)
}
func (h *SkillHandler) ListVersions(c *gin.Context) {
userID, _ := c.Get("userID")
userRole, _ := c.Get("userRole")
skillID, err := strconv.Atoi(c.Param("id"))
if err != nil {
utils.Error(c, http.StatusBadRequest, "invalid skill ID")
return
}
items, err := h.service.ListVersions(userID.(int), skillID)
items, err := h.service.ListVersions(userID.(int), userRole.(string), skillID)
if err != nil {
utils.HandleError(c, err)
return
@@ -148,12 +141,13 @@ func (h *SkillHandler) ListVersions(c *gin.Context) {
func (h *SkillHandler) ListScanResults(c *gin.Context) {
userID, _ := c.Get("userID")
userRole, _ := c.Get("userRole")
skillID, err := strconv.Atoi(c.Param("id"))
if err != nil {
utils.Error(c, http.StatusBadRequest, "invalid skill ID")
return
}
items, err := h.service.ListScanResults(userID.(int), skillID)
items, err := h.service.ListScanResults(userID.(int), userRole.(string), skillID)
if err != nil {
utils.HandleError(c, err)
return
@@ -200,9 +194,9 @@ func (h *SkillHandler) AttachSkillToInstance(c *gin.Context) {
}
userID, _ := c.Get("userID")
userRole, _ := c.Get("userRole")
item, err := h.service.AttachSkillToInstanceForActor(instanceID, req.SkillID, userID.(int), fmt.Sprint(userRole))
item, err := h.service.AttachSkillToInstance(userID.(int), userRole.(string), instanceID, req.SkillID)
if err != nil {
utils.HandleError(c, err)
utils.HandleHubError(c, err)
return
}
utils.Success(c, http.StatusCreated, "Skill attached to instance successfully", item)
backend/internal/handlers/skill_hub_handler.go
+253
View File
@@ -0,0 +1,253 @@
package handlers
import (
"encoding/json"
"fmt"
"net/http"
"strconv"
"strings"
"clawreef/internal/services"
"clawreef/internal/utils"
"github.com/gin-gonic/gin"
)
type SkillHubHandler struct {
service services.SkillService
instanceService services.InstanceService
}
func NewSkillHubHandler(service services.SkillService, instanceService services.InstanceService) *SkillHubHandler {
return &SkillHubHandler{service: service, instanceService: instanceService}
}
func (h *SkillHubHandler) ListCatalog(c *gin.Context) {
userID, _ := c.Get("userID")
userRole, _ := c.Get("userRole")
query := services.SkillHubCatalogQuery{
TagKeys: c.QueryArray("tag_keys"),
Search: strings.TrimSpace(c.Query("q")),
Page: parseIntDefault(c.Query("page"), 1),
PageSize: parseIntDefault(c.Query("page_size"), 20),
}
result, err := h.service.ListHubCatalog(userID.(int), userRole.(string), query)
if err != nil {
utils.HandleError(c, err)
return
}
utils.Success(c, http.StatusOK, "Skill hub catalog retrieved successfully", result)
}
func (h *SkillHubHandler) ListTags(c *gin.Context) {
userRole, _ := c.Get("userRole")
items, err := h.service.ListHubTags(userRole.(string))
if err != nil {
utils.HandleError(c, err)
return
}
utils.Success(c, http.StatusOK, "Skill hub tags retrieved successfully", items)
}
func (h *SkillHubHandler) ListMine(c *gin.Context) {
userID, _ := c.Get("userID")
items, err := h.service.ListMyHubSkills(userID.(int))
if err != nil {
utils.HandleError(c, err)
return
}
utils.Success(c, http.StatusOK, "My skill hub items retrieved successfully", items)
}
func (h *SkillHubHandler) ListAttachable(c *gin.Context) {
userID, _ := c.Get("userID")
userRole, _ := c.Get("userRole")
items, err := h.service.ListAttachableSkills(userID.(int), userRole.(string))
if err != nil {
utils.HandleError(c, err)
return
}
utils.Success(c, http.StatusOK, "Attachable skills retrieved successfully", items)
}
func (h *SkillHubHandler) GetSkill(c *gin.Context) {
userID, _ := c.Get("userID")
userRole, _ := c.Get("userRole")
skillID, err := strconv.Atoi(c.Param("id"))
if err != nil {
utils.Error(c, http.StatusBadRequest, "invalid skill ID")
return
}
item, err := h.service.GetSkillHubDetail(userID.(int), userRole.(string), skillID)
if err != nil {
utils.HandleError(c, err)
return
}
utils.Success(c, http.StatusOK, "Skill hub item retrieved successfully", item)
}
func (h *SkillHubHandler) PreviewImportSkills(c *gin.Context) {
userID, _ := c.Get("userID")
fileHeader, err := c.FormFile("file")
if err != nil {
utils.Error(c, http.StatusBadRequest, "file is required")
return
}
items, err := h.service.PreviewHubImport(c.Request.Context(), userID.(int), fileHeader)
if err != nil {
utils.HandleError(c, err)
return
}
utils.Success(c, http.StatusOK, "Skill import preview generated successfully", items)
}
func (h *SkillHubHandler) ImportSkills(c *gin.Context) {
userID, _ := c.Get("userID")
fileHeader, err := c.FormFile("file")
if err != nil {
utils.Error(c, http.StatusBadRequest, "file is required")
return
}
var decisions []services.SkillImportDecision
if raw := strings.TrimSpace(c.PostForm("decisions")); raw != "" {
if err := json.Unmarshal([]byte(raw), &decisions); err != nil {
utils.Error(c, http.StatusBadRequest, "invalid decisions payload")
return
}
}
items, err := h.service.ImportHubArchiveWithDecisions(c.Request.Context(), userID.(int), fileHeader, decisions)
if err != nil {
utils.HandleError(c, err)
return
}
utils.Success(c, http.StatusCreated, "Skills imported successfully", items)
}
func (h *SkillHubHandler) PublishSkill(c *gin.Context) {
userID, _ := c.Get("userID")
userRole, _ := c.Get("userRole")
skillID, err := strconv.Atoi(c.Param("id"))
if err != nil {
utils.Error(c, http.StatusBadRequest, "invalid skill ID")
return
}
var req services.PublishSkillHubRequest
if err := c.ShouldBindJSON(&req); err != nil {
utils.ValidationError(c, err)
return
}
item, err := h.service.PublishToHub(userID.(int), userRole.(string), skillID, req.TagIDs)
if err != nil {
utils.HandleHubError(c, err)
return
}
utils.Success(c, http.StatusOK, "Skill published to hub successfully", item)
}
func (h *SkillHubHandler) UnpublishSkill(c *gin.Context) {
userID, _ := c.Get("userID")
userRole, _ := c.Get("userRole")
skillID, err := strconv.Atoi(c.Param("id"))
if err != nil {
utils.Error(c, http.StatusBadRequest, "invalid skill ID")
return
}
item, err := h.service.UnpublishFromHub(userID.(int), userRole.(string), skillID)
if err != nil {
utils.HandleError(c, err)
return
}
utils.Success(c, http.StatusOK, "Skill unpublished from hub successfully", item)
}
func (h *SkillHubHandler) UpdateTags(c *gin.Context) {
userID, _ := c.Get("userID")
userRole, _ := c.Get("userRole")
skillID, err := strconv.Atoi(c.Param("id"))
if err != nil {
utils.Error(c, http.StatusBadRequest, "invalid skill ID")
return
}
var req services.UpdateSkillHubTagsRequest
if err := c.ShouldBindJSON(&req); err != nil {
utils.ValidationError(c, err)
return
}
item, err := h.service.UpdateHubTags(userID.(int), userRole.(string), skillID, req.TagIDs)
if err != nil {
utils.HandleHubError(c, err)
return
}
utils.Success(c, http.StatusOK, "Skill hub tags updated successfully", item)
}
func (h *SkillHubHandler) DeleteSkill(c *gin.Context) {
userID, _ := c.Get("userID")
userRole, _ := c.Get("userRole")
skillID, err := strconv.Atoi(c.Param("id"))
if err != nil {
utils.Error(c, http.StatusBadRequest, "invalid skill ID")
return
}
if err := h.service.DeleteSkill(userID.(int), userRole.(string), skillID); err != nil {
utils.HandleError(c, err)
return
}
utils.Success(c, http.StatusOK, "Skill deleted successfully", nil)
}
func (h *SkillHubHandler) DownloadSkill(c *gin.Context) {
userID, _ := c.Get("userID")
userRole, _ := c.Get("userRole")
skillID, err := strconv.Atoi(c.Param("id"))
if err != nil {
utils.Error(c, http.StatusBadRequest, "invalid skill ID")
return
}
content, fileName, err := h.service.DownloadSkill(userID.(int), userRole.(string), skillID)
if err != nil {
utils.HandleError(c, err)
return
}
c.Header("Content-Type", "application/zip")
c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", fileName))
c.Data(http.StatusOK, "application/zip", content)
}
func (h *SkillHubHandler) InstallSkill(c *gin.Context) {
userID, _ := c.Get("userID")
userRole, _ := c.Get("userRole")
skillID, err := strconv.Atoi(c.Param("id"))
if err != nil {
utils.Error(c, http.StatusBadRequest, "invalid skill ID")
return
}
var req services.InstallHubSkillRequest
if err := c.ShouldBindJSON(&req); err != nil {
utils.ValidationError(c, err)
return
}
item, err := h.service.InstallHubSkill(userID.(int), userRole.(string), skillID, req.InstanceID)
if err != nil {
utils.HandleHubError(c, err)
return
}
utils.Success(c, http.StatusCreated, "Skill installed to instance successfully", item)
}
func (h *SkillHubHandler) ListAdminSkills(c *gin.Context) {
items, err := h.service.ListAllHubSkillsAdmin()
if err != nil {
utils.HandleError(c, err)
return
}
utils.Success(c, http.StatusOK, "Admin skill hub items retrieved successfully", items)
}
func parseIntDefault(raw string, fallback int) int {
value, err := strconv.Atoi(strings.TrimSpace(raw))
if err != nil || value <= 0 {
return fallback
}
return value
}
backend/internal/middleware/auth_middleware.go
+1
View File
@@ -94,6 +94,7 @@ func GatewayAuth(instanceRepo repository.InstanceRepository, bindingRepos ...rep
c.Set("userID", instance.UserID)
c.Set("instanceID", instance.ID)
c.Set("instanceType", strings.TrimSpace(instance.Type))
c.Set("instanceMode", gatewayInstanceMode(instance.InstanceMode, instance.RuntimeType))
c.Set("runtimeType", strings.TrimSpace(instance.RuntimeType))
if bindingRepo != nil {
backend/internal/models/skill.go
+4
View File
@@ -11,6 +11,9 @@ type Skill struct {
CurrentVersionID *int `db:"current_version_id" json:"current_version_id,omitempty"`
SourceType string `db:"source_type" json:"source_type"`
Status string `db:"status" json:"status"`
Visibility string `db:"visibility" json:"visibility"`
PublishedAt *time.Time `db:"published_at" json:"published_at,omitempty"`
PublishedBy *int `db:"published_by" json:"published_by,omitempty"`
RiskLevel string `db:"risk_level" json:"risk_level"`
LastScannedAt *time.Time `db:"last_scanned_at" json:"last_scanned_at,omitempty"`
LastScanResultID *int `db:"last_scan_result_id" json:"last_scan_result_id,omitempty"`
@@ -58,6 +61,7 @@ type InstanceSkill struct {
SkillVersionID *int `db:"skill_version_id" json:"skill_version_id,omitempty"`
SourceType string `db:"source_type" json:"source_type"`
InstallPath *string `db:"install_path" json:"install_path,omitempty"`
WorkspaceDir *string `db:"workspace_dir" json:"workspace_dir,omitempty"`
ObservedHash *string `db:"observed_hash" json:"observed_hash,omitempty"`
Status string `db:"status" json:"status"`
LastSeenAt *time.Time `db:"last_seen_at" json:"last_seen_at,omitempty"`
backend/internal/models/skill_hub_tag.go
+25
View File
@@ -0,0 +1,25 @@
package models
import "time"
type SkillHubTag struct {
ID int `db:"id,primarykey,autoincrement" json:"id"`
TagKey string `db:"tag_key" json:"tag_key"`
Name string `db:"name" json:"name"`
Description *string `db:"description" json:"description,omitempty"`
SortOrder int `db:"sort_order" json:"sort_order"`
AdminOnly bool `db:"admin_only" json:"admin_only"`
CreatedAt time.Time `db:"created_at" json:"created_at"`
UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
}
func (t SkillHubTag) TableName() string { return "skill_hub_tags" }
type SkillHubTagAssignment struct {
ID int `db:"id,primarykey,autoincrement" json:"id"`
SkillID int `db:"skill_id" json:"skill_id"`
TagID int `db:"tag_id" json:"tag_id"`
CreatedAt time.Time `db:"created_at" json:"created_at"`
}
func (a SkillHubTagAssignment) TableName() string { return "skill_hub_tag_assignments" }
backend/internal/models/skill_package_materialize_job.go
+26
View File
@@ -0,0 +1,26 @@
package models
import "time"
type SkillPackageMaterializeJob struct {
ID int `db:"id,primarykey,autoincrement" json:"id"`
InstanceID int `db:"instance_id" json:"instance_id"`
SkillID int `db:"skill_id" json:"skill_id"`
BlobID int `db:"blob_id" json:"blob_id"`
WorkspaceDir string `db:"workspace_dir" json:"workspace_dir"`
ContentHash string `db:"content_hash" json:"content_hash"`
Status string `db:"status" json:"status"`
AttemptCount int `db:"attempt_count" json:"attempt_count"`
MaxAttempts int `db:"max_attempts" json:"max_attempts"`
LastError *string `db:"last_error" json:"last_error,omitempty"`
IdempotencyKey string `db:"idempotency_key" json:"idempotency_key"`
TriggerSource string `db:"trigger_source" json:"trigger_source"`
StartedAt *time.Time `db:"started_at" json:"started_at,omitempty"`
FinishedAt *time.Time `db:"finished_at" json:"finished_at,omitempty"`
CreatedAt time.Time `db:"created_at" json:"created_at"`
UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
}
func (j SkillPackageMaterializeJob) TableName() string {
return "skill_package_materialize_jobs"
}
backend/internal/repository/audit_event_repository.go
+19
View File
@@ -14,6 +14,7 @@ type AuditEventRepository interface {
Create(event *models.AuditEvent) error
ListByTraceID(traceID string) ([]models.AuditEvent, error)
ListRecent(limit int) ([]models.AuditEvent, error)
CountRecentByInstanceAndEventType(instanceID int, eventType string, since time.Time) (int, error)
}
type auditEventRepository struct {
@@ -92,3 +93,21 @@ func (r *auditEventRepository) ListRecent(limit int) ([]models.AuditEvent, error
}
return items, nil
}
func (r *auditEventRepository) CountRecentByInstanceAndEventType(instanceID int, eventType string, since time.Time) (int, error) {
row, err := r.sess.SQL().QueryRow(`
SELECT COUNT(*)
FROM audit_events
WHERE instance_id = ?
AND event_type = ?
AND created_at >= ?
`, instanceID, eventType, since)
if err != nil {
return 0, fmt.Errorf("failed to count audit events by instance and type: %w", err)
}
var count int
if err := row.Scan(&count); err != nil {
return 0, fmt.Errorf("failed to scan audit event count: %w", err)
}
return count, nil
}
backend/internal/repository/chat_session_repository.go
+9
View File
@@ -12,6 +12,7 @@ import (
// ChatSessionRepository defines repository operations for chat sessions.
type ChatSessionRepository interface {
GetBySessionID(sessionID string) (*models.ChatSession, error)
ListByInstanceID(instanceID int) ([]models.ChatSession, error)
Save(session *models.ChatSession) error
}
@@ -60,6 +61,14 @@ func (r *chatSessionRepository) GetBySessionID(sessionID string) (*models.ChatSe
return &item, nil
}
func (r *chatSessionRepository) ListByInstanceID(instanceID int) ([]models.ChatSession, error) {
var items []models.ChatSession
if err := r.sess.Collection("chat_sessions").Find(db.Cond{"instance_id": instanceID}).OrderBy("-last_activity_at").All(&items); err != nil {
return nil, fmt.Errorf("failed to list chat sessions by instance id: %w", err)
}
return items, nil
}
func (r *chatSessionRepository) Save(session *models.ChatSession) error {
now := time.Now()
existing, err := r.GetBySessionID(session.SessionID)
backend/internal/repository/cost_record_repository.go
+58
View File
@@ -9,12 +9,23 @@ import (
"github.com/upper/db/v4"
)
// InstanceSessionCostAggregate summarizes cost usage for one session on an instance.
type InstanceSessionCostAggregate struct {
SessionID string
PromptTokens int
CompletionTokens int
TotalTokens int
EstimatedCost float64
Currency string
}
// CostRecordRepository defines repository operations for token and money accounting.
type CostRecordRepository interface {
Create(record *models.CostRecord) error
ListByTraceID(traceID string) ([]models.CostRecord, error)
ListByUserID(userID, limit int) ([]models.CostRecord, error)
ListRecent(limit int) ([]models.CostRecord, error)
AggregateCostByInstanceSession(instanceID int, filter SessionUsageFilter) ([]InstanceSessionCostAggregate, error)
}
type costRecordRepository struct {
@@ -110,3 +121,50 @@ func (r *costRecordRepository) ListRecent(limit int) ([]models.CostRecord, error
}
return items, nil
}
func (r *costRecordRepository) AggregateCostByInstanceSession(instanceID int, filter SessionUsageFilter) ([]InstanceSessionCostAggregate, error) {
query := `
SELECT cr.session_id,
COALESCE(SUM(cr.prompt_tokens), 0),
COALESCE(SUM(cr.completion_tokens), 0),
COALESCE(SUM(cr.total_tokens), 0),
COALESCE(SUM(cr.estimated_cost), 0),
COALESCE(MAX(cr.currency), 'USD')
FROM cost_records cr
INNER JOIN model_invocations mi
ON mi.trace_id = cr.trace_id
AND mi.instance_id = cr.instance_id
AND mi.status != ?
WHERE cr.instance_id = ?
AND cr.session_id IS NOT NULL
AND cr.session_id != ''`
args := []interface{}{models.ModelInvocationStatusBlocked, instanceID}
query, args = appendTimeFilter(query, args, filter, "mi.created_at")
query += `
GROUP BY cr.session_id`
rows, err := r.sess.SQL().Query(query, args...)
if err != nil {
return nil, fmt.Errorf("failed to aggregate cost records by instance session: %w", err)
}
defer rows.Close()
items := make([]InstanceSessionCostAggregate, 0)
for rows.Next() {
var item InstanceSessionCostAggregate
if err := rows.Scan(
&item.SessionID,
&item.PromptTokens,
&item.CompletionTokens,
&item.TotalTokens,
&item.EstimatedCost,
&item.Currency,
); err != nil {
return nil, fmt.Errorf("failed to scan session cost aggregate: %w", err)
}
items = append(items, item)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("failed to iterate session cost aggregates: %w", err)
}
return items, nil
}
backend/internal/repository/instance_command_repository.go
+21
View File
@@ -2,6 +2,7 @@ package repository
import (
"fmt"
"strings"
"time"
"clawreef/internal/models"
@@ -16,6 +17,7 @@ type InstanceCommandRepository interface {
GetByInstanceIdempotencyKey(instanceID int, idempotencyKey string) (*models.InstanceCommand, error)
GetNextPendingByInstance(instanceID int) (*models.InstanceCommand, error)
ListByInstanceID(instanceID int, limit int) ([]models.InstanceCommand, error)
FindLatestFailedCollectSkillPackage(skillExternalID string) (*models.InstanceCommand, error)
}
type instanceCommandRepository struct {
@@ -95,3 +97,22 @@ func (r *instanceCommandRepository) ListByInstanceID(instanceID int, limit int)
}
return items, nil
}
func (r *instanceCommandRepository) FindLatestFailedCollectSkillPackage(skillExternalID string) (*models.InstanceCommand, error) {
skillExternalID = strings.TrimSpace(skillExternalID)
if skillExternalID == "" {
return nil, nil
}
pattern := fmt.Sprintf("%%\"skill_id\":\"%s\"%%", skillExternalID)
var item models.InstanceCommand
if err := r.sess.Collection("instance_commands").Find(db.Cond{
"command_type": "collect_skill_package",
"status": "failed",
}).And("payload_json LIKE ?", pattern).OrderBy("-finished_at", "-id").One(&item); err != nil {
if err == db.ErrNoMoreRows {
return nil, nil
}
return nil, fmt.Errorf("failed to find failed collect skill package command: %w", err)
}
return &item, nil
}
backend/internal/repository/model_invocation_repository.go
+147
View File
@@ -10,6 +10,17 @@ import (
"github.com/upper/db/v4"
)
// InstanceSessionTokenAggregate summarizes token usage for one session on an instance.
type InstanceSessionTokenAggregate struct {
SessionID string
PromptTokens int
CompletionTokens int
TotalTokens int
InvocationCount int
FirstSeenAt time.Time
LastSeenAt time.Time
}
// ModelInvocationRepository defines repository operations for governed model calls.
type ModelInvocationRepository interface {
Create(invocation *models.ModelInvocation) error
@@ -18,6 +29,9 @@ type ModelInvocationRepository interface {
ListBySessionID(sessionID string, limit int) ([]models.ModelInvocation, error)
ListByUserID(userID, limit int) ([]models.ModelInvocation, error)
ListRecent(limit int) ([]models.ModelInvocation, error)
AggregateByInstanceSession(instanceID int, filter SessionUsageFilter) ([]InstanceSessionTokenAggregate, error)
ListRecentByInstanceSession(instanceID int, sessionID string, limit int, filter SessionUsageFilter) ([]models.ModelInvocation, error)
CountDistinctSessionsByInstance(instanceID int, filter SessionUsageFilter) (int, error)
}
type modelInvocationRepository struct {
@@ -147,6 +161,139 @@ func (r *modelInvocationRepository) ListRecent(limit int) ([]models.ModelInvocat
return items, nil
}
func (r *modelInvocationRepository) AggregateByInstanceSession(instanceID int, filter SessionUsageFilter) ([]InstanceSessionTokenAggregate, error) {
query := `
SELECT session_id,
COALESCE(SUM(prompt_tokens), 0),
COALESCE(SUM(completion_tokens), 0),
COALESCE(SUM(total_tokens), 0),
COUNT(*),
MIN(created_at),
MAX(created_at)
FROM model_invocations
WHERE instance_id = ?
AND session_id IS NOT NULL
AND session_id != ''
AND status != ?`
args := []interface{}{instanceID, models.ModelInvocationStatusBlocked}
query, args = appendTimeFilter(query, args, filter, "created_at")
query += `
GROUP BY session_id`
rows, err := r.sess.SQL().Query(query, args...)
if err != nil {
return nil, fmt.Errorf("failed to aggregate model invocations by instance session: %w", err)
}
defer rows.Close()
items := make([]InstanceSessionTokenAggregate, 0)
for rows.Next() {
var item InstanceSessionTokenAggregate
if err := rows.Scan(
&item.SessionID,
&item.PromptTokens,
&item.CompletionTokens,
&item.TotalTokens,
&item.InvocationCount,
&item.FirstSeenAt,
&item.LastSeenAt,
); err != nil {
return nil, fmt.Errorf("failed to scan session token aggregate: %w", err)
}
items = append(items, item)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("failed to iterate session token aggregates: %w", err)
}
return items, nil
}
func (r *modelInvocationRepository) ListRecentByInstanceSession(instanceID int, sessionID string, limit int, filter SessionUsageFilter) ([]models.ModelInvocation, error) {
if limit <= 0 {
limit = 20
}
query := `
SELECT id, trace_id, session_id, request_id, user_id, instance_id, instance_mode, runtime_type, gateway_id, runtime_pod_id, model_id, provider_type, requested_model, actual_provider_model, traffic_class, request_payload, response_payload, prompt_tokens, completion_tokens, total_tokens, cached_tokens, reasoning_tokens, latency_ms, is_streaming, status, error_message, created_at, completed_at
FROM model_invocations
WHERE instance_id = ?
AND session_id = ?
AND status != ?`
args := []interface{}{instanceID, sessionID, models.ModelInvocationStatusBlocked}
query, args = appendTimeFilter(query, args, filter, "created_at")
query += `
ORDER BY created_at DESC
LIMIT ?`
args = append(args, limit)
rows, err := r.sess.SQL().Query(query, args...)
if err != nil {
return nil, fmt.Errorf("failed to list model invocations by instance session: %w", err)
}
defer rows.Close()
items := make([]models.ModelInvocation, 0, limit)
for rows.Next() {
var item models.ModelInvocation
if err := rows.Scan(
&item.ID,
&item.TraceID,
&item.SessionID,
&item.RequestID,
&item.UserID,
&item.InstanceID,
&item.InstanceMode,
&item.RuntimeType,
&item.GatewayID,
&item.RuntimePodID,
&item.ModelID,
&item.ProviderType,
&item.RequestedModel,
&item.ActualProviderModel,
&item.TrafficClass,
&item.RequestPayload,
&item.ResponsePayload,
&item.PromptTokens,
&item.CompletionTokens,
&item.TotalTokens,
&item.CachedTokens,
&item.ReasoningTokens,
&item.LatencyMs,
&item.IsStreaming,
&item.Status,
&item.ErrorMessage,
&item.CreatedAt,
&item.CompletedAt,
); err != nil {
return nil, fmt.Errorf("failed to scan model invocation by instance session: %w", err)
}
items = append(items, item)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("failed to iterate model invocations by instance session: %w", err)
}
return items, nil
}
func (r *modelInvocationRepository) CountDistinctSessionsByInstance(instanceID int, filter SessionUsageFilter) (int, error) {
query := `
SELECT COUNT(DISTINCT session_id)
FROM model_invocations
WHERE instance_id = ?
AND session_id IS NOT NULL
AND session_id != ''
AND status != ?`
args := []interface{}{instanceID, models.ModelInvocationStatusBlocked}
query, args = appendTimeFilter(query, args, filter, "created_at")
row, err := r.sess.SQL().QueryRow(query, args...)
if err != nil {
return 0, fmt.Errorf("failed to count distinct sessions by instance: %w", err)
}
var count int
if err := row.Scan(&count); err != nil {
return 0, fmt.Errorf("failed to scan distinct session count: %w", err)
}
return count, nil
}
func isDuplicateIndexError(err error) bool {
if err == nil {
return false
backend/internal/repository/session_usage_filter.go
+24
View File
@@ -0,0 +1,24 @@
package repository
import (
"fmt"
"time"
)
// SessionUsageFilter optionally bounds session usage aggregates by timestamp.
type SessionUsageFilter struct {
Since *time.Time
Until *time.Time
}
func appendTimeFilter(query string, args []interface{}, filter SessionUsageFilter, column string) (string, []interface{}) {
if filter.Since != nil {
query += fmt.Sprintf(" AND %s >= ?", column)
args = append(args, *filter.Since)
}
if filter.Until != nil {
query += fmt.Sprintf(" AND %s < ?", column)
args = append(args, *filter.Until)
}
return query, args
}
backend/internal/repository/session_usage_filter_test.go
+39
View File
@@ -0,0 +1,39 @@
package repository
import (
"strings"
"testing"
"time"
)
func TestAppendTimeFilterAddsSinceAndUntilClauses(t *testing.T) {
since := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
until := time.Date(2026, 7, 2, 0, 0, 0, 0, time.UTC)
query := "SELECT 1 FROM model_invocations WHERE instance_id = ?"
args := []interface{}{9}
query, args = appendTimeFilter(query, args, SessionUsageFilter{
Since: &since,
Until: &until,
}, "created_at")
if !strings.Contains(query, "created_at >= ?") || !strings.Contains(query, "created_at < ?") {
t.Fatalf("expected created_at bounds in query, got %q", query)
}
if len(args) != 3 {
t.Fatalf("expected 3 args, got %d (%v)", len(args), args)
}
if args[1] != since || args[2] != until {
t.Fatalf("unexpected bound args: %+v", args[1:])
}
}
func TestAppendTimeFilterEmptyFilterLeavesQueryUnchanged(t *testing.T) {
query := "SELECT 1 FROM cost_records WHERE instance_id = ?"
args := []interface{}{9}
updatedQuery, updatedArgs := appendTimeFilter(query, args, SessionUsageFilter{}, "recorded_at")
if updatedQuery != query || len(updatedArgs) != 1 {
t.Fatalf("expected unchanged query/args, got query=%q args=%v", updatedQuery, updatedArgs)
}
}
backend/internal/repository/skill_package_materialize_job_repository.go
+335
View File
@@ -0,0 +1,335 @@
package repository
import (
"context"
"database/sql"
"errors"
"fmt"
"strings"
"time"
"clawreef/internal/models"
"github.com/upper/db/v4"
)
type SkillPackageMaterializeBackfillCandidate struct {
InstanceID int
SkillID int
BlobID int
WorkspaceDir string
ContentHash string
}
type SkillPackageMaterializeJobRepository interface {
Create(job *models.SkillPackageMaterializeJob) error
GetByID(id int) (*models.SkillPackageMaterializeJob, error)
GetByIdempotencyKey(key string) (*models.SkillPackageMaterializeJob, error)
ClaimNextPending(ctx context.Context, limit int) ([]models.SkillPackageMaterializeJob, error)
MarkSucceeded(id int) error
MarkFailed(id int, errMsg string, retryable bool) error
MarkRunning(id int) error
ReleaseToPending(id int) error
ResetForRetry(skillID int) error
RequeueExisting(id, blobID int, contentHash, workspaceDir string) error
FindLatestBySkillID(skillID int) (*models.SkillPackageMaterializeJob, error)
CountPendingByInstance(instanceID int) (int, error)
ListBackfillCandidates(limit int) ([]SkillPackageMaterializeBackfillCandidate, error)
}
type skillPackageMaterializeJobRepository struct {
sess db.Session
}
func NewSkillPackageMaterializeJobRepository(sess db.Session) SkillPackageMaterializeJobRepository {
return &skillPackageMaterializeJobRepository{sess: sess}
}
func (r *skillPackageMaterializeJobRepository) Create(job *models.SkillPackageMaterializeJob) error {
existing, err := r.GetByIdempotencyKey(job.IdempotencyKey)
if err != nil {
return err
}
if existing != nil {
*job = *existing
return nil
}
if strings.TrimSpace(job.Status) == "" {
job.Status = "pending"
}
if job.MaxAttempts <= 0 {
job.MaxAttempts = 5
}
if strings.TrimSpace(job.TriggerSource) == "" {
job.TriggerSource = "sync"
}
ensureTimestamps(&job.CreatedAt, &job.UpdatedAt)
res, err := r.sess.Collection("skill_package_materialize_jobs").Insert(job)
if err != nil {
if isDuplicateEntryError(err) {
existing, findErr := r.GetByIdempotencyKey(job.IdempotencyKey)
if findErr != nil {
return findErr
}
if existing != nil {
*job = *existing
return nil
}
}
return fmt.Errorf("failed to create skill package materialize job: %w", err)
}
if id, ok := res.ID().(int64); ok {
job.ID = int(id)
}
return nil
}
func (r *skillPackageMaterializeJobRepository) GetByID(id int) (*models.SkillPackageMaterializeJob, error) {
var item models.SkillPackageMaterializeJob
if err := r.sess.Collection("skill_package_materialize_jobs").Find(db.Cond{"id": id}).One(&item); err != nil {
if err == db.ErrNoMoreRows {
return nil, nil
}
return nil, fmt.Errorf("failed to get skill package materialize job: %w", err)
}
return &item, nil
}
func (r *skillPackageMaterializeJobRepository) GetByIdempotencyKey(key string) (*models.SkillPackageMaterializeJob, error) {
key = strings.TrimSpace(key)
if key == "" {
return nil, nil
}
var item models.SkillPackageMaterializeJob
if err := r.sess.Collection("skill_package_materialize_jobs").Find(db.Cond{"idempotency_key": key}).One(&item); err != nil {
if err == db.ErrNoMoreRows {
return nil, nil
}
return nil, fmt.Errorf("failed to get skill package materialize job by idempotency key: %w", err)
}
return &item, nil
}
func (r *skillPackageMaterializeJobRepository) ClaimNextPending(ctx context.Context, limit int) ([]models.SkillPackageMaterializeJob, error) {
if limit <= 0 {
limit = 1
}
var ids []int
iter := r.sess.SQL().IteratorContext(ctx, `
SELECT id FROM skill_package_materialize_jobs
WHERE status = 'pending'
ORDER BY created_at ASC, id ASC
LIMIT ?`, limit)
for iter.Next() {
var id int
if err := iter.Scan(&id); err != nil {
iter.Close()
return nil, fmt.Errorf("failed to scan pending materialize job id: %w", err)
}
ids = append(ids, id)
}
if err := iter.Err(); err != nil {
iter.Close()
return nil, fmt.Errorf("failed to list pending materialize jobs: %w", err)
}
iter.Close()
now := time.Now().UTC()
claimed := make([]models.SkillPackageMaterializeJob, 0, len(ids))
for _, id := range ids {
res, err := r.sess.SQL().ExecContext(ctx, `
UPDATE skill_package_materialize_jobs
SET status = 'running',
attempt_count = attempt_count + 1,
started_at = ?,
updated_at = ?
WHERE id = ? AND status = 'pending'`, now, now, id)
if err != nil {
return nil, fmt.Errorf("failed to claim materialize job %d: %w", id, err)
}
affected, err := res.RowsAffected()
if err != nil {
return nil, fmt.Errorf("failed to read claim rows affected for job %d: %w", id, err)
}
if affected == 0 {
continue
}
item, err := r.GetByID(id)
if err != nil {
return nil, err
}
if item != nil {
claimed = append(claimed, *item)
}
}
return claimed, nil
}
func (r *skillPackageMaterializeJobRepository) MarkSucceeded(id int) error {
now := time.Now().UTC()
_, err := r.sess.SQL().Exec(`
UPDATE skill_package_materialize_jobs
SET status = 'succeeded',
finished_at = ?,
last_error = NULL,
updated_at = ?
WHERE id = ?`, now, now, id)
if err != nil {
return fmt.Errorf("failed to mark materialize job succeeded: %w", err)
}
return nil
}
func (r *skillPackageMaterializeJobRepository) MarkRunning(id int) error {
now := time.Now().UTC()
_, err := r.sess.SQL().Exec(`
UPDATE skill_package_materialize_jobs
SET status = 'running',
attempt_count = attempt_count + 1,
started_at = ?,
updated_at = ?
WHERE id = ? AND status = 'pending'`, now, now, id)
if err != nil {
return fmt.Errorf("failed to mark materialize job running: %w", err)
}
return nil
}
func (r *skillPackageMaterializeJobRepository) MarkFailed(id int, errMsg string, retryable bool) error {
job, err := r.GetByID(id)
if err != nil {
return err
}
if job == nil {
return fmt.Errorf("materialize job not found")
}
now := time.Now().UTC()
status := "failed"
if retryable && job.AttemptCount < job.MaxAttempts {
status = "pending"
}
trimmed := strings.TrimSpace(errMsg)
var lastError *string
if trimmed != "" {
lastError = &trimmed
}
update := map[string]interface{}{
"status": status,
"last_error": lastError,
"updated_at": now,
}
if status == "failed" {
update["finished_at"] = now
}
if err := r.sess.Collection("skill_package_materialize_jobs").Find(db.Cond{"id": id}).Update(update); err != nil {
return fmt.Errorf("failed to mark materialize job failed: %w", err)
}
return nil
}
func (r *skillPackageMaterializeJobRepository) ReleaseToPending(id int) error {
now := time.Now().UTC()
_, err := r.sess.SQL().Exec(`
UPDATE skill_package_materialize_jobs
SET status = 'pending',
started_at = NULL,
updated_at = ?
WHERE id = ? AND status = 'running'`, now, id)
if err != nil {
return fmt.Errorf("failed to release materialize job to pending: %w", err)
}
return nil
}
func (r *skillPackageMaterializeJobRepository) ResetForRetry(skillID int) error {
job, err := r.FindLatestBySkillID(skillID)
if err != nil {
return err
}
if job == nil {
return fmt.Errorf("materialize job not found")
}
return r.RequeueExisting(job.ID, job.BlobID, job.ContentHash, job.WorkspaceDir)
}
func (r *skillPackageMaterializeJobRepository) RequeueExisting(id, blobID int, contentHash, workspaceDir string) error {
now := time.Now().UTC()
_, err := r.sess.SQL().Exec(`
UPDATE skill_package_materialize_jobs
SET status = 'pending',
last_error = NULL,
finished_at = NULL,
started_at = NULL,
blob_id = ?,
content_hash = ?,
workspace_dir = ?,
updated_at = ?
WHERE id = ?`, blobID, strings.TrimSpace(contentHash), sanitizeMaterializeWorkspaceDir(workspaceDir), now, id)
if err != nil {
return fmt.Errorf("failed to requeue materialize job: %w", err)
}
return nil
}
func sanitizeMaterializeWorkspaceDir(value string) string {
return strings.TrimSpace(value)
}
func (r *skillPackageMaterializeJobRepository) FindLatestBySkillID(skillID int) (*models.SkillPackageMaterializeJob, error) {
var item models.SkillPackageMaterializeJob
if err := r.sess.Collection("skill_package_materialize_jobs").Find(db.Cond{"skill_id": skillID}).OrderBy("-created_at", "-id").One(&item); err != nil {
if err == db.ErrNoMoreRows {
return nil, nil
}
return nil, fmt.Errorf("failed to find latest materialize job: %w", err)
}
return &item, nil
}
func (r *skillPackageMaterializeJobRepository) CountPendingByInstance(instanceID int) (int, error) {
row, err := r.sess.SQL().QueryRow(`
SELECT COUNT(*) FROM skill_package_materialize_jobs
WHERE instance_id = ? AND status IN ('pending', 'running')`, instanceID)
if err != nil {
return 0, fmt.Errorf("failed to count pending materialize jobs: %w", err)
}
var count int
if err := row.Scan(&count); err != nil {
return 0, fmt.Errorf("failed to scan pending materialize job count: %w", err)
}
return count, nil
}
func (r *skillPackageMaterializeJobRepository) ListBackfillCandidates(limit int) ([]SkillPackageMaterializeBackfillCandidate, error) {
if limit <= 0 {
limit = 500
}
iter := r.sess.SQL().Iterator(`
SELECT isk.instance_id, isk.skill_id, sv.blob_id, isk.workspace_dir, sb.content_hash
FROM instance_skills isk
JOIN instances i ON i.id = isk.instance_id
JOIN skills s ON s.id = isk.skill_id
JOIN skill_versions sv ON sv.id = s.current_version_id
JOIN skill_blobs sb ON sb.id = sv.blob_id
WHERE isk.status = 'active'
AND (LOWER(TRIM(i.instance_mode)) = 'lite' OR LOWER(TRIM(i.runtime_type)) = 'gateway')
AND TRIM(sb.object_key) = ''
AND isk.workspace_dir IS NOT NULL
AND TRIM(isk.workspace_dir) <> ''
ORDER BY isk.updated_at ASC
LIMIT ?`, limit)
defer iter.Close()
result := make([]SkillPackageMaterializeBackfillCandidate, 0)
for iter.Next() {
var item SkillPackageMaterializeBackfillCandidate
if err := iter.Scan(&item.InstanceID, &item.SkillID, &item.BlobID, &item.WorkspaceDir, &item.ContentHash); err != nil {
return nil, fmt.Errorf("failed to scan backfill candidate: %w", err)
}
result = append(result, item)
}
if err := iter.Err(); err != nil && !errors.Is(err, sql.ErrNoRows) {
return nil, fmt.Errorf("failed to list backfill candidates: %w", err)
}
return result, nil
}
backend/internal/repository/skill_repository.go
+125 -9
View File
@@ -27,7 +27,9 @@ type SkillRepository interface {
GetVersionBySkillAndBlob(skillID, blobID int) (*models.SkillVersion, error)
GetLatestVersionBySkillID(skillID int) (*models.SkillVersion, error)
CreateVersion(version *models.SkillVersion) error
UpdateVersion(version *models.SkillVersion) error
ListInstanceSkills(instanceID int) ([]models.InstanceSkill, error)
ListActiveInstanceSkillsBySkillID(skillID int) ([]models.InstanceSkill, error)
GetInstanceSkill(instanceID, skillID int) (*models.InstanceSkill, error)
UpsertInstanceSkill(item *models.InstanceSkill) error
MarkInstanceSkillRemoved(instanceID int, skillID int, observedAt time.Time) error
@@ -39,6 +41,12 @@ type SkillRepository interface {
ListScanResultsByBlobID(blobID int) ([]models.SkillScanResult, error)
GetLatestScanResultByBlobID(blobID int) (*models.SkillScanResult, error)
GetLatestScanResultBySkillID(skillID int) (*models.SkillScanResult, error)
ListHubTags(includeAdminOnly bool) ([]models.SkillHubTag, error)
GetHubTagByID(id int) (*models.SkillHubTag, error)
ListHubTagsBySkillID(skillID int) ([]models.SkillHubTag, error)
ReplaceSkillTagAssignments(skillID int, tagIDs []int) error
ListPublicHubSkills() ([]models.Skill, error)
ListSkillsForHubAdmin() ([]models.Skill, error)
}
type skillRepository struct{ sess db.Session }
@@ -74,7 +82,7 @@ func (r *skillRepository) GetSkillByID(id int) (*models.Skill, error) {
func (r *skillRepository) GetSkillByUserKey(userID int, skillKey string) (*models.Skill, error) {
var item models.Skill
if err := r.sess.Collection("skills").Find(db.Cond{"user_id": userID, "skill_key": skillKey}).One(&item); err != nil {
if err := r.sess.Collection("skills").Find(db.Cond{"user_id": userID, "skill_key": skillKey, "status": "active"}).One(&item); err != nil {
if err == db.ErrNoMoreRows {
return nil, nil
}
@@ -84,6 +92,9 @@ func (r *skillRepository) GetSkillByUserKey(userID int, skillKey string) (*model
}
func (r *skillRepository) CreateSkill(skill *models.Skill) error {
if strings.TrimSpace(skill.Visibility) == "" {
skill.Visibility = "private"
}
ensureTimestamps(&skill.CreatedAt, &skill.UpdatedAt)
res, err := r.sess.Collection("skills").Insert(skill)
if err != nil {
@@ -219,6 +230,16 @@ func (r *skillRepository) CreateVersion(version *models.SkillVersion) error {
return nil
}
func (r *skillRepository) UpdateVersion(version *models.SkillVersion) error {
if version.UpdatedAt.IsZero() {
version.UpdatedAt = time.Now().UTC()
}
if err := r.sess.Collection("skill_versions").Find(db.Cond{"id": version.ID}).Update(version); err != nil {
return fmt.Errorf("failed to update skill version: %w", err)
}
return nil
}
func (r *skillRepository) ListInstanceSkills(instanceID int) ([]models.InstanceSkill, error) {
var items []models.InstanceSkill
if err := r.sess.Collection("instance_skills").Find(db.Cond{"instance_id": instanceID}).OrderBy("-updated_at", "-id").All(&items); err != nil {
@@ -227,6 +248,17 @@ func (r *skillRepository) ListInstanceSkills(instanceID int) ([]models.InstanceS
return items, nil
}
func (r *skillRepository) ListActiveInstanceSkillsBySkillID(skillID int) ([]models.InstanceSkill, error) {
var items []models.InstanceSkill
if err := r.sess.Collection("instance_skills").Find(db.Cond{
"skill_id": skillID,
"status NOT IN": []string{"removed", "missing"},
}).OrderBy("-updated_at", "-id").All(&items); err != nil {
return nil, fmt.Errorf("failed to list active instance skills by skill id: %w", err)
}
return items, nil
}
func (r *skillRepository) GetInstanceSkill(instanceID, skillID int) (*models.InstanceSkill, error) {
var item models.InstanceSkill
if err := r.sess.Collection("instance_skills").Find(db.Cond{"instance_id": instanceID, "skill_id": skillID}).One(&item); err != nil {
@@ -262,6 +294,9 @@ func (r *skillRepository) UpsertInstanceSkill(item *models.InstanceSkill) error
if item.UpdatedAt.IsZero() {
item.UpdatedAt = time.Now().UTC()
}
if existing.SourceType == "injected_by_clawmanager" && item.SourceType == "discovered_in_instance" {
item.SourceType = existing.SourceType
}
if err := r.sess.Collection("instance_skills").Find(db.Cond{"id": existing.ID}).Update(item); err != nil {
return fmt.Errorf("failed to update instance skill after duplicate insert: %w", err)
}
@@ -277,6 +312,9 @@ func (r *skillRepository) UpsertInstanceSkill(item *models.InstanceSkill) error
if item.UpdatedAt.IsZero() {
item.UpdatedAt = time.Now().UTC()
}
if existing.SourceType == "injected_by_clawmanager" && item.SourceType == "discovered_in_instance" {
item.SourceType = existing.SourceType
}
if err := r.sess.Collection("instance_skills").Find(db.Cond{"id": existing.ID}).Update(item); err != nil {
return fmt.Errorf("failed to update instance skill: %w", err)
}
@@ -407,11 +445,9 @@ func workspaceDeleteTargetsSkillKey(deletedPath string, skillKey string) bool {
if len(segments) <= 2 {
return true
}
for i, segment := range segments {
if segment == "skills" && i+1 < len(segments) && segments[i+1] == key {
return true
}
if segment == ".openclaw" || segment == "openclaw" {
for _, segment := range segments {
switch segment {
case "skills", ".hermes", "hermes", ".openclaw", "openclaw":
return true
}
}
@@ -419,7 +455,8 @@ func workspaceDeleteTargetsSkillKey(deletedPath string, skillKey string) bool {
}
func isRemovedInstanceSkillRecord(item models.InstanceSkill) bool {
return strings.EqualFold(strings.TrimSpace(item.Status), "removed") || item.RemovedAt != nil
status := strings.ToLower(strings.TrimSpace(item.Status))
return status == "removed" || status == "missing"
}
func (r *skillRepository) MarkMissingInstanceSkills(instanceID int, activeSkillIDs []int, observedAt time.Time) error {
@@ -432,11 +469,14 @@ func (r *skillRepository) MarkMissingInstanceSkills(instanceID int, activeSkillI
return fmt.Errorf("failed to list stale instance skills: %w", err)
}
for _, item := range items {
item.Status = "removed"
if strings.EqualFold(strings.TrimSpace(item.Status), "removed") {
continue
}
item.Status = "missing"
item.RemovedAt = &observedAt
item.UpdatedAt = observedAt
if err := r.sess.Collection("instance_skills").Find(db.Cond{"id": item.ID}).Update(item); err != nil {
return fmt.Errorf("failed to mark instance skill removed: %w", err)
return fmt.Errorf("failed to mark instance skill missing: %w", err)
}
}
return nil
@@ -491,3 +531,79 @@ func (r *skillRepository) GetLatestScanResultBySkillID(skillID int) (*models.Ski
}
return r.GetScanResultByID(*skill.LastScanResultID)
}
func (r *skillRepository) ListHubTags(includeAdminOnly bool) ([]models.SkillHubTag, error) {
var items []models.SkillHubTag
query := r.sess.Collection("skill_hub_tags").Find()
if !includeAdminOnly {
query = query.And(db.Cond{"admin_only": false})
}
if err := query.OrderBy("sort_order", "id").All(&items); err != nil {
return nil, fmt.Errorf("failed to list skill hub tags: %w", err)
}
return items, nil
}
func (r *skillRepository) GetHubTagByID(id int) (*models.SkillHubTag, error) {
var item models.SkillHubTag
if err := r.sess.Collection("skill_hub_tags").Find(db.Cond{"id": id}).One(&item); err != nil {
if err == db.ErrNoMoreRows {
return nil, nil
}
return nil, fmt.Errorf("failed to get skill hub tag: %w", err)
}
return &item, nil
}
func (r *skillRepository) ListHubTagsBySkillID(skillID int) ([]models.SkillHubTag, error) {
var assignments []models.SkillHubTagAssignment
if err := r.sess.Collection("skill_hub_tag_assignments").Find(db.Cond{"skill_id": skillID}).All(&assignments); err != nil {
return nil, fmt.Errorf("failed to list skill hub tag assignments: %w", err)
}
if len(assignments) == 0 {
return []models.SkillHubTag{}, nil
}
tagIDs := make([]interface{}, 0, len(assignments))
for _, item := range assignments {
tagIDs = append(tagIDs, item.TagID)
}
var tags []models.SkillHubTag
if err := r.sess.Collection("skill_hub_tags").Find(db.Cond{"id IN": tagIDs}).OrderBy("sort_order", "id").All(&tags); err != nil {
return nil, fmt.Errorf("failed to list skill hub tags by skill id: %w", err)
}
return tags, nil
}
func (r *skillRepository) ReplaceSkillTagAssignments(skillID int, tagIDs []int) error {
if err := r.sess.Collection("skill_hub_tag_assignments").Find(db.Cond{"skill_id": skillID}).Delete(); err != nil {
return fmt.Errorf("failed to clear skill hub tag assignments: %w", err)
}
for _, tagID := range tagIDs {
assignment := &models.SkillHubTagAssignment{SkillID: skillID, TagID: tagID}
ensureTimestamps(&assignment.CreatedAt, nil)
if _, err := r.sess.Collection("skill_hub_tag_assignments").Insert(assignment); err != nil {
return fmt.Errorf("failed to create skill hub tag assignment: %w", err)
}
}
return nil
}
func (r *skillRepository) ListPublicHubSkills() ([]models.Skill, error) {
var items []models.Skill
if err := r.sess.Collection("skills").Find(db.Cond{
"visibility": "public",
"source_type": "uploaded",
"status": "active",
}).OrderBy("-published_at", "-updated_at", "-id").All(&items); err != nil {
return nil, fmt.Errorf("failed to list public hub skills: %w", err)
}
return items, nil
}
func (r *skillRepository) ListSkillsForHubAdmin() ([]models.Skill, error) {
var items []models.Skill
if err := r.sess.Collection("skills").Find(db.Cond{"source_type": "uploaded"}).OrderBy("-updated_at", "-id").All(&items); err != nil {
return nil, fmt.Errorf("failed to list skills for hub admin: %w", err)
}
return items, nil
}
backend/internal/repository/skill_repository_test.go
+24
View File
@@ -0,0 +1,24 @@
package repository
import "testing"
func TestWorkspaceDeleteTargetsSkillKeyHermesNestedPath(t *testing.T) {
path := "home/.hermes/skills/productivity/my-skill"
if !workspaceDeleteTargetsSkillKey(path, "my-skill") {
t.Fatalf("expected nested hermes skill path to match")
}
}
func TestWorkspaceDeleteTargetsSkillKeyOpenClawFlatPath(t *testing.T) {
path := "home/.openclaw/workspace/skills/paper-ranker"
if !workspaceDeleteTargetsSkillKey(path, "paper-ranker") {
t.Fatalf("expected openclaw flat skill path to match")
}
}
func TestWorkspaceDeleteTargetsSkillKeyRejectsMismatchedLeaf(t *testing.T) {
path := "home/.hermes/skills/productivity/other-skill"
if workspaceDeleteTargetsSkillKey(path, "my-skill") {
t.Fatalf("expected mismatched leaf to be rejected")
}
}
backend/internal/services/ai_observability_governance_overview_test.go
+109
View File
@@ -0,0 +1,109 @@
package services
import (
"context"
"testing"
"time"
"clawreef/internal/models"
"clawreef/internal/repository"
)
type stubGovernanceInstanceRepo struct {
instances []models.Instance
}
func (s *stubGovernanceInstanceRepo) Create(*models.Instance) error { panic("not used") }
func (s *stubGovernanceInstanceRepo) GetByID(int) (*models.Instance, error) {
panic("not used")
}
func (s *stubGovernanceInstanceRepo) GetByAccessToken(string) (*models.Instance, error) {
panic("not used")
}
func (s *stubGovernanceInstanceRepo) GetByAgentBootstrapToken(string) (*models.Instance, error) {
panic("not used")
}
func (s *stubGovernanceInstanceRepo) GetAll(int, int) ([]models.Instance, error) { panic("not used") }
func (s *stubGovernanceInstanceRepo) CountAll() (int, error) { panic("not used") }
func (s *stubGovernanceInstanceRepo) GetByUserID(int, int, int) ([]models.Instance, error) {
panic("not used")
}
func (s *stubGovernanceInstanceRepo) CountByUserID(int) (int, error) { panic("not used") }
func (s *stubGovernanceInstanceRepo) CountActiveByMode(context.Context, string) (int, error) {
panic("not used")
}
func (s *stubGovernanceInstanceRepo) ExistsByUserIDAndName(int, string) (bool, error) {
panic("not used")
}
func (s *stubGovernanceInstanceRepo) GetAllRunning() ([]models.Instance, error) {
return s.instances, nil
}
func (s *stubGovernanceInstanceRepo) GetV2DesiredRunning(context.Context, int) ([]models.Instance, error) {
panic("not used")
}
func (s *stubGovernanceInstanceRepo) GetV2Creating(context.Context, int) ([]models.Instance, error) {
panic("not used")
}
func (s *stubGovernanceInstanceRepo) UpdateRuntimeState(context.Context, int, string, int, *string) error {
panic("not used")
}
func (s *stubGovernanceInstanceRepo) SetWorkspacePath(context.Context, int, string) error {
panic("not used")
}
func (s *stubGovernanceInstanceRepo) UpdateWorkspaceUsage(context.Context, int, int64) error {
panic("not used")
}
func (s *stubGovernanceInstanceRepo) Update(*models.Instance) error { panic("not used") }
func (s *stubGovernanceInstanceRepo) Delete(int) error { panic("not used") }
type stubGovernanceRuntimeStatusRepo struct {
external map[int]bool
}
func (s *stubGovernanceRuntimeStatusRepo) GetByInstanceID(instanceID int) (*models.InstanceRuntimeStatus, error) {
if s.external[instanceID] {
raw := `{"llm_config_status":"external"}`
return &models.InstanceRuntimeStatus{SystemInfoJSON: &raw}, nil
}
return nil, nil
}
func (s *stubGovernanceRuntimeStatusRepo) Create(*models.InstanceRuntimeStatus) error { panic("not used") }
func (s *stubGovernanceRuntimeStatusRepo) Update(*models.InstanceRuntimeStatus) error { panic("not used") }
func TestGetLLMGovernanceOverviewSummarizesManagedRuntimeInstances(t *testing.T) {
now := time.Now().UTC()
service := &aiObservabilityService{
invocationRepo: &stubSessionUsageInvocationRepo{
aggregates: []repository.InstanceSessionTokenAggregate{
{SessionID: "agent:openclaw:main", TotalTokens: 10, LastSeenAt: now, FirstSeenAt: now},
},
},
costRepo: &stubSessionUsageCostRepo{},
chatSessionRepo: &stubSessionUsageChatSessionRepo{},
auditRepo: &stubSessionUsageAuditRepo{},
instanceRepo: &stubGovernanceInstanceRepo{
instances: []models.Instance{
{ID: 1, UserID: 9, Name: "oc-1", Type: "openclaw", Status: "running"},
{ID: 2, UserID: 9, Name: "oc-2", Type: "openclaw", Status: "running"},
{ID: 3, UserID: 9, Name: "ubuntu", Type: "ubuntu", Status: "running"},
},
},
runtimeStatusRepo: &stubGovernanceRuntimeStatusRepo{
external: map[int]bool{2: true},
},
}
overview, err := service.GetLLMGovernanceOverview()
if err != nil {
t.Fatalf("GetLLMGovernanceOverview failed: %v", err)
}
if overview.TotalManagedInstances != 2 {
t.Fatalf("expected 2 managed instances, got %d", overview.TotalManagedInstances)
}
if overview.ExternalConfigCount != 1 {
t.Fatalf("expected 1 external config instance, got %d", overview.ExternalConfigCount)
}
if len(overview.Items) != 2 {
t.Fatalf("expected 2 overview items, got %d", len(overview.Items))
}
}
backend/internal/services/ai_observability_service.go
+630 -10
View File
@@ -4,11 +4,13 @@ import (
"encoding/json"
"fmt"
"sort"
"strconv"
"strings"
"time"
"clawreef/internal/models"
"clawreef/internal/repository"
"clawreef/internal/utils"
)
// AuditQuery contains query options for AI audit list views.
@@ -160,11 +162,152 @@ type CostRecordView struct {
RecordedAt time.Time `json:"recorded_at"`
}
// InstanceSessionUsageQuery contains query options for instance session usage views.
type InstanceSessionUsageQuery struct {
Page int
Limit int
Search string
Since *time.Time
Until *time.Time
}
// InstanceSessionUsageSummary summarizes token usage across all sessions on an instance.
type InstanceSessionUsageSummary struct {
TotalPromptTokens int `json:"total_prompt_tokens"`
TotalCompletionTokens int `json:"total_completion_tokens"`
TotalTokens int `json:"total_tokens"`
TotalEstimatedCost float64 `json:"total_estimated_cost"`
Currency string `json:"currency"`
SessionCount int `json:"session_count"`
}
// InstanceSessionUsageCompliance reports session attribution quality for an instance.
type InstanceSessionUsageCompliance struct {
FallbackSessionCount int `json:"fallback_session_count"`
HasFallbackSessions bool `json:"has_fallback_sessions"`
RecentFallbackAuditCount int `json:"recent_fallback_audit_count"`
}
// InstanceSessionUsageItem is one session row for instance usage reporting.
type InstanceSessionUsageItem struct {
SessionID string `json:"session_id"`
SessionKey string `json:"session_key"`
Title *string `json:"title,omitempty"`
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
EstimatedCost float64 `json:"estimated_cost"`
Currency string `json:"currency"`
InvocationCount int `json:"invocation_count"`
FirstSeenAt time.Time `json:"first_seen_at"`
LastSeenAt time.Time `json:"last_seen_at"`
}
// InstanceSessionUsageResult is the paginated instance session usage response.
type InstanceSessionUsageResult struct {
Summary InstanceSessionUsageSummary `json:"summary"`
Compliance InstanceSessionUsageCompliance `json:"compliance"`
Items []InstanceSessionUsageItem `json:"items"`
Total int `json:"total"`
Page int `json:"page"`
Limit int `json:"limit"`
}
// InstanceSessionTrace is a recent trace row within one session.
type InstanceSessionTrace struct {
TraceID string `json:"trace_id"`
RequestedModel string `json:"requested_model"`
Status string `json:"status"`
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
CreatedAt time.Time `json:"created_at"`
}
// InstanceSessionUsageDetail is the drill-down payload for one session on an instance.
type InstanceSessionUsageDetail struct {
SessionID string `json:"session_id"`
SessionKey string `json:"session_key"`
Title *string `json:"title,omitempty"`
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
EstimatedCost float64 `json:"estimated_cost"`
Currency string `json:"currency"`
InvocationCount int `json:"invocation_count"`
FirstSeenAt time.Time `json:"first_seen_at"`
LastSeenAt time.Time `json:"last_seen_at"`
ModelBreakdown []CostBreakdownItem `json:"model_breakdown"`
RecentTraces []InstanceSessionTrace `json:"recent_traces"`
}
// InstanceLLMGovernanceStatus summarizes LLM governance health for one instance.
type InstanceLLMGovernanceStatus struct {
ConfigStatus string `json:"config_status"`
SessionFallbackRate float64 `json:"session_fallback_rate"`
RecentEgressBlockCount int `json:"recent_egress_block_count"`
IsCompliant bool `json:"is_compliant"`
}
// LLMGovernanceOverviewItem summarizes governance for one managed runtime instance.
type LLMGovernanceOverviewItem struct {
InstanceID int `json:"instance_id"`
InstanceName string `json:"instance_name"`
InstanceType string `json:"instance_type"`
UserID int `json:"user_id"`
ConfigStatus string `json:"config_status"`
SessionFallbackRate float64 `json:"session_fallback_rate"`
RecentEgressBlockCount int `json:"recent_egress_block_count"`
IsCompliant bool `json:"is_compliant"`
}
// LLMGovernanceOverview aggregates governance signals across managed runtime instances.
type LLMGovernanceOverview struct {
TotalManagedInstances int `json:"total_managed_instances"`
NonCompliantCount int `json:"non_compliant_count"`
ExternalConfigCount int `json:"external_config_count"`
HighFallbackCount int `json:"high_fallback_count"`
Items []LLMGovernanceOverviewItem `json:"items"`
}
// InstanceSessionUsageOverviewQuery contains query options for admin session usage overview.
type InstanceSessionUsageOverviewQuery struct {
Page int
Limit int
Search string
Since *time.Time
Until *time.Time
}
// InstanceSessionUsageOverviewItem summarizes session usage for one managed runtime instance.
type InstanceSessionUsageOverviewItem struct {
InstanceID int `json:"instance_id"`
InstanceName string `json:"instance_name"`
InstanceType string `json:"instance_type"`
UserID int `json:"user_id"`
Summary InstanceSessionUsageSummary `json:"summary"`
Compliance InstanceSessionUsageCompliance `json:"compliance"`
}
// InstanceSessionUsageOverview aggregates session usage across managed runtime instances.
type InstanceSessionUsageOverview struct {
Summary InstanceSessionUsageSummary `json:"summary"`
Items []InstanceSessionUsageOverviewItem `json:"items"`
Total int `json:"total"`
Page int `json:"page"`
Limit int `json:"limit"`
}
// AIObservabilityService provides read APIs for audit and cost reporting.
type AIObservabilityService interface {
ListAuditItems(query AuditQuery) (*AuditListResult, error)
GetTraceDetail(traceID string) (*AuditTraceDetail, error)
GetCostOverview(query CostQuery) (*CostOverview, error)
GetInstanceSessionUsage(instanceID int, query InstanceSessionUsageQuery) (*InstanceSessionUsageResult, error)
GetInstanceSessionUsageDetail(instanceID int, sessionID string, filter repository.SessionUsageFilter) (*InstanceSessionUsageDetail, error)
GetInstanceLLMGovernanceStatus(instanceID int, runtimeSystemInfo map[string]interface{}) (*InstanceLLMGovernanceStatus, error)
GetLLMGovernanceOverview() (*LLMGovernanceOverview, error)
GetAdminSessionUsageOverview(query InstanceSessionUsageOverviewQuery) (*InstanceSessionUsageOverview, error)
}
type aiObservabilityService struct {
@@ -173,9 +316,11 @@ type aiObservabilityService struct {
costRepo repository.CostRecordRepository
riskHitRepo repository.RiskHitRepository
chatMessageRepo repository.ChatMessageRepository
chatSessionRepo repository.ChatSessionRepository
llmModelRepo repository.LLMModelRepository
instanceRepo repository.InstanceRepository
userRepo repository.UserRepository
instanceRepo repository.InstanceRepository
userRepo repository.UserRepository
runtimeStatusRepo repository.InstanceRuntimeStatusRepository
}
// NewAIObservabilityService creates a new observability reporting service.
@@ -185,19 +330,23 @@ func NewAIObservabilityService(
costRepo repository.CostRecordRepository,
riskHitRepo repository.RiskHitRepository,
chatMessageRepo repository.ChatMessageRepository,
chatSessionRepo repository.ChatSessionRepository,
llmModelRepo repository.LLMModelRepository,
instanceRepo repository.InstanceRepository,
userRepo repository.UserRepository,
runtimeStatusRepo repository.InstanceRuntimeStatusRepository,
) AIObservabilityService {
return &aiObservabilityService{
invocationRepo: invocationRepo,
auditRepo: auditRepo,
costRepo: costRepo,
riskHitRepo: riskHitRepo,
chatMessageRepo: chatMessageRepo,
llmModelRepo: llmModelRepo,
instanceRepo: instanceRepo,
userRepo: userRepo,
invocationRepo: invocationRepo,
auditRepo: auditRepo,
costRepo: costRepo,
riskHitRepo: riskHitRepo,
chatMessageRepo: chatMessageRepo,
chatSessionRepo: chatSessionRepo,
llmModelRepo: llmModelRepo,
instanceRepo: instanceRepo,
userRepo: userRepo,
runtimeStatusRepo: runtimeStatusRepo,
}
}
@@ -1847,3 +1996,474 @@ func valueOrCostTotalTokens(cost *models.CostRecord) int {
func pointerToInt(value int) *int {
return &value
}
func (s *aiObservabilityService) GetInstanceSessionUsage(instanceID int, query InstanceSessionUsageQuery) (*InstanceSessionUsageResult, error) {
page, limit := normalizePageLimit(query.Page, query.Limit, 20, 100)
filter := repository.SessionUsageFilter{Since: query.Since, Until: query.Until}
allItems, err := s.mergeInstanceSessionUsageItems(instanceID, filter)
if err != nil {
return nil, err
}
items := filterSessionUsageItems(allItems, query.Search)
summary, compliance := summarizeSessionUsageItems(allItems, filter, instanceID, s.invocationRepo, s.auditRepo)
total := len(items)
start := (page - 1) * limit
if start > total {
start = total
}
end := start + limit
if end > total {
end = total
}
paged := make([]InstanceSessionUsageItem, 0, end-start)
if start < end {
paged = append(paged, items[start:end]...)
}
return &InstanceSessionUsageResult{
Summary: summary,
Compliance: compliance,
Items: paged,
Total: total,
Page: page,
Limit: limit,
}, nil
}
func (s *aiObservabilityService) GetInstanceSessionUsageDetail(instanceID int, sessionID string, filter repository.SessionUsageFilter) (*InstanceSessionUsageDetail, error) {
sessionID = strings.TrimSpace(sessionID)
if sessionID == "" {
return nil, fmt.Errorf("session id is required")
}
items, err := s.mergeInstanceSessionUsageItems(instanceID, filter)
if err != nil {
return nil, err
}
var matched *InstanceSessionUsageItem
for index := range items {
if items[index].SessionID == sessionID {
matched = &items[index]
break
}
}
if matched == nil {
return nil, fmt.Errorf("session usage not found")
}
invocations, err := s.invocationRepo.ListRecentByInstanceSession(instanceID, sessionID, 20, filter)
if err != nil {
return nil, fmt.Errorf("failed to list recent invocations for session: %w", err)
}
modelTotals := map[string]*CostBreakdownItem{}
recentTraces := make([]InstanceSessionTrace, 0, len(invocations))
for _, invocation := range invocations {
modelRow := modelTotals[invocation.RequestedModel]
if modelRow == nil {
modelRow = &CostBreakdownItem{Label: invocation.RequestedModel}
modelTotals[invocation.RequestedModel] = modelRow
}
modelRow.PromptTokens += invocation.PromptTokens
modelRow.CompletionTokens += invocation.CompletionTokens
modelRow.TotalTokens += invocation.TotalTokens
if s.costRepo != nil {
if costs, costErr := s.costRepo.ListByTraceID(invocation.TraceID); costErr == nil {
for _, cost := range costs {
modelRow.EstimatedCost += cost.EstimatedCost
}
}
}
recentTraces = append(recentTraces, InstanceSessionTrace{
TraceID: invocation.TraceID,
RequestedModel: invocation.RequestedModel,
Status: invocation.Status,
PromptTokens: invocation.PromptTokens,
CompletionTokens: invocation.CompletionTokens,
TotalTokens: invocation.TotalTokens,
CreatedAt: invocation.CreatedAt,
})
}
return &InstanceSessionUsageDetail{
SessionID: matched.SessionID,
SessionKey: matched.SessionKey,
Title: matched.Title,
PromptTokens: matched.PromptTokens,
CompletionTokens: matched.CompletionTokens,
TotalTokens: matched.TotalTokens,
EstimatedCost: matched.EstimatedCost,
Currency: matched.Currency,
InvocationCount: matched.InvocationCount,
FirstSeenAt: matched.FirstSeenAt,
LastSeenAt: matched.LastSeenAt,
ModelBreakdown: s.completeModelBreakdowns(modelTotals),
RecentTraces: recentTraces,
}, nil
}
func (s *aiObservabilityService) GetInstanceLLMGovernanceStatus(instanceID int, runtimeSystemInfo map[string]interface{}) (*InstanceLLMGovernanceStatus, error) {
items, err := s.mergeInstanceSessionUsageItems(instanceID, repository.SessionUsageFilter{})
if err != nil {
return nil, err
}
fallbackCount := 0
for _, item := range items {
if utils.IsTraceFallbackSessionID(item.SessionID) {
fallbackCount++
}
}
fallbackRate := 0.0
if len(items) > 0 {
fallbackRate = float64(fallbackCount) / float64(len(items))
}
configStatus := classifyLLMConfigStatusFromSystemInfo(runtimeSystemInfo)
since := time.Now().UTC().Add(-24 * time.Hour)
egressBlockCount := 0
if s.auditRepo != nil {
if count, err := s.auditRepo.CountRecentByInstanceAndEventType(instanceID, "egress.llm.blocked", since); err == nil {
egressBlockCount = count
}
}
isCompliant := fallbackRate == 0 && configStatus != "external"
if configStatus == "gateway" {
isCompliant = fallbackRate == 0
}
status := &InstanceLLMGovernanceStatus{
ConfigStatus: configStatus,
SessionFallbackRate: fallbackRate,
RecentEgressBlockCount: egressBlockCount,
IsCompliant: isCompliant,
}
return status, nil
}
func (s *aiObservabilityService) GetLLMGovernanceOverview() (*LLMGovernanceOverview, error) {
if s.instanceRepo == nil {
return &LLMGovernanceOverview{Items: []LLMGovernanceOverviewItem{}}, nil
}
instances, err := s.instanceRepo.GetAllRunning()
if err != nil {
return nil, fmt.Errorf("failed to list running instances: %w", err)
}
overview := &LLMGovernanceOverview{
Items: make([]LLMGovernanceOverviewItem, 0),
}
for _, instance := range instances {
if !supportsManagedRuntimeIntegration(instance.Type) {
continue
}
systemInfo := decodeRuntimeSystemInfo(s.runtimeStatusRepo, instance.ID)
status, err := s.GetInstanceLLMGovernanceStatus(instance.ID, systemInfo)
if err != nil {
return nil, err
}
if status == nil {
continue
}
item := LLMGovernanceOverviewItem{
InstanceID: instance.ID,
InstanceName: instance.Name,
InstanceType: instance.Type,
UserID: instance.UserID,
ConfigStatus: status.ConfigStatus,
SessionFallbackRate: status.SessionFallbackRate,
RecentEgressBlockCount: status.RecentEgressBlockCount,
IsCompliant: status.IsCompliant,
}
overview.TotalManagedInstances++
overview.Items = append(overview.Items, item)
if !status.IsCompliant {
overview.NonCompliantCount++
}
if status.ConfigStatus == "external" {
overview.ExternalConfigCount++
}
if status.SessionFallbackRate > 0 {
overview.HighFallbackCount++
}
}
return overview, nil
}
func (s *aiObservabilityService) GetAdminSessionUsageOverview(query InstanceSessionUsageOverviewQuery) (*InstanceSessionUsageOverview, error) {
if s.instanceRepo == nil {
return &InstanceSessionUsageOverview{Items: []InstanceSessionUsageOverviewItem{}}, nil
}
page, limit := normalizePageLimit(query.Page, query.Limit, 20, 100)
filter := repository.SessionUsageFilter{Since: query.Since, Until: query.Until}
search := strings.ToLower(strings.TrimSpace(query.Search))
instances, err := s.instanceRepo.GetAllRunning()
if err != nil {
return nil, fmt.Errorf("failed to list running instances: %w", err)
}
allItems := make([]InstanceSessionUsageOverviewItem, 0)
globalSummary := InstanceSessionUsageSummary{Currency: "USD"}
globalCurrencyCounts := map[string]int{}
for _, instance := range instances {
if !supportsManagedRuntimeIntegration(instance.Type) {
continue
}
if search != "" {
nameMatch := strings.Contains(strings.ToLower(instance.Name), search)
idMatch := strings.Contains(strconv.Itoa(instance.ID), search)
typeMatch := strings.Contains(strings.ToLower(instance.Type), search)
if !nameMatch && !idMatch && !typeMatch {
continue
}
}
sessionItems, mergeErr := s.mergeInstanceSessionUsageItems(instance.ID, filter)
if mergeErr != nil {
return nil, mergeErr
}
summary, compliance := summarizeSessionUsageItems(sessionItems, filter, instance.ID, s.invocationRepo, s.auditRepo)
item := InstanceSessionUsageOverviewItem{
InstanceID: instance.ID,
InstanceName: instance.Name,
InstanceType: instance.Type,
UserID: instance.UserID,
Summary: summary,
Compliance: compliance,
}
allItems = append(allItems, item)
globalSummary.TotalPromptTokens += summary.TotalPromptTokens
globalSummary.TotalCompletionTokens += summary.TotalCompletionTokens
globalSummary.TotalTokens += summary.TotalTokens
globalSummary.TotalEstimatedCost += summary.TotalEstimatedCost
globalSummary.SessionCount += summary.SessionCount
if strings.TrimSpace(summary.Currency) != "" {
globalCurrencyCounts[summary.Currency]++
}
}
if len(globalCurrencyCounts) > 0 {
globalSummary.Currency = pickDominantCurrency(globalCurrencyCounts)
}
sort.Slice(allItems, func(i, j int) bool {
if allItems[i].Summary.TotalTokens == allItems[j].Summary.TotalTokens {
return allItems[i].InstanceID > allItems[j].InstanceID
}
return allItems[i].Summary.TotalTokens > allItems[j].Summary.TotalTokens
})
total := len(allItems)
start := (page - 1) * limit
if start > total {
start = total
}
end := start + limit
if end > total {
end = total
}
paged := make([]InstanceSessionUsageOverviewItem, 0, end-start)
if start < end {
paged = append(paged, allItems[start:end]...)
}
return &InstanceSessionUsageOverview{
Summary: globalSummary,
Items: paged,
Total: total,
Page: page,
Limit: limit,
}, nil
}
func summarizeSessionUsageItems(
allItems []InstanceSessionUsageItem,
filter repository.SessionUsageFilter,
instanceID int,
invocationRepo repository.ModelInvocationRepository,
auditRepo repository.AuditEventRepository,
) (InstanceSessionUsageSummary, InstanceSessionUsageCompliance) {
summary := InstanceSessionUsageSummary{Currency: "USD"}
compliance := InstanceSessionUsageCompliance{}
currencyCounts := map[string]int{}
for _, item := range allItems {
summary.TotalPromptTokens += item.PromptTokens
summary.TotalCompletionTokens += item.CompletionTokens
summary.TotalTokens += item.TotalTokens
summary.TotalEstimatedCost += item.EstimatedCost
if strings.TrimSpace(item.Currency) != "" {
currencyCounts[item.Currency]++
}
if utils.IsTraceFallbackSessionID(item.SessionID) {
compliance.FallbackSessionCount++
}
}
summary.SessionCount = len(allItems)
if invocationRepo != nil {
if count, countErr := invocationRepo.CountDistinctSessionsByInstance(instanceID, filter); countErr == nil {
summary.SessionCount = count
}
}
compliance.HasFallbackSessions = compliance.FallbackSessionCount > 0
if auditRepo != nil {
since := time.Now().UTC().Add(-24 * time.Hour)
if filter.Since != nil && filter.Since.After(since) {
since = *filter.Since
}
if count, err := auditRepo.CountRecentByInstanceAndEventType(instanceID, "gateway.session.fallback", since); err == nil {
compliance.RecentFallbackAuditCount = count
}
}
if len(currencyCounts) > 0 {
summary.Currency = pickDominantCurrency(currencyCounts)
}
return summary, compliance
}
func pickDominantCurrency(currencyCounts map[string]int) string {
bestCurrency := "USD"
bestCount := 0
for currency, count := range currencyCounts {
if count > bestCount {
bestCurrency = currency
bestCount = count
}
}
return bestCurrency
}
func decodeRuntimeSystemInfo(runtimeRepo repository.InstanceRuntimeStatusRepository, instanceID int) map[string]interface{} {
if runtimeRepo == nil {
return nil
}
status, err := runtimeRepo.GetByInstanceID(instanceID)
if err != nil || status == nil || status.SystemInfoJSON == nil {
return nil
}
raw := strings.TrimSpace(*status.SystemInfoJSON)
if raw == "" {
return nil
}
systemInfo := map[string]interface{}{}
if err := json.Unmarshal([]byte(raw), &systemInfo); err != nil {
return nil
}
return systemInfo
}
func (s *aiObservabilityService) mergeInstanceSessionUsageItems(instanceID int, filter repository.SessionUsageFilter) ([]InstanceSessionUsageItem, error) {
tokenAggs, err := s.invocationRepo.AggregateByInstanceSession(instanceID, filter)
if err != nil {
return nil, fmt.Errorf("failed to aggregate session tokens: %w", err)
}
costAggs, err := s.costRepo.AggregateCostByInstanceSession(instanceID, filter)
if err != nil {
return nil, fmt.Errorf("failed to aggregate session costs: %w", err)
}
sessionMeta := map[string]models.ChatSession{}
if s.chatSessionRepo != nil {
sessions, listErr := s.chatSessionRepo.ListByInstanceID(instanceID)
if listErr != nil {
return nil, fmt.Errorf("failed to list chat sessions: %w", listErr)
}
for _, session := range sessions {
sessionMeta[session.SessionID] = session
}
}
costBySession := map[string]repository.InstanceSessionCostAggregate{}
for _, cost := range costAggs {
costBySession[cost.SessionID] = cost
}
items := make([]InstanceSessionUsageItem, 0, len(tokenAggs))
for _, token := range tokenAggs {
item := InstanceSessionUsageItem{
SessionID: token.SessionID,
SessionKey: utils.FormatOpenClawSessionKey(token.SessionID),
PromptTokens: token.PromptTokens,
CompletionTokens: token.CompletionTokens,
TotalTokens: token.TotalTokens,
InvocationCount: token.InvocationCount,
FirstSeenAt: token.FirstSeenAt,
LastSeenAt: token.LastSeenAt,
Currency: "USD",
}
if cost, ok := costBySession[token.SessionID]; ok {
item.EstimatedCost = cost.EstimatedCost
if strings.TrimSpace(cost.Currency) != "" {
item.Currency = cost.Currency
}
}
if session, ok := sessionMeta[token.SessionID]; ok && session.Title != nil {
item.Title = session.Title
}
items = append(items, item)
}
sort.Slice(items, func(i, j int) bool {
return items[i].LastSeenAt.After(items[j].LastSeenAt)
})
return items, nil
}
func filterSessionUsageItems(items []InstanceSessionUsageItem, search string) []InstanceSessionUsageItem {
search = strings.ToLower(strings.TrimSpace(search))
if search == "" {
return items
}
filtered := make([]InstanceSessionUsageItem, 0, len(items))
for _, item := range items {
haystacks := []string{
strings.ToLower(item.SessionID),
strings.ToLower(item.SessionKey),
}
if item.Title != nil {
haystacks = append(haystacks, strings.ToLower(*item.Title))
}
matched := false
for _, candidate := range haystacks {
if strings.Contains(candidate, search) {
matched = true
break
}
}
if matched {
filtered = append(filtered, item)
}
}
return filtered
}
func classifyLLMConfigStatusFromSystemInfo(systemInfo map[string]interface{}) string {
if len(systemInfo) == 0 {
return "unknown"
}
if raw, ok := systemInfo["llm_config_status"]; ok {
if value, ok := raw.(string); ok && strings.TrimSpace(value) != "" {
return strings.TrimSpace(value)
}
}
if raw, ok := systemInfo["llm_provider_base_url"]; ok {
if value, ok := raw.(string); ok {
lower := strings.ToLower(strings.TrimSpace(value))
switch {
case strings.Contains(lower, "gateway/llm"), strings.Contains(lower, "clawmanager"):
return "gateway"
case strings.Contains(lower, "openai.com"), strings.Contains(lower, "anthropic.com"):
return "external"
}
}
}
return "unknown"
}
backend/internal/services/ai_observability_session_usage_test.go
+400
View File
@@ -0,0 +1,400 @@
package services
import (
"testing"
"time"
"clawreef/internal/models"
"clawreef/internal/repository"
)
type stubSessionUsageInvocationRepo struct {
aggregates []repository.InstanceSessionTokenAggregate
invocations []models.ModelInvocation
}
func (s *stubSessionUsageInvocationRepo) Create(*models.ModelInvocation) error { return nil }
func (s *stubSessionUsageInvocationRepo) GetByID(int) (*models.ModelInvocation, error) {
return nil, nil
}
func (s *stubSessionUsageInvocationRepo) ListByTraceID(string) ([]models.ModelInvocation, error) {
return nil, nil
}
func (s *stubSessionUsageInvocationRepo) ListBySessionID(string, int) ([]models.ModelInvocation, error) {
return nil, nil
}
func (s *stubSessionUsageInvocationRepo) ListByUserID(int, int) ([]models.ModelInvocation, error) {
return nil, nil
}
func (s *stubSessionUsageInvocationRepo) ListRecent(int) ([]models.ModelInvocation, error) {
return nil, nil
}
func (s *stubSessionUsageInvocationRepo) AggregateByInstanceSession(int, repository.SessionUsageFilter) ([]repository.InstanceSessionTokenAggregate, error) {
return s.aggregates, nil
}
func (s *stubSessionUsageInvocationRepo) ListRecentByInstanceSession(int, string, int, repository.SessionUsageFilter) ([]models.ModelInvocation, error) {
return s.invocations, nil
}
func (s *stubSessionUsageInvocationRepo) CountDistinctSessionsByInstance(int, repository.SessionUsageFilter) (int, error) {
return len(s.aggregates), nil
}
type stubSessionUsageCostRepo struct {
aggregates []repository.InstanceSessionCostAggregate
byTraceID map[string][]models.CostRecord
}
func (s *stubSessionUsageCostRepo) Create(*models.CostRecord) error { return nil }
func (s *stubSessionUsageCostRepo) ListByTraceID(traceID string) ([]models.CostRecord, error) {
if s.byTraceID == nil {
return nil, nil
}
return s.byTraceID[traceID], nil
}
func (s *stubSessionUsageCostRepo) ListByUserID(int, int) ([]models.CostRecord, error) {
return nil, nil
}
func (s *stubSessionUsageCostRepo) ListRecent(int) ([]models.CostRecord, error) { return nil, nil }
func (s *stubSessionUsageCostRepo) AggregateCostByInstanceSession(int, repository.SessionUsageFilter) ([]repository.InstanceSessionCostAggregate, error) {
return s.aggregates, nil
}
type stubSessionUsageChatSessionRepo struct {
sessions []models.ChatSession
}
func (s *stubSessionUsageChatSessionRepo) GetBySessionID(string) (*models.ChatSession, error) {
return nil, nil
}
func (s *stubSessionUsageChatSessionRepo) ListByInstanceID(int) ([]models.ChatSession, error) {
return s.sessions, nil
}
func (s *stubSessionUsageChatSessionRepo) Save(*models.ChatSession) error { return nil }
type stubSessionUsageAuditRepo struct {
counts map[string]int
}
func (s *stubSessionUsageAuditRepo) Create(*models.AuditEvent) error { return nil }
func (s *stubSessionUsageAuditRepo) ListByTraceID(string) ([]models.AuditEvent, error) {
return nil, nil
}
func (s *stubSessionUsageAuditRepo) ListRecent(int) ([]models.AuditEvent, error) {
return nil, nil
}
func (s *stubSessionUsageAuditRepo) CountRecentByInstanceAndEventType(instanceID int, eventType string, since time.Time) (int, error) {
if s.counts == nil {
return 0, nil
}
return s.counts[eventType], nil
}
func TestGetInstanceSessionUsageMergesInvocationCostAndSessionMetadata(t *testing.T) {
now := time.Date(2026, 7, 2, 10, 0, 0, 0, time.UTC)
title := "Weather chat"
service := &aiObservabilityService{
invocationRepo: &stubSessionUsageInvocationRepo{
aggregates: []repository.InstanceSessionTokenAggregate{
{
SessionID: "agent:openclaw:main",
PromptTokens: 100,
CompletionTokens: 40,
TotalTokens: 140,
InvocationCount: 2,
FirstSeenAt: now.Add(-time.Hour),
LastSeenAt: now,
},
},
},
costRepo: &stubSessionUsageCostRepo{
aggregates: []repository.InstanceSessionCostAggregate{
{
SessionID: "agent:openclaw:main",
EstimatedCost: 0.12,
Currency: "USD",
},
},
},
chatSessionRepo: &stubSessionUsageChatSessionRepo{
sessions: []models.ChatSession{
{SessionID: "agent:openclaw:main", Title: &title},
},
},
}
result, err := service.GetInstanceSessionUsage(9, InstanceSessionUsageQuery{Page: 1, Limit: 10})
if err != nil {
t.Fatalf("GetInstanceSessionUsage failed: %v", err)
}
if len(result.Items) != 1 {
t.Fatalf("expected 1 item, got %d", len(result.Items))
}
item := result.Items[0]
if item.SessionKey != "main" || item.TotalTokens != 140 || item.EstimatedCost != 0.12 {
t.Fatalf("unexpected merged item: %+v", item)
}
if item.Title == nil || *item.Title != title {
t.Fatalf("expected title merge, got %+v", item.Title)
}
}
func TestGetInstanceSessionUsageSearchFiltersBySessionKey(t *testing.T) {
now := time.Now().UTC()
service := &aiObservabilityService{
invocationRepo: &stubSessionUsageInvocationRepo{
aggregates: []repository.InstanceSessionTokenAggregate{
{SessionID: "agent:openclaw:main", TotalTokens: 10, LastSeenAt: now},
{SessionID: "agent:openclaw:research", TotalTokens: 20, LastSeenAt: now},
},
},
costRepo: &stubSessionUsageCostRepo{},
chatSessionRepo: &stubSessionUsageChatSessionRepo{},
}
result, err := service.GetInstanceSessionUsage(9, InstanceSessionUsageQuery{Page: 1, Limit: 10, Search: "research"})
if err != nil {
t.Fatalf("GetInstanceSessionUsage failed: %v", err)
}
if len(result.Items) != 1 || result.Items[0].SessionKey != "research" {
t.Fatalf("unexpected filtered items: %+v", result.Items)
}
if result.Summary.SessionCount != 2 || result.Summary.TotalTokens != 30 {
t.Fatalf("summary should ignore search filter, got %+v", result.Summary)
}
}
func TestGetInstanceSessionUsageComplianceCountsFallbackSessions(t *testing.T) {
now := time.Now().UTC()
service := &aiObservabilityService{
invocationRepo: &stubSessionUsageInvocationRepo{
aggregates: []repository.InstanceSessionTokenAggregate{
{SessionID: "agent:openclaw:main", TotalTokens: 10, LastSeenAt: now},
{SessionID: "sess_trc_abc", TotalTokens: 5, LastSeenAt: now},
},
},
costRepo: &stubSessionUsageCostRepo{},
chatSessionRepo: &stubSessionUsageChatSessionRepo{},
}
result, err := service.GetInstanceSessionUsage(9, InstanceSessionUsageQuery{Page: 1, Limit: 10})
if err != nil {
t.Fatalf("GetInstanceSessionUsage failed: %v", err)
}
if !result.Compliance.HasFallbackSessions || result.Compliance.FallbackSessionCount != 1 {
t.Fatalf("unexpected compliance: %+v", result.Compliance)
}
}
func TestGetInstanceSessionUsageComplianceIncludesFallbackAuditCount(t *testing.T) {
now := time.Now().UTC()
service := &aiObservabilityService{
invocationRepo: &stubSessionUsageInvocationRepo{
aggregates: []repository.InstanceSessionTokenAggregate{
{SessionID: "agent:openclaw:main", TotalTokens: 10, LastSeenAt: now},
},
},
costRepo: &stubSessionUsageCostRepo{},
chatSessionRepo: &stubSessionUsageChatSessionRepo{},
auditRepo: &stubSessionUsageAuditRepo{
counts: map[string]int{"gateway.session.fallback": 3},
},
}
result, err := service.GetInstanceSessionUsage(9, InstanceSessionUsageQuery{Page: 1, Limit: 10})
if err != nil {
t.Fatalf("GetInstanceSessionUsage failed: %v", err)
}
if result.Compliance.RecentFallbackAuditCount != 3 {
t.Fatalf("expected recent fallback audit count 3, got %+v", result.Compliance)
}
}
func TestGetInstanceSessionUsageDetailBuildsModelBreakdown(t *testing.T) {
now := time.Now().UTC()
service := &aiObservabilityService{
invocationRepo: &stubSessionUsageInvocationRepo{
aggregates: []repository.InstanceSessionTokenAggregate{
{
SessionID: "agent:openclaw:main",
TotalTokens: 30,
LastSeenAt: now,
FirstSeenAt: now,
InvocationCount: 1,
},
},
invocations: []models.ModelInvocation{
{
TraceID: "trc_1",
RequestedModel: "auto",
Status: models.ModelInvocationStatusCompleted,
PromptTokens: 20,
CompletionTokens: 10,
TotalTokens: 30,
CreatedAt: now,
},
},
},
costRepo: &stubSessionUsageCostRepo{
byTraceID: map[string][]models.CostRecord{
"trc_1": {
{TraceID: "trc_1", EstimatedCost: 0.05, Currency: "USD"},
},
},
},
chatSessionRepo: &stubSessionUsageChatSessionRepo{},
llmModelRepo: &stubLLMModelRepository{},
}
detail, err := service.GetInstanceSessionUsageDetail(9, "agent:openclaw:main", repository.SessionUsageFilter{})
if err != nil {
t.Fatalf("GetInstanceSessionUsageDetail failed: %v", err)
}
if len(detail.ModelBreakdown) != 1 || detail.ModelBreakdown[0].Label != "auto" {
t.Fatalf("unexpected model breakdown: %+v", detail.ModelBreakdown)
}
if detail.ModelBreakdown[0].EstimatedCost != 0.05 {
t.Fatalf("expected model breakdown cost 0.05, got %+v", detail.ModelBreakdown[0])
}
if len(detail.RecentTraces) != 1 || detail.RecentTraces[0].TraceID != "trc_1" {
t.Fatalf("unexpected recent traces: %+v", detail.RecentTraces)
}
}
func TestGetInstanceLLMGovernanceStatusUnknownConfigUsesFallbackOnly(t *testing.T) {
now := time.Now().UTC()
service := &aiObservabilityService{
invocationRepo: &stubSessionUsageInvocationRepo{
aggregates: []repository.InstanceSessionTokenAggregate{
{SessionID: "agent:openclaw:main", TotalTokens: 10, LastSeenAt: now, FirstSeenAt: now},
},
},
costRepo: &stubSessionUsageCostRepo{},
chatSessionRepo: &stubSessionUsageChatSessionRepo{},
auditRepo: &stubSessionUsageAuditRepo{
counts: map[string]int{"egress.llm.blocked": 2},
},
}
status, err := service.GetInstanceLLMGovernanceStatus(9, map[string]interface{}{})
if err != nil {
t.Fatalf("GetInstanceLLMGovernanceStatus failed: %v", err)
}
if !status.IsCompliant || status.ConfigStatus != "unknown" || status.RecentEgressBlockCount != 2 {
t.Fatalf("unexpected governance status: %+v", status)
}
}
func TestGetInstanceLLMGovernanceStatusExternalConfigIsNonCompliant(t *testing.T) {
now := time.Now().UTC()
service := &aiObservabilityService{
invocationRepo: &stubSessionUsageInvocationRepo{
aggregates: []repository.InstanceSessionTokenAggregate{
{SessionID: "agent:openclaw:main", TotalTokens: 10, LastSeenAt: now, FirstSeenAt: now},
},
},
costRepo: &stubSessionUsageCostRepo{},
chatSessionRepo: &stubSessionUsageChatSessionRepo{},
}
status, err := service.GetInstanceLLMGovernanceStatus(9, map[string]interface{}{
"llm_config_status": "external",
})
if err != nil {
t.Fatalf("GetInstanceLLMGovernanceStatus failed: %v", err)
}
if status.IsCompliant {
t.Fatalf("expected external config to be non-compliant")
}
}
func TestGetAdminSessionUsageOverviewAggregatesManagedInstances(t *testing.T) {
now := time.Now().UTC()
service := &aiObservabilityService{
invocationRepo: &stubSessionUsageInvocationRepo{
aggregates: []repository.InstanceSessionTokenAggregate{
{SessionID: "agent:openclaw:main", TotalTokens: 100, LastSeenAt: now, FirstSeenAt: now},
},
},
costRepo: &stubSessionUsageCostRepo{},
chatSessionRepo: &stubSessionUsageChatSessionRepo{},
instanceRepo: &stubGovernanceInstanceRepo{
instances: []models.Instance{
{ID: 1, UserID: 9, Name: "oc-1", Type: "openclaw", Status: "running"},
{ID: 2, UserID: 10, Name: "ubuntu", Type: "ubuntu", Status: "running"},
},
},
}
overview, err := service.GetAdminSessionUsageOverview(InstanceSessionUsageOverviewQuery{Page: 1, Limit: 10})
if err != nil {
t.Fatalf("GetAdminSessionUsageOverview failed: %v", err)
}
if overview.Total != 1 || len(overview.Items) != 1 {
t.Fatalf("expected 1 managed instance item, got total=%d items=%d", overview.Total, len(overview.Items))
}
if overview.Summary.TotalTokens != 100 {
t.Fatalf("unexpected global summary: %+v", overview.Summary)
}
}
func TestGetAdminSessionUsageOverviewFiltersBySearch(t *testing.T) {
now := time.Now().UTC()
service := &aiObservabilityService{
invocationRepo: &stubSessionUsageInvocationRepo{
aggregates: []repository.InstanceSessionTokenAggregate{
{SessionID: "agent:openclaw:main", TotalTokens: 100, LastSeenAt: now, FirstSeenAt: now},
},
},
costRepo: &stubSessionUsageCostRepo{},
chatSessionRepo: &stubSessionUsageChatSessionRepo{},
instanceRepo: &stubGovernanceInstanceRepo{
instances: []models.Instance{
{ID: 1, UserID: 9, Name: "alpha-openclaw", Type: "openclaw", Status: "running"},
{ID: 2, UserID: 9, Name: "beta-openclaw", Type: "openclaw", Status: "running"},
},
},
}
overview, err := service.GetAdminSessionUsageOverview(InstanceSessionUsageOverviewQuery{
Page: 1,
Limit: 10,
Search: "alpha",
})
if err != nil {
t.Fatalf("GetAdminSessionUsageOverview failed: %v", err)
}
if overview.Total != 1 || len(overview.Items) != 1 || overview.Items[0].InstanceName != "alpha-openclaw" {
t.Fatalf("unexpected filtered overview: total=%d items=%+v", overview.Total, overview.Items)
}
}
func TestGetAdminSessionUsageOverviewAcceptsSinceQuery(t *testing.T) {
now := time.Now().UTC()
since := now.Add(-24 * time.Hour)
service := &aiObservabilityService{
invocationRepo: &stubSessionUsageInvocationRepo{
aggregates: []repository.InstanceSessionTokenAggregate{
{SessionID: "agent:openclaw:main", TotalTokens: 50, LastSeenAt: now, FirstSeenAt: now},
},
},
costRepo: &stubSessionUsageCostRepo{},
chatSessionRepo: &stubSessionUsageChatSessionRepo{},
instanceRepo: &stubGovernanceInstanceRepo{
instances: []models.Instance{
{ID: 1, UserID: 9, Name: "oc-1", Type: "openclaw", Status: "running"},
},
},
}
overview, err := service.GetAdminSessionUsageOverview(InstanceSessionUsageOverviewQuery{
Page: 1,
Limit: 10,
Since: &since,
})
if err != nil {
t.Fatalf("GetAdminSessionUsageOverview failed: %v", err)
}
if overview.Total != 1 || overview.Summary.TotalTokens != 50 {
t.Fatalf("unexpected since-filtered overview: total=%d summary=%+v", overview.Total, overview.Summary)
}
}
backend/internal/services/instance_env.go
+67
View File
@@ -3,6 +3,7 @@ package services
import (
"encoding/json"
"fmt"
"os"
"regexp"
"strconv"
"strings"
@@ -10,6 +11,66 @@ import (
"clawreef/internal/models"
)
var protectedManagedRuntimeEnvKeys = map[string]struct{}{
"CLAWMANAGER_LLM_BASE_URL": {},
"CLAWMANAGER_LLM_API_KEY": {},
"CLAWMANAGER_LLM_MODEL": {},
"CLAWMANAGER_LLM_PROVIDER": {},
"CLAWMANAGER_INSTANCE_TOKEN": {},
"OPENAI_BASE_URL": {},
"OPENAI_API_BASE": {},
"OPENAI_API_KEY": {},
"OPENAI_MODEL": {},
}
func isLLMGovernanceStrictEnabled() bool {
switch strings.ToLower(strings.TrimSpace(os.Getenv("CLAWMANAGER_LLM_GOVERNANCE_STRICT"))) {
case "0", "false", "no", "off":
return false
default:
return true
}
}
func isInstanceNetworkLockEnabled() bool {
switch strings.ToLower(strings.TrimSpace(os.Getenv("CLAWMANAGER_INSTANCE_NETWORK_LOCK"))) {
case "1", "true", "yes", "on":
return true
default:
return false
}
}
func isProtectedManagedRuntimeEnvKey(key string) bool {
_, ok := protectedManagedRuntimeEnvKeys[strings.ToUpper(strings.TrimSpace(key))]
return ok
}
func validateManagedRuntimeEnvironmentOverrides(instanceType string, overrides map[string]string) error {
if !supportsManagedRuntimeIntegration(instanceType) || !isLLMGovernanceStrictEnabled() {
return nil
}
for key := range overrides {
if isProtectedManagedRuntimeEnvKey(key) {
return fmt.Errorf("environment override %s is managed by the platform", strings.ToUpper(strings.TrimSpace(key)))
}
}
return nil
}
func applyProtectedManagedRuntimeEnv(target, protected map[string]string) map[string]string {
if len(protected) == 0 {
return target
}
if target == nil {
target = map[string]string{}
}
for key, value := range protected {
target[key] = value
}
return target
}
const (
defaultInstanceSHMSizeGB = 1
maxInstanceSHMSizeGB = 8
@@ -127,6 +188,9 @@ func buildInstancePodEnv(instance *models.Instance, runtimeEnv, gatewayEnv, agen
delete(resolved, "SUBFOLDER")
}
resolved = mergeEnvMaps(resolved, overrides)
if supportsManagedRuntimeIntegration(instance.Type) && isLLMGovernanceStrictEnabled() {
resolved = applyProtectedManagedRuntimeEnv(resolved, mergeEnvMaps(gatewayEnv, agentEnv))
}
return resolved, nil
}
@@ -145,6 +209,9 @@ func buildInstanceGatewayEnv(instance *models.Instance, gatewayEnv map[string]st
resolved = withInstanceProxyEnv(instance.Type, instance.ID, resolved)
resolved["CLAWMANAGER_RUNTIME_TYPE"] = normalizeInstanceRuntimeType(instance.RuntimeType)
resolved = mergeEnvMaps(resolved, overrides)
if supportsManagedRuntimeIntegration(instance.Type) && isLLMGovernanceStrictEnabled() {
resolved = applyProtectedManagedRuntimeEnv(resolved, gatewayEnv)
}
return resolved, nil
}
backend/internal/services/instance_env_test.go
+41 -75
View File
@@ -1,25 +1,44 @@
package services
import (
"strings"
"testing"
"clawreef/internal/models"
)
func TestNormalizeEnvironmentOverrides(t *testing.T) {
overrides, err := normalizeEnvironmentOverrides(map[string]string{
" FOO ": "bar",
"BAR_2": "",
func TestValidateManagedRuntimeEnvironmentOverridesRejectsProtectedKeys(t *testing.T) {
err := validateManagedRuntimeEnvironmentOverrides("openclaw", map[string]string{
"OPENAI_BASE_URL": "https://api.openai.com/v1",
})
if err == nil {
t.Fatal("expected protected env override to be rejected")
}
}
func TestApplyProtectedManagedRuntimeEnvRestoresGatewayValues(t *testing.T) {
target := map[string]string{
"OPENAI_BASE_URL": "https://api.openai.com/v1",
"CUSTOM": "value",
}
protected := map[string]string{
"OPENAI_BASE_URL": "http://gateway.example/api/v1/gateway/llm",
}
result := applyProtectedManagedRuntimeEnv(target, protected)
if result["OPENAI_BASE_URL"] != protected["OPENAI_BASE_URL"] {
t.Fatalf("expected protected gateway url, got %q", result["OPENAI_BASE_URL"])
}
if result["CUSTOM"] != "value" {
t.Fatalf("expected custom override to remain")
}
}
func TestValidateManagedRuntimeEnvironmentOverridesAllowsCustomKeys(t *testing.T) {
err := validateManagedRuntimeEnvironmentOverrides("openclaw", map[string]string{
"CUSTOM_FLAG": "1",
})
if err != nil {
t.Fatalf("normalizeEnvironmentOverrides returned error: %v", err)
}
if overrides["FOO"] != "bar" {
t.Fatalf("expected trimmed key FOO to be preserved")
}
if value, ok := overrides["BAR_2"]; !ok || value != "" {
t.Fatalf("expected empty override value to be preserved")
t.Fatalf("expected custom override to be allowed, got %v", err)
}
}
@@ -82,73 +101,20 @@ func TestBuildInstancePodEnvAppliesOverridesAfterDefaults(t *testing.T) {
}
}
func TestBuildInstancePodEnvNormalizesDesktopStreamProfile(t *testing.T) {
raw, err := marshalEnvironmentOverrides(map[string]string{
"CLAWMANAGER_DESKTOP_STREAM_PROFILE": "standard",
"SELKIES_ENCODER": "x264enc",
"SELKIES_FRAMERATE": "35",
"SELKIES_H264_CRF": "34",
func TestValidateManagedRuntimeEnvironmentOverridesSkipsNonManagedTypes(t *testing.T) {
err := validateManagedRuntimeEnvironmentOverrides("ubuntu", map[string]string{
"OPENAI_BASE_URL": "https://api.openai.com/v1",
})
if err != nil {
t.Fatalf("marshalEnvironmentOverrides returned error: %v", err)
}
env, err := buildInstancePodEnv(&models.Instance{
ID: 42,
Type: "openclaw",
RuntimeType: RuntimeBackendDesktop,
EnvironmentOverridesJSON: raw,
}, nil, nil, nil)
if err != nil {
t.Fatalf("buildInstancePodEnv returned error: %v", err)
}
if got := env["SELKIES_ENCODER"]; got != "x264enc,jpeg" {
t.Fatalf("SELKIES_ENCODER = %q, want x264enc,jpeg", got)
}
if got := env["SELKIES_USE_CSS_SCALING"]; got != "true" {
t.Fatalf("SELKIES_USE_CSS_SCALING = %q, want true", got)
t.Fatalf("expected non-managed type to skip validation, got %v", err)
}
}
func TestPopSHMSizeGB(t *testing.T) {
tests := []struct {
name string
value string
hasValue bool
runtime string
memoryGB int
want int
}{
{name: "desktop minimum preset", runtime: "desktop", memoryGB: 4, want: defaultInstanceSHMSizeGB},
{name: "desktop medium preset", runtime: "desktop", memoryGB: 8, want: 2},
{name: "desktop large preset", runtime: "desktop", memoryGB: 12, want: 4},
{name: "desktop small fallback", runtime: "desktop", memoryGB: 2, want: defaultInstanceSHMSizeGB},
{name: "shell default unchanged", runtime: "shell", memoryGB: 8, want: defaultInstanceSHMSizeGB},
{name: "disable", value: "0", hasValue: true, runtime: "desktop", memoryGB: 8, want: 0},
{name: "custom", value: "4", hasValue: true, runtime: "desktop", memoryGB: 4, want: 4},
{name: "clamp", value: "128", hasValue: true, runtime: "desktop", memoryGB: 8, want: maxInstanceSHMSizeGB},
{name: "invalid keeps dynamic desktop default", value: "nope", hasValue: true, runtime: "desktop", memoryGB: 8, want: 2},
{name: "negative keeps dynamic desktop default", value: "-1", hasValue: true, runtime: "desktop", memoryGB: 4, want: defaultInstanceSHMSizeGB},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
extraEnv := map[string]string{"KEEP": "value"}
if tt.hasValue {
extraEnv["SHM_SIZE_GB"] = tt.value
}
got := popSHMSizeGB(extraEnv, tt.runtime, tt.memoryGB)
if got != tt.want {
t.Fatalf("expected shm size %d, got %d", tt.want, got)
}
if _, ok := extraEnv["SHM_SIZE_GB"]; ok {
t.Fatalf("expected SHM_SIZE_GB to be removed from extra env")
}
if extraEnv["KEEP"] != "value" {
t.Fatalf("expected unrelated env to be preserved")
}
})
func TestValidateManagedRuntimeEnvironmentOverridesErrorMessage(t *testing.T) {
err := validateManagedRuntimeEnvironmentOverrides("openclaw", map[string]string{
"openai_api_key": "sk-test",
})
if err == nil || !strings.Contains(err.Error(), "OPENAI_API_KEY") {
t.Fatalf("expected normalized key in error, got %v", err)
}
}
backend/internal/services/instance_runtime.go
+1
View File
@@ -142,6 +142,7 @@ func withInstanceProxyEnv(instanceType string, instanceID int, env map[string]st
merged["https_proxy"] = proxyURL
merged["NO_PROXY"] = noProxy
merged["no_proxy"] = noProxy
merged["CLAWMANAGER_EGRESS_INSTANCE_ID"] = fmt.Sprintf("%d", instanceID)
}
if usesWebtopImage(instanceType) {
backend/internal/services/instance_runtime_status_service.go
+28 -5
View File
@@ -19,10 +19,13 @@ type AgentStateReportRequest struct {
}
type AgentRuntimePayload struct {
OpenClawStatus string `json:"openclaw_status"`
OpenClawPID *int `json:"openclaw_pid,omitempty"`
OpenClawVersion string `json:"openclaw_version"`
CurrentConfigRevisionID *int `json:"current_config_revision_id,omitempty"`
OpenClawStatus string `json:"openclaw_status"`
OpenClawPID *int `json:"openclaw_pid,omitempty"`
OpenClawVersion string `json:"openclaw_version"`
CurrentConfigRevisionID *int `json:"current_config_revision_id,omitempty"`
LLMConfigFingerprint *string `json:"llm_config_fingerprint,omitempty"`
LLMConfigStatus *string `json:"llm_config_status,omitempty"`
LLMProviderBaseURL *string `json:"llm_provider_base_url,omitempty"`
}
type InstanceRuntimeStatusPayload struct {
@@ -90,7 +93,7 @@ func (s *instanceRuntimeStatusService) Report(session *AgentSession, req AgentSt
status.CurrentConfigRevisionID = req.Runtime.CurrentConfigRevisionID
status.LastReportedAt = reportedAt
systemInfoJSON, err := marshalOptionalJSON(req.SystemInfo)
systemInfoJSON, err := marshalOptionalJSON(mergeLLMConfigIntoSystemInfo(req.SystemInfo, req.Runtime))
if err != nil {
return fmt.Errorf("failed to encode system info: %w", err)
}
@@ -152,6 +155,26 @@ func (s *instanceRuntimeStatusService) GetByInstanceID(instanceID int) (*Instanc
return payload, nil
}
func mergeLLMConfigIntoSystemInfo(systemInfo map[string]interface{}, runtime AgentRuntimePayload) map[string]interface{} {
merged := map[string]interface{}{}
for key, value := range systemInfo {
merged[key] = value
}
if runtime.LLMConfigFingerprint != nil && strings.TrimSpace(*runtime.LLMConfigFingerprint) != "" {
merged["llm_config_fingerprint"] = strings.TrimSpace(*runtime.LLMConfigFingerprint)
}
if runtime.LLMConfigStatus != nil && strings.TrimSpace(*runtime.LLMConfigStatus) != "" {
merged["llm_config_status"] = strings.TrimSpace(*runtime.LLMConfigStatus)
}
if runtime.LLMProviderBaseURL != nil && strings.TrimSpace(*runtime.LLMProviderBaseURL) != "" {
merged["llm_provider_base_url"] = strings.TrimSpace(*runtime.LLMProviderBaseURL)
}
if len(merged) == 0 {
return nil
}
return merged
}
func (s *instanceRuntimeStatusService) UpsertInfraStatus(instanceID int, infraStatus string) error {
status, err := s.getOrCreate(instanceID)
if err != nil {
backend/internal/services/instance_service.go
+86 -37
View File
@@ -49,6 +49,9 @@ func (s *instanceService) ValidateCreateRequests(userID int, requests []CreateIn
if err != nil {
return err
}
if err := validateManagedRuntimeEnvironmentOverrides(requests[idx].Type, environmentOverrides); err != nil {
return err
}
if _, err := marshalEnvironmentOverrides(environmentOverrides); err != nil {
return err
}
@@ -283,6 +286,9 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models
if err != nil {
return nil, err
}
if err := validateManagedRuntimeEnvironmentOverrides(req.Type, environmentOverrides); err != nil {
return nil, err
}
if profile, ok := normalizeDesktopStreamProfile(req.DesktopStreamProfile); !ok {
return nil, fmt.Errorf("invalid desktop stream profile")
} else if profile != "" {
@@ -465,26 +471,23 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models
var bootstrapSnapshot *models.OpenClawInjectionSnapshot
var bootstrapSecretName string
if supportsRuntimeConfigInjection(instance.Type) && s.openClawConfigService != nil && req.OpenClawConfigPlan != nil && hasOpenClawConfigSelections(*req.OpenClawConfigPlan) {
bootstrapSnapshot, err = s.openClawConfigService.CreateSnapshotForInstance(userID, instance, req.OpenClawConfigPlan)
if err != nil {
if snapshot, snapshotErr := s.createRuntimeBootstrapSnapshot(userID, instance, req.OpenClawConfigPlan); snapshotErr != nil {
s.instanceRepo.Delete(instance.ID)
return nil, fmt.Errorf("failed to compile runtime bootstrap config: %w", snapshotErr)
} else if snapshot != nil {
bootstrapSnapshot = snapshot
instance.OpenClawConfigSnapshotID = &bootstrapSnapshot.ID
instance.UpdatedAt = time.Now()
if err := s.instanceRepo.Update(instance); err != nil {
s.instanceRepo.Delete(instance.ID)
return nil, fmt.Errorf("failed to compile runtime bootstrap config: %w", err)
return nil, fmt.Errorf("failed to persist runtime snapshot reference: %w", err)
}
if bootstrapSnapshot != nil {
instance.OpenClawConfigSnapshotID = &bootstrapSnapshot.ID
instance.UpdatedAt = time.Now()
if err := s.instanceRepo.Update(instance); err != nil {
s.instanceRepo.Delete(instance.ID)
return nil, fmt.Errorf("failed to persist runtime snapshot reference: %w", err)
}
bootstrapSecretName, err = s.openClawConfigService.EnsureSnapshotSecret(ctx, userID, instance, bootstrapSnapshot.ID)
if err != nil {
_ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err)
s.instanceRepo.Delete(instance.ID)
return nil, fmt.Errorf("failed to provision runtime bootstrap secret: %w", err)
}
bootstrapSecretName, err = s.openClawConfigService.EnsureSnapshotSecret(ctx, userID, instance, bootstrapSnapshot.ID)
if err != nil {
_ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err)
s.instanceRepo.Delete(instance.ID)
return nil, fmt.Errorf("failed to provision runtime bootstrap secret: %w", err)
}
}
@@ -502,6 +505,14 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models
s.instanceRepo.Delete(instance.ID)
return nil, fmt.Errorf("failed to create PVC: %w", err)
}
if err := EnsureInstanceWorkspacePathForServerScan(ctx, s.instanceRepo, instance); err != nil {
s.pvcService.DeletePVC(ctx, userID, instance.ID)
if bootstrapSnapshot != nil {
_ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err)
}
s.instanceRepo.Delete(instance.ID)
return nil, err
}
nodeSelector, err := s.pvcService.NodeSelectorForPVC(ctx, userID, instance.ID, storageClass)
if err != nil {
@@ -513,15 +524,14 @@ func (s *instanceService) Create(userID int, req CreateInstanceRequest) (*models
return nil, fmt.Errorf("failed to resolve PVC node selector: %w", err)
}
// Ensure any legacy per-instance network policy is removed before creating pod.
// This keeps new pods unrestricted even if older versions created netpols.
if err := s.networkPolicyService.DeletePolicy(ctx, userID, instance.ID, instance.Name); err != nil {
// Managed runtime network policy: optional egress lock when enabled.
if err := s.syncInstanceNetworkPolicy(ctx, userID, instance); err != nil {
s.pvcService.DeletePVC(ctx, userID, instance.ID)
if bootstrapSnapshot != nil {
_ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err)
}
s.instanceRepo.Delete(instance.ID)
return nil, fmt.Errorf("failed to delete network policy: %w", err)
return nil, err
}
// Create Pod
@@ -743,24 +753,20 @@ func (s *instanceService) createV2Instance(ctx context.Context, userID int, req
return nil, fmt.Errorf("failed to provision lite agent bootstrap token: %w", err)
}
if supportsRuntimeConfigInjection(instance.Type) && s.openClawConfigService != nil && req.OpenClawConfigPlan != nil && hasOpenClawConfigSelections(*req.OpenClawConfigPlan) {
bootstrapSnapshot, err := s.openClawConfigService.CreateSnapshotForInstance(userID, instance, req.OpenClawConfigPlan)
if err != nil {
if snapshot, snapshotErr := s.createRuntimeBootstrapSnapshot(userID, instance, req.OpenClawConfigPlan); snapshotErr != nil {
_ = s.instanceRepo.Delete(instance.ID)
return nil, fmt.Errorf("failed to compile lite runtime bootstrap config: %w", snapshotErr)
} else if snapshot != nil {
instance.OpenClawConfigSnapshotID = &snapshot.ID
instance.UpdatedAt = time.Now()
if err := s.instanceRepo.Update(instance); err != nil {
_ = s.openClawConfigService.MarkSnapshotFailed(snapshot, err)
_ = s.instanceRepo.Delete(instance.ID)
return nil, fmt.Errorf("failed to compile lite runtime bootstrap config: %w", err)
return nil, fmt.Errorf("failed to persist lite runtime snapshot reference: %w", err)
}
if bootstrapSnapshot != nil {
instance.OpenClawConfigSnapshotID = &bootstrapSnapshot.ID
instance.UpdatedAt = time.Now()
if err := s.instanceRepo.Update(instance); err != nil {
_ = s.openClawConfigService.MarkSnapshotFailed(bootstrapSnapshot, err)
_ = s.instanceRepo.Delete(instance.ID)
return nil, fmt.Errorf("failed to persist lite runtime snapshot reference: %w", err)
}
if err := s.openClawConfigService.MarkSnapshotActive(bootstrapSnapshot); err != nil {
_ = s.instanceRepo.Delete(instance.ID)
return nil, fmt.Errorf("failed to activate lite runtime bootstrap snapshot: %w", err)
}
if err := s.openClawConfigService.MarkSnapshotActive(snapshot); err != nil {
_ = s.instanceRepo.Delete(instance.ID)
return nil, fmt.Errorf("failed to activate lite runtime bootstrap snapshot: %w", err)
}
}
@@ -883,6 +889,9 @@ func (s *instanceService) Start(instanceID int) error {
if err != nil {
return fmt.Errorf("failed to resolve instance environment: %w", err)
}
if err := EnsureInstanceWorkspacePathForServerScan(ctx, s.instanceRepo, instance); err != nil {
return err
}
bootstrapSecretName := ""
if supportsRuntimeConfigInjection(instance.Type) && s.openClawConfigService != nil && instance.OpenClawConfigSnapshotID != nil && *instance.OpenClawConfigSnapshotID > 0 {
@@ -1168,6 +1177,7 @@ func (s *instanceService) buildAgentEnv(instance *models.Instance) (map[string]s
"CLAWMANAGER_AGENT_INSTANCE_ID": fmt.Sprintf("%d", instance.ID),
"CLAWMANAGER_AGENT_PERSISTENT_DIR": managedRuntimePersistentDir(instance),
"CLAWMANAGER_AGENT_PROTOCOL_VERSION": AgentProtocolVersionV1,
"CLAWMANAGER_AGENT_RUNTIME_TYPE": strings.ToLower(strings.TrimSpace(instance.Type)),
}, nil
}
@@ -1180,6 +1190,42 @@ func supportsManagedRuntimeIntegration(instanceType string) bool {
}
}
func (s *instanceService) createRuntimeBootstrapSnapshot(userID int, instance *models.Instance, plan *OpenClawConfigPlan) (*models.OpenClawInjectionSnapshot, error) {
if !supportsRuntimeConfigInjection(instance.Type) || s.openClawConfigService == nil {
return nil, nil
}
if plan != nil && hasOpenClawConfigSelections(*plan) {
return s.openClawConfigService.CreateSnapshotForInstance(userID, instance, plan)
}
if supportsManagedRuntimeIntegration(instance.Type) {
return s.openClawConfigService.CreateDefaultLLMGovernanceSnapshot(userID, instance)
}
return nil, nil
}
func (s *instanceService) syncInstanceNetworkPolicy(ctx context.Context, userID int, instance *models.Instance) error {
if instance == nil {
return nil
}
if isLiteRuntimeInstance(instance) {
// Lite/gateway-pool instances share runtime pods; per-instance NetworkPolicy does not apply.
if err := s.networkPolicyService.DeletePolicy(ctx, userID, instance.ID, instance.Name); err != nil {
return fmt.Errorf("failed to delete network policy: %w", err)
}
return nil
}
if isInstanceNetworkLockEnabled() && supportsManagedRuntimeIntegration(instance.Type) {
if err := s.networkPolicyService.EnsureDefaultPolicy(ctx, userID, instance.ID, instance.Name); err != nil {
return fmt.Errorf("failed to ensure network policy: %w", err)
}
return nil
}
if err := s.networkPolicyService.DeletePolicy(ctx, userID, instance.ID, instance.Name); err != nil {
return fmt.Errorf("failed to delete network policy: %w", err)
}
return nil
}
func supportsRuntimeConfigInjection(instanceType string) bool {
switch strings.ToLower(strings.TrimSpace(instanceType)) {
case "openclaw", "hermes":
@@ -1794,6 +1840,9 @@ func (s *instanceService) Update(instanceID int, req UpdateInstanceRequest) erro
return err
}
environmentOverrides = applyDesktopStreamProfileEnv(environmentOverrides, profile)
if err := validateManagedRuntimeEnvironmentOverrides(instance.Type, environmentOverrides); err != nil {
return err
}
environmentOverridesJSON, err := marshalEnvironmentOverrides(environmentOverrides)
if err != nil {
return err
backend/internal/services/instance_service_v2_test.go
+8
View File
@@ -698,6 +698,14 @@ func (s *liteOpenClawConfigStub) CreateSnapshotForInstance(userID int, instance
return s.snapshot, nil
}
func (s *liteOpenClawConfigStub) CreateDefaultLLMGovernanceSnapshot(userID int, instance *models.Instance) (*models.OpenClawInjectionSnapshot, error) {
return s.CreateSnapshotForInstance(userID, instance, nil)
}
func (s *liteOpenClawConfigStub) EnsurePlatformLLMGatewayResource(userID int) (*models.OpenClawConfigResource, error) {
return nil, nil
}
func (s *liteOpenClawConfigStub) MarkSnapshotActive(snapshot *models.OpenClawInjectionSnapshot) error {
s.activated = true
if snapshot != nil {
backend/internal/services/instance_workspace_path.go
+63
View File
@@ -0,0 +1,63 @@
package services
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"clawreef/internal/models"
"clawreef/internal/repository"
"clawreef/internal/services/k8s"
)
func hostPathWorkspaceScanEnabled() bool {
client := k8s.GetClient()
return client != nil && client.HostPathFallbackEnabled
}
func instancePersistentHostPath(userID, instanceID int) (string, bool) {
if !hostPathWorkspaceScanEnabled() || userID <= 0 || instanceID <= 0 {
return "", false
}
hostPathPrefix := "/data/clawreef"
if client := k8s.GetClient(); client != nil && strings.TrimSpace(client.HostPathPrefix) != "" {
hostPathPrefix = strings.TrimSpace(client.HostPathPrefix)
}
return filepath.Join(hostPathPrefix, fmt.Sprintf("user-%d", userID), fmt.Sprintf("instance-%d", instanceID)), true
}
func proDesktopWorkspaceScanEligible(instance *models.Instance) bool {
if instance == nil || isLiteRuntimeInstance(instance) {
return false
}
if v2Type, ok := v2RuntimeTypeForInstance(instance); ok && strings.TrimSpace(v2Type) != "" {
return false
}
return supportsManagedRuntimeIntegration(instance.Type)
}
func EnsureInstanceWorkspacePathForServerScan(ctx context.Context, repo repository.InstanceRepository, instance *models.Instance) error {
if repo == nil || instance == nil {
return nil
}
if instance.WorkspacePath != nil && strings.TrimSpace(*instance.WorkspacePath) != "" {
return nil
}
if !proDesktopWorkspaceScanEligible(instance) {
return nil
}
hostPath, ok := instancePersistentHostPath(instance.UserID, instance.ID)
if !ok {
return nil
}
if _, err := os.Stat(hostPath); err != nil {
return nil
}
if err := repo.SetWorkspacePath(ctx, instance.ID, hostPath); err != nil {
return fmt.Errorf("failed to persist pro desktop workspace path: %w", err)
}
instance.WorkspacePath = &hostPath
return nil
}
backend/internal/services/k8s/instance_deployment_service.go
+1
View File
@@ -244,6 +244,7 @@ func BuildInstanceDeployment(client *Client, config PodConfig, replicas int32) *
"runtime-type": runtimeType,
"managed-by": "clawreef",
}
appendManagedRuntimeLabels(config.Type, labels)
return &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
backend/internal/services/k8s/network_policy_service.go
+4 -3
View File
@@ -55,9 +55,10 @@ func (s *NetworkPolicyService) EnsureDefaultPolicy(ctx context.Context, userID,
Spec: networkingv1.NetworkPolicySpec{
PodSelector: metav1.LabelSelector{
MatchLabels: map[string]string{
"app": "clawreef",
"instance-id": instanceLabel,
"managed-by": "clawreef",
"app": "clawreef",
"instance-id": instanceLabel,
"managed-by": "clawreef",
"clawmanager.io/managed-runtime": "true",
},
},
PolicyTypes: []networkingv1.PolicyType{
backend/internal/services/k8s/pod_service.go
+1
View File
@@ -156,6 +156,7 @@ func (s *PodService) CreatePod(ctx context.Context, config PodConfig) (*corev1.P
"runtime-type": runtimeType,
"managed-by": "clawreef",
}
appendManagedRuntimeLabels(config.Type, labels)
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
backend/internal/services/k8s/runtime_labels.go
+13
View File
@@ -0,0 +1,13 @@
package k8s
import "strings"
func appendManagedRuntimeLabels(instanceType string, labels map[string]string) {
if labels == nil {
return
}
switch strings.ToLower(strings.TrimSpace(instanceType)) {
case "openclaw", "hermes":
labels["clawmanager.io/managed-runtime"] = "true"
}
}
backend/internal/services/openclaw_config_service.go
+2
View File
@@ -263,6 +263,8 @@ type OpenClawConfigService interface {
CompilePreview(userID int, plan OpenClawConfigPlan) (*OpenClawConfigCompilePreview, error)
PlanWithoutTeamMemberLeaderOnlyChannels(userID int, plan *OpenClawConfigPlan) (*OpenClawConfigPlan, error)
CreateSnapshotForInstance(userID int, instance *models.Instance, plan *OpenClawConfigPlan) (*models.OpenClawInjectionSnapshot, error)
CreateDefaultLLMGovernanceSnapshot(userID int, instance *models.Instance) (*models.OpenClawInjectionSnapshot, error)
EnsurePlatformLLMGatewayResource(userID int) (*models.OpenClawConfigResource, error)
MarkSnapshotActive(snapshot *models.OpenClawInjectionSnapshot) error
MarkSnapshotFailed(snapshot *models.OpenClawInjectionSnapshot, err error) error
EnsureSnapshotSecret(ctx context.Context, userID int, instance *models.Instance, snapshotID int) (string, error)
backend/internal/services/openclaw_platform_llm_gateway.go
+88
View File
@@ -0,0 +1,88 @@
package services
import (
"encoding/json"
"fmt"
"time"
"clawreef/internal/models"
)
const (
PlatformLLMGatewayResourceKey = "platform-llm-gateway"
PlatformLLMGatewayResourceName = "Platform LLM Gateway"
)
var platformLLMGatewayAgentContent = json.RawMessage(`{
"schemaVersion": 1,
"kind": "agent",
"format": "agent/platform-llm-gateway@v1",
"dependsOn": [],
"config": {
"models": {
"providers": {
"clawmanager": {
"type": "openai-compatible",
"baseUrl": "${CLAWMANAGER_LLM_BASE_URL}",
"apiKey": "${CLAWMANAGER_LLM_API_KEY}",
"default": true
}
},
"primary": "auto/auto"
}
}
}`)
func (s *openClawConfigService) EnsurePlatformLLMGatewayResource(userID int) (*models.OpenClawConfigResource, error) {
if userID <= 0 {
return nil, fmt.Errorf("user id is required")
}
existing, err := s.repo.GetResourceByUserTypeKey(userID, OpenClawConfigResourceTypeAgent, PlatformLLMGatewayResourceKey)
if err != nil {
return nil, err
}
if existing != nil {
return existing, nil
}
description := "Built-in agent config that routes OpenClaw LLM calls through the ClawManager AI Gateway."
now := time.Now()
item := &models.OpenClawConfigResource{
UserID: userID,
ResourceType: OpenClawConfigResourceTypeAgent,
ResourceKey: PlatformLLMGatewayResourceKey,
Name: PlatformLLMGatewayResourceName,
Description: &description,
Enabled: true,
Version: 1,
TagsJSON: encodeStringArray([]string{"builtin", "llm-governance", "platform"}),
ContentJSON: string(platformLLMGatewayAgentContent),
CreatedAt: now,
UpdatedAt: now,
}
if err := s.repo.CreateResource(item); err != nil {
return nil, err
}
return item, nil
}
func (s *openClawConfigService) CreateDefaultLLMGovernanceSnapshot(userID int, instance *models.Instance) (*models.OpenClawInjectionSnapshot, error) {
if instance == nil || !supportsManagedRuntimeIntegration(instance.Type) {
return nil, nil
}
resource, err := s.EnsurePlatformLLMGatewayResource(userID)
if err != nil {
return nil, err
}
if resource == nil || resource.ID <= 0 {
return nil, fmt.Errorf("failed to provision platform llm gateway resource")
}
plan := &OpenClawConfigPlan{
Mode: OpenClawConfigPlanModeManual,
ResourceIDs: []int{resource.ID},
}
return s.CreateSnapshotForInstance(userID, instance, plan)
}
backend/internal/services/openclaw_platform_llm_gateway_test.go
+152
View File
@@ -0,0 +1,152 @@
package services
import (
"fmt"
"strings"
"testing"
"time"
"clawreef/internal/models"
)
type platformLLMGatewayRepoStub struct {
nextResourceID int
nextSnapshotID int
resources map[string]*models.OpenClawConfigResource
}
func newPlatformLLMGatewayRepoStub() *platformLLMGatewayRepoStub {
return &platformLLMGatewayRepoStub{
nextResourceID: 1,
nextSnapshotID: 1,
resources: map[string]*models.OpenClawConfigResource{},
}
}
func (s *platformLLMGatewayRepoStub) resourceKey(userID int, resourceType, resourceKey string) string {
return fmt.Sprintf("%d:%s:%s", userID, resourceType, resourceKey)
}
func (s *platformLLMGatewayRepoStub) ListResources(userID int, resourceType string) ([]models.OpenClawConfigResource, error) {
return nil, nil
}
func (s *platformLLMGatewayRepoStub) GetResourceByID(id int) (*models.OpenClawConfigResource, error) {
for _, resource := range s.resources {
if resource.ID == id {
copy := *resource
return &copy, nil
}
}
return nil, nil
}
func (s *platformLLMGatewayRepoStub) GetResourceByUserTypeKey(userID int, resourceType, resourceKey string) (*models.OpenClawConfigResource, error) {
resource := s.resources[s.resourceKey(userID, resourceType, resourceKey)]
if resource == nil {
return nil, nil
}
copy := *resource
return &copy, nil
}
func (s *platformLLMGatewayRepoStub) CreateResource(resource *models.OpenClawConfigResource) error {
if resource == nil {
return nil
}
resource.ID = s.nextResourceID
s.nextResourceID++
s.resources[s.resourceKey(resource.UserID, resource.ResourceType, resource.ResourceKey)] = resource
return nil
}
func (s *platformLLMGatewayRepoStub) UpdateResource(resource *models.OpenClawConfigResource) error {
return nil
}
func (s *platformLLMGatewayRepoStub) DeleteResource(id int) error { return nil }
func (s *platformLLMGatewayRepoStub) ListBundles(userID int) ([]models.OpenClawConfigBundle, error) {
return nil, nil
}
func (s *platformLLMGatewayRepoStub) GetBundleByID(id int) (*models.OpenClawConfigBundle, error) {
return nil, nil
}
func (s *platformLLMGatewayRepoStub) CreateBundle(bundle *models.OpenClawConfigBundle) error { return nil }
func (s *platformLLMGatewayRepoStub) UpdateBundle(bundle *models.OpenClawConfigBundle) error {
return nil
}
func (s *platformLLMGatewayRepoStub) DeleteBundle(id int) error { return nil }
func (s *platformLLMGatewayRepoStub) ListBundleItems(bundleID int) ([]models.OpenClawConfigBundleItem, error) {
return nil, nil
}
func (s *platformLLMGatewayRepoStub) ReplaceBundleItems(bundleID int, items []models.OpenClawConfigBundleItem) error {
return nil
}
func (s *platformLLMGatewayRepoStub) ListBundleSkills(bundleID int) ([]models.OpenClawConfigBundleSkill, error) {
return nil, nil
}
func (s *platformLLMGatewayRepoStub) ReplaceBundleSkills(bundleID int, items []models.OpenClawConfigBundleSkill) error {
return nil
}
func (s *platformLLMGatewayRepoStub) CreateSnapshot(snapshot *models.OpenClawInjectionSnapshot) error {
if snapshot == nil {
return nil
}
snapshot.ID = s.nextSnapshotID
s.nextSnapshotID++
return nil
}
func (s *platformLLMGatewayRepoStub) UpdateSnapshot(snapshot *models.OpenClawInjectionSnapshot) error {
return nil
}
func (s *platformLLMGatewayRepoStub) GetSnapshotByID(id int) (*models.OpenClawInjectionSnapshot, error) {
return nil, nil
}
func (s *platformLLMGatewayRepoStub) ListSnapshotsByUser(userID int, limit int) ([]models.OpenClawInjectionSnapshot, error) {
return nil, nil
}
func (s *platformLLMGatewayRepoStub) ListActiveSnapshots(userID int) ([]models.OpenClawInjectionSnapshot, error) {
return nil, nil
}
func (s *platformLLMGatewayRepoStub) UpdateSnapshotIfUnchanged(snapshot *models.OpenClawInjectionSnapshot, expectedUpdatedAt time.Time) (bool, error) {
return true, nil
}
func TestEnsurePlatformLLMGatewayResourceCreatesBuiltinAgentResource(t *testing.T) {
repo := newPlatformLLMGatewayRepoStub()
service := &openClawConfigService{repo: repo}
resource, err := service.EnsurePlatformLLMGatewayResource(9)
if err != nil {
t.Fatalf("EnsurePlatformLLMGatewayResource returned error: %v", err)
}
if resource == nil || resource.ID <= 0 {
t.Fatalf("expected created resource, got %+v", resource)
}
if resource.ResourceType != OpenClawConfigResourceTypeAgent || resource.ResourceKey != PlatformLLMGatewayResourceKey {
t.Fatalf("unexpected resource identity: %+v", resource)
}
again, err := service.EnsurePlatformLLMGatewayResource(9)
if err != nil {
t.Fatalf("second EnsurePlatformLLMGatewayResource returned error: %v", err)
}
if again == nil || again.ID != resource.ID {
t.Fatalf("expected same resource id, got %+v want %d", again, resource.ID)
}
}
func TestCreateDefaultLLMGovernanceSnapshotCompilesPlatformGatewayAgent(t *testing.T) {
repo := newPlatformLLMGatewayRepoStub()
service := &openClawConfigService{repo: repo}
instance := &models.Instance{ID: 42, UserID: 9, Type: "openclaw", Name: "oc-42"}
snapshot, err := service.CreateDefaultLLMGovernanceSnapshot(9, instance)
if err != nil {
t.Fatalf("CreateDefaultLLMGovernanceSnapshot returned error: %v", err)
}
if snapshot == nil {
t.Fatal("expected snapshot")
}
if snapshot.Mode != OpenClawConfigPlanModeManual {
t.Fatalf("snapshot mode = %q, want manual", snapshot.Mode)
}
if !strings.Contains(snapshot.ResolvedResourcesJSON, PlatformLLMGatewayResourceKey) {
t.Fatalf("expected platform gateway resource in snapshot, got %s", snapshot.ResolvedResourcesJSON)
}
}
backend/internal/services/runtime_agent_client.go
+14
View File
@@ -17,6 +17,7 @@ type RuntimeAgentClient interface {
CreateGateway(ctx context.Context, endpoint string, req RuntimeAgentCreateGatewayRequest) (*RuntimeAgentCreateGatewayResponse, error)
DeleteGateway(ctx context.Context, endpoint, gatewayID string) error
Drain(ctx context.Context, endpoint string) error
ResyncInstanceSkills(ctx context.Context, endpoint string, instanceID int, mode string) error
}
type RuntimeAgentPortRange struct {
@@ -91,6 +92,19 @@ func (c *runtimeAgentHTTPClient) Drain(ctx context.Context, endpoint string) err
return c.do(ctx, http.MethodPost, endpoint, "/v1/drain", map[string]bool{"draining": true}, nil)
}
func (c *runtimeAgentHTTPClient) ResyncInstanceSkills(ctx context.Context, endpoint string, instanceID int, mode string) error {
mode = strings.TrimSpace(mode)
if mode == "" {
mode = "full"
}
body := map[string]any{
"instance_id": instanceID,
"mode": mode,
"trigger": "manual",
}
return c.do(ctx, http.MethodPost, endpoint, "/v1/skills/resync", body, nil)
}
func (c *runtimeAgentHTTPClient) do(ctx context.Context, method, endpoint, path string, body any, out any) error {
endpoint = strings.TrimRight(endpoint, "/")
var reader io.Reader
backend/internal/services/runtime_agent_client_test.go
+24
View File
@@ -192,6 +192,30 @@ func TestRuntimeAgentClientDrainSendsJSONBody(t *testing.T) {
}
}
func TestRuntimeAgentClientResyncInstanceSkills(t *testing.T) {
var gotMethod, gotPath, gotToken string
var body map[string]any
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotMethod = r.Method
gotPath = r.URL.Path
gotToken = r.Header.Get("X-ClawManager-Control-Token")
_ = json.NewDecoder(r.Body).Decode(&body)
w.WriteHeader(http.StatusAccepted)
}))
defer server.Close()
client := NewRuntimeAgentClient("secret")
if err := client.ResyncInstanceSkills(context.Background(), server.URL, 12, "full"); err != nil {
t.Fatalf("ResyncInstanceSkills returned error: %v", err)
}
if gotMethod != http.MethodPost || gotPath != "/v1/skills/resync" || gotToken != "secret" {
t.Fatalf("unexpected request: %s %s token=%q", gotMethod, gotPath, gotToken)
}
if body["instance_id"] != float64(12) || body["mode"] != "full" {
t.Fatalf("unexpected body: %#v", body)
}
}
func TestRuntimeAgentClientConflict(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "no free port", http.StatusConflict)
backend/internal/services/runtime_scheduler_test.go
+3
View File
@@ -2367,6 +2367,9 @@ func (c *fakeRuntimeAgentClient) Drain(ctx context.Context, endpoint string) err
c.drainEndpoints = append(c.drainEndpoints, endpoint)
return nil
}
func (c *fakeRuntimeAgentClient) ResyncInstanceSkills(ctx context.Context, endpoint string, instanceID int, mode string) error {
return nil
}
type fakeRuntimeEventService struct {
published []fakeRuntimeEvent
backend/internal/services/skill_content_md5_test.go
+34
View File
@@ -0,0 +1,34 @@
package services
import (
"errors"
"testing"
"clawreef/internal/utils"
)
func TestHashDirectoryGoldenWeatherFixture(t *testing.T) {
files := map[string][]byte{
"src/main.py": []byte("print('weather')\n"),
}
got := hashDirectory(files)
want := referenceSkillContentMD5(files)
if got != want {
t.Fatalf("hashDirectory() = %s, want %s", got, want)
}
}
func TestHubErrorMD5MismatchCode(t *testing.T) {
err := utils.NewHubError(
"skill_package_md5_mismatch",
"skill package md5 mismatch: expected abc got def",
map[string]string{"expected": "abc", "computed": "def"},
)
var hubErr *utils.HubError
if !errors.As(err, &hubErr) {
t.Fatal("expected HubError")
}
if hubErr.Code != "skill_package_md5_mismatch" {
t.Fatalf("expected skill_package_md5_mismatch, got %q", hubErr.Code)
}
}
backend/internal/services/skill_hub_repo_stub_test.go
+302
View File
@@ -0,0 +1,302 @@
package services
import (
"time"
"clawreef/internal/models"
)
type skillRepoStub struct {
skills map[int]*models.Skill
blobs map[int]*models.SkillBlob
versions map[int]*models.SkillVersion
tags map[int]*models.SkillHubTag
tagAssignments map[int][]int
instanceSkillsBySkillID map[int][]models.InstanceSkill
instanceSkills []models.InstanceSkill
hardDeleteCalled bool
}
func (s *skillRepoStub) ListSkillsByUser(userID int) ([]models.Skill, error) {
items := make([]models.Skill, 0)
for _, skill := range s.skills {
if skill.UserID == userID {
items = append(items, *skill)
}
}
return items, nil
}
func (s *skillRepoStub) ListAllSkills() ([]models.Skill, error) {
items := make([]models.Skill, 0, len(s.skills))
for _, skill := range s.skills {
items = append(items, *skill)
}
return items, nil
}
func (s *skillRepoStub) GetSkillByID(id int) (*models.Skill, error) {
if skill, ok := s.skills[id]; ok {
copy := *skill
return &copy, nil
}
return nil, nil
}
func (s *skillRepoStub) GetSkillByUserKey(userID int, skillKey string) (*models.Skill, error) {
for _, skill := range s.skills {
if skill.UserID == userID && skill.SkillKey == skillKey && skill.Status == skillStatusActive {
copy := *skill
return &copy, nil
}
}
return nil, nil
}
func (s *skillRepoStub) CreateSkill(*models.Skill) error { return nil }
func (s *skillRepoStub) UpdateSkill(skill *models.Skill) error {
if s.skills == nil {
s.skills = map[int]*models.Skill{}
}
copy := *skill
s.skills[skill.ID] = &copy
return nil
}
func (s *skillRepoStub) DeleteSkill(int) error {
s.hardDeleteCalled = true
return nil
}
func (s *skillRepoStub) GetBlobByContentHash(string) (*models.SkillBlob, error) { return nil, nil }
func (s *skillRepoStub) GetBlobByID(id int) (*models.SkillBlob, error) {
if blob, ok := s.blobs[id]; ok {
copy := *blob
return &copy, nil
}
return nil, nil
}
func (s *skillRepoStub) CreateBlob(blob *models.SkillBlob) error {
if s.blobs == nil {
s.blobs = map[int]*models.SkillBlob{}
}
if blob.ID == 0 {
blob.ID = len(s.blobs) + 1
}
copy := *blob
s.blobs[blob.ID] = &copy
return nil
}
func (s *skillRepoStub) UpdateBlob(blob *models.SkillBlob) error {
if s.blobs == nil {
s.blobs = map[int]*models.SkillBlob{}
}
copy := *blob
s.blobs[blob.ID] = &copy
return nil
}
func (s *skillRepoStub) ListVersionsBySkillID(int) ([]models.SkillVersion, error) {
return nil, nil
}
func (s *skillRepoStub) GetVersionByID(id int) (*models.SkillVersion, error) {
if version, ok := s.versions[id]; ok {
copy := *version
return &copy, nil
}
return nil, nil
}
func (s *skillRepoStub) GetVersionBySkillAndBlob(int, int) (*models.SkillVersion, error) {
return nil, nil
}
func (s *skillRepoStub) GetLatestVersionBySkillID(int) (*models.SkillVersion, error) {
return nil, nil
}
func (s *skillRepoStub) CreateVersion(*models.SkillVersion) error { return nil }
func (s *skillRepoStub) UpdateVersion(version *models.SkillVersion) error {
if s.versions == nil {
s.versions = map[int]*models.SkillVersion{}
}
copy := *version
s.versions[version.ID] = &copy
return nil
}
func (s *skillRepoStub) ListInstanceSkills(int) ([]models.InstanceSkill, error) {
return nil, nil
}
func (s *skillRepoStub) ListActiveInstanceSkillsBySkillID(skillID int) ([]models.InstanceSkill, error) {
if s.instanceSkillsBySkillID != nil {
if items, ok := s.instanceSkillsBySkillID[skillID]; ok {
return filterActiveInstanceSkills(items), nil
}
}
items := make([]models.InstanceSkill, 0)
for _, item := range s.instanceSkills {
if item.SkillID == skillID && item.Status != "removed" && item.Status != "missing" {
items = append(items, item)
}
}
return items, nil
}
func filterActiveInstanceSkills(items []models.InstanceSkill) []models.InstanceSkill {
active := make([]models.InstanceSkill, 0, len(items))
for _, item := range items {
if item.Status != "removed" && item.Status != "missing" {
active = append(active, item)
}
}
return active
}
func (s *skillRepoStub) GetInstanceSkill(instanceID, skillID int) (*models.InstanceSkill, error) {
for _, item := range s.instanceSkills {
if item.InstanceID == instanceID && item.SkillID == skillID {
copy := item
return &copy, nil
}
}
return nil, nil
}
func (s *skillRepoStub) UpsertInstanceSkill(*models.InstanceSkill) error { return nil }
func (s *skillRepoStub) MarkInstanceSkillRemoved(int, int, time.Time) error { return nil }
func (s *skillRepoStub) MarkInstanceSkillRemovedBySkillKey(int, string, time.Time) error {
return nil
}
func (s *skillRepoStub) MarkInstanceSkillsRemovedByWorkspacePath(int, string, time.Time) error {
return nil
}
func (s *skillRepoStub) MarkMissingInstanceSkills(int, []int, time.Time) error { return nil }
func (s *skillRepoStub) CreateScanResult(result *models.SkillScanResult) error {
if result.ID == 0 {
result.ID = 99
}
return nil
}
func (s *skillRepoStub) GetScanResultByID(int) (*models.SkillScanResult, error) { return nil, nil }
func (s *skillRepoStub) ListScanResultsByBlobID(int) ([]models.SkillScanResult, error) {
return nil, nil
}
func (s *skillRepoStub) GetLatestScanResultByBlobID(int) (*models.SkillScanResult, error) {
return nil, nil
}
func (s *skillRepoStub) GetLatestScanResultBySkillID(int) (*models.SkillScanResult, error) {
return nil, nil
}
func (s *skillRepoStub) ListHubTags(bool) ([]models.SkillHubTag, error) { return nil, nil }
func (s *skillRepoStub) GetHubTagByID(id int) (*models.SkillHubTag, error) {
if tag, ok := s.tags[id]; ok {
copy := *tag
return &copy, nil
}
return nil, nil
}
func (s *skillRepoStub) ListHubTagsBySkillID(skillID int) ([]models.SkillHubTag, error) {
tagIDs := s.tagAssignments[skillID]
result := make([]models.SkillHubTag, 0, len(tagIDs))
for _, tagID := range tagIDs {
if tag, ok := s.tags[tagID]; ok {
result = append(result, *tag)
}
}
return result, nil
}
func (s *skillRepoStub) ReplaceSkillTagAssignments(skillID int, tagIDs []int) error {
if s.tagAssignments == nil {
s.tagAssignments = map[int][]int{}
}
s.tagAssignments[skillID] = append([]int(nil), tagIDs...)
return nil
}
func (s *skillRepoStub) ListPublicHubSkills() ([]models.Skill, error) {
items := make([]models.Skill, 0)
for _, skill := range s.skills {
if skill.Visibility == skillVisibilityPublic && skill.Status == skillStatusActive {
items = append(items, *skill)
}
}
return items, nil
}
func (s *skillRepoStub) ListSkillsForHubAdmin() ([]models.Skill, error) {
return s.ListAllSkills()
}
type hubTagRepoStub struct {
tags map[int]*models.SkillHubTag
}
func (s *hubTagRepoStub) ListSkillsByUser(int) ([]models.Skill, error) { return nil, nil }
func (s *hubTagRepoStub) ListAllSkills() ([]models.Skill, error) { return nil, nil }
func (s *hubTagRepoStub) GetSkillByID(int) (*models.Skill, error) { return nil, nil }
func (s *hubTagRepoStub) GetSkillByUserKey(int, string) (*models.Skill, error) { return nil, nil }
func (s *hubTagRepoStub) CreateSkill(*models.Skill) error { return nil }
func (s *hubTagRepoStub) UpdateSkill(*models.Skill) error { return nil }
func (s *hubTagRepoStub) DeleteSkill(int) error { return nil }
func (s *hubTagRepoStub) GetBlobByContentHash(string) (*models.SkillBlob, error) { return nil, nil }
func (s *hubTagRepoStub) GetBlobByID(int) (*models.SkillBlob, error) { return nil, nil }
func (s *hubTagRepoStub) CreateBlob(*models.SkillBlob) error { return nil }
func (s *hubTagRepoStub) UpdateBlob(*models.SkillBlob) error { return nil }
func (s *hubTagRepoStub) ListVersionsBySkillID(int) ([]models.SkillVersion, error) { return nil, nil }
func (s *hubTagRepoStub) GetVersionByID(int) (*models.SkillVersion, error) { return nil, nil }
func (s *hubTagRepoStub) GetVersionBySkillAndBlob(int, int) (*models.SkillVersion, error) {
return nil, nil
}
func (s *hubTagRepoStub) GetLatestVersionBySkillID(int) (*models.SkillVersion, error) {
return nil, nil
}
func (s *hubTagRepoStub) CreateVersion(*models.SkillVersion) error { return nil }
func (s *hubTagRepoStub) UpdateVersion(*models.SkillVersion) error { return nil }
func (s *hubTagRepoStub) ListInstanceSkills(int) ([]models.InstanceSkill, error) {
return nil, nil
}
func (s *hubTagRepoStub) ListActiveInstanceSkillsBySkillID(int) ([]models.InstanceSkill, error) {
return nil, nil
}
func (s *hubTagRepoStub) GetInstanceSkill(int, int) (*models.InstanceSkill, error) { return nil, nil }
func (s *hubTagRepoStub) UpsertInstanceSkill(*models.InstanceSkill) error { return nil }
func (s *hubTagRepoStub) MarkInstanceSkillRemoved(int, int, time.Time) error { return nil }
func (s *hubTagRepoStub) MarkInstanceSkillRemovedBySkillKey(int, string, time.Time) error {
return nil
}
func (s *hubTagRepoStub) MarkInstanceSkillsRemovedByWorkspacePath(int, string, time.Time) error {
return nil
}
func (s *hubTagRepoStub) MarkMissingInstanceSkills(int, []int, time.Time) error { return nil }
func (s *hubTagRepoStub) CreateScanResult(*models.SkillScanResult) error { return nil }
func (s *hubTagRepoStub) GetScanResultByID(int) (*models.SkillScanResult, error) { return nil, nil }
func (s *hubTagRepoStub) ListScanResultsByBlobID(int) ([]models.SkillScanResult, error) {
return nil, nil
}
func (s *hubTagRepoStub) GetLatestScanResultByBlobID(int) (*models.SkillScanResult, error) {
return nil, nil
}
func (s *hubTagRepoStub) GetLatestScanResultBySkillID(int) (*models.SkillScanResult, error) {
return nil, nil
}
func (s *hubTagRepoStub) ListHubTags(bool) ([]models.SkillHubTag, error) { return nil, nil }
func (s *hubTagRepoStub) GetHubTagByID(id int) (*models.SkillHubTag, error) {
if tag, ok := s.tags[id]; ok {
return tag, nil
}
return nil, nil
}
func (s *hubTagRepoStub) ListHubTagsBySkillID(int) ([]models.SkillHubTag, error) { return nil, nil }
func (s *hubTagRepoStub) ReplaceSkillTagAssignments(int, []int) error { return nil }
func (s *hubTagRepoStub) ListPublicHubSkills() ([]models.Skill, error) { return nil, nil }
func (s *hubTagRepoStub) ListSkillsForHubAdmin() ([]models.Skill, error) { return nil, nil }
backend/internal/services/skill_hub_service.go
+808
View File
@@ -0,0 +1,808 @@
package services
import (
"context"
"fmt"
"mime/multipart"
"strings"
"time"
"clawreef/internal/models"
)
const (
skillVisibilityPrivate = "private"
skillVisibilityPublic = "public"
)
type SkillHubTagPayload struct {
ID int `json:"id"`
TagKey string `json:"tag_key"`
Name string `json:"name"`
Description *string `json:"description,omitempty"`
SortOrder int `json:"sort_order"`
AdminOnly bool `json:"admin_only"`
}
type SkillHubCatalogQuery struct {
TagKeys []string
Search string
Page int
PageSize int
}
type SkillHubCatalogResponse struct {
Items []SkillPayload `json:"items"`
Total int `json:"total"`
Page int `json:"page"`
PageSize int `json:"page_size"`
TotalPages int `json:"total_pages"`
}
type PublishSkillHubRequest struct {
TagIDs []int `json:"tag_ids" binding:"required,min=1"`
}
type UpdateSkillHubTagsRequest struct {
TagIDs []int `json:"tag_ids" binding:"required,min=1"`
}
type InstallHubSkillRequest struct {
InstanceID int `json:"instance_id" binding:"required,min=1"`
}
func isAdminRole(role string) bool {
return strings.EqualFold(strings.TrimSpace(role), "admin")
}
func (s *skillService) skillBlobForPublish(skill *models.Skill) (*models.SkillBlob, error) {
if skill == nil || skill.CurrentVersionID == nil {
return nil, fmt.Errorf("skill has no version")
}
version, err := s.repo.GetVersionByID(*skill.CurrentVersionID)
if err != nil {
return nil, err
}
if version == nil {
return nil, fmt.Errorf("skill has no version")
}
blob, err := s.repo.GetBlobByID(version.BlobID)
if err != nil {
return nil, err
}
if blob == nil {
return nil, fmt.Errorf("skill blob not found")
}
return blob, nil
}
func isHubPublishableBlob(blob *models.SkillBlob) bool {
if blob == nil {
return false
}
if !strings.EqualFold(strings.TrimSpace(blob.ScanStatus), "completed") {
return false
}
risk := strings.ToLower(strings.TrimSpace(blob.RiskLevel))
if risk != skillRiskNone && risk != skillRiskLow {
return false
}
if strings.TrimSpace(blob.ObjectKey) == "" {
return false
}
return true
}
func (s *skillService) isHubPublishable(skill *models.Skill, blob *models.SkillBlob) bool {
if skill == nil || blob == nil || isDeletedSkill(skill) {
return false
}
if !isUserManagedSkill(*skill) && !strings.EqualFold(strings.TrimSpace(skill.SourceType), skillSourceDiscovered) {
return false
}
if !strings.EqualFold(strings.TrimSpace(skill.Status), "active") {
return false
}
return isHubPublishableBlob(blob)
}
func (s *skillService) CanDownloadSkill(actorUserID int, actorRole string, skill *models.Skill) bool {
if skill == nil || !isUserManagedSkill(*skill) {
return false
}
return s.CanViewSkill(actorUserID, actorRole, skill)
}
func (s *skillService) CanViewSkill(actorUserID int, actorRole string, skill *models.Skill) bool {
if skill == nil || isDeletedSkill(skill) {
return false
}
if isAdminRole(actorRole) {
return isUserManagedSkill(*skill) || strings.EqualFold(skill.SourceType, skillSourceDiscovered)
}
if skill.UserID == actorUserID {
return isUserManagedSkill(*skill) || strings.EqualFold(skill.SourceType, skillSourceDiscovered)
}
return isUserManagedSkill(*skill) && strings.EqualFold(strings.TrimSpace(skill.Visibility), skillVisibilityPublic)
}
func (s *skillService) CanAttachSkill(actorUserID int, actorRole string, skill *models.Skill, instance *models.Instance) bool {
if skill == nil || instance == nil {
return false
}
if !isUserManagedSkill(*skill) {
return false
}
if !strings.EqualFold(strings.TrimSpace(skill.Status), "active") {
return false
}
if skill.RiskLevel == skillRiskMedium || skill.RiskLevel == skillRiskHigh {
return false
}
if isAdminRole(actorRole) {
return true
}
if instance.UserID != actorUserID {
return false
}
if skill.UserID == actorUserID {
return true
}
return strings.EqualFold(strings.TrimSpace(skill.Visibility), skillVisibilityPublic)
}
func (s *skillService) hubTagsToPayload(tags []models.SkillHubTag) []SkillHubTagPayload {
result := make([]SkillHubTagPayload, 0, len(tags))
for _, tag := range tags {
result = append(result, SkillHubTagPayload{
ID: tag.ID,
TagKey: tag.TagKey,
Name: tag.Name,
Description: tag.Description,
SortOrder: tag.SortOrder,
AdminOnly: tag.AdminOnly,
})
}
return result
}
func (s *skillService) liteInstanceForSkill(skillID int) *models.Instance {
if s == nil || s.repo == nil || s.instanceRepo == nil || skillID <= 0 {
return nil
}
items, err := s.repo.ListActiveInstanceSkillsBySkillID(skillID)
if err != nil || len(items) == 0 {
return nil
}
for _, item := range items {
instance, err := s.instanceRepo.GetByID(item.InstanceID)
if err != nil || instance == nil {
continue
}
if isLiteRuntimeInstance(instance) {
return instance
}
}
return nil
}
func (s *skillService) enrichSkillPayload(payload *SkillPayload, skill models.Skill, instance *models.Instance) error {
if instance == nil {
instance = s.liteInstanceForSkill(skill.ID)
}
tags, err := s.repo.ListHubTagsBySkillID(skill.ID)
if err != nil {
return err
}
payload.Visibility = skill.Visibility
if strings.TrimSpace(payload.Visibility) == "" {
payload.Visibility = skillVisibilityPrivate
}
payload.PublishedAt = skill.PublishedAt
payload.PublishedBy = skill.PublishedBy
payload.Tags = s.hubTagsToPayload(tags)
blob, blobErr := s.skillBlobForPublish(&skill)
if blobErr == nil {
payload.Publishable = s.isHubPublishable(&skill, blob)
payload.ScanStatus = blob.ScanStatus
} else {
payload.Publishable = false
}
skipAgentCollectFailure := instance != nil && isLiteRuntimeInstance(instance)
payload.PublishBlockedReason = s.publishBlockedReasonForSkill(&skill, blob, blobErr, payload.Publishable, skipAgentCollectFailure)
if s.materializeService != nil && blobErr == nil && blob != nil && strings.TrimSpace(blob.ObjectKey) == "" {
if status, materializeErr := s.materializeService.GetObservedStatus(skill.ID, blob); status != nil {
payload.PackageMaterializeStatus = status
payload.PackageMaterializeError = materializeErr
}
}
if collectErr := s.resolvePackageCollectError(skill.ID, blob, blobErr, skipAgentCollectFailure); collectErr != nil {
payload.PackageCollectError = collectErr
}
if s.userRepo != nil {
owner, err := s.userRepo.GetByID(skill.UserID)
if err != nil {
return err
}
if owner != nil {
payload.OwnerUsername = &owner.Username
}
}
return nil
}
func truncateCollectError(value string, maxLen int) string {
value = strings.TrimSpace(value)
if maxLen <= 0 || len(value) <= maxLen {
return value
}
return value[:maxLen] + "..."
}
func (s *skillService) resolvePackageCollectError(skillID int, blob *models.SkillBlob, blobErr error, skipAgentCollectFailure bool) *string {
if blobErr != nil || blob == nil || strings.TrimSpace(blob.ObjectKey) != "" {
return nil
}
if s.materializeService != nil {
if _, materializeErr := s.materializeService.GetObservedStatus(skillID, blob); materializeErr != nil {
return materializeErr
}
job, err := s.materializeService.FindLatestBySkillID(skillID)
if err == nil && job != nil && job.LastError != nil && strings.TrimSpace(*job.LastError) != "" {
summary := truncateCollectError(*job.LastError, 512)
if summary != "" {
return &summary
}
}
}
if skipAgentCollectFailure {
return nil
}
cmd, err := s.latestCollectPackageFailure(skillID)
if err != nil || cmd == nil {
return nil
}
if cmd.ErrorMessage == nil {
return nil
}
summary := truncateCollectError(*cmd.ErrorMessage, 512)
if summary == "" {
return nil
}
return &summary
}
func (s *skillService) latestCollectPackageFailure(skillID int) (*models.InstanceCommand, error) {
if s.commandRepo == nil {
return nil, nil
}
return s.commandRepo.FindLatestFailedCollectSkillPackage(formatExternalSkillID(skillID))
}
func (s *skillService) publishBlockedReasonForSkill(skill *models.Skill, blob *models.SkillBlob, blobErr error, publishable bool, skipAgentCollectFailure bool) *string {
if publishable || skill == nil {
return nil
}
reason := func(value string) *string {
return &value
}
if isDeletedSkill(skill) {
return reason("skill_deleted")
}
if !strings.EqualFold(strings.TrimSpace(skill.Status), skillStatusActive) {
return reason("skill_inactive")
}
if blobErr != nil || blob == nil {
return reason("skill_package_pending")
}
if strings.TrimSpace(blob.ObjectKey) == "" {
if s.materializeService != nil {
if job, err := s.materializeService.FindLatestBySkillID(skill.ID); err == nil && job != nil {
if blocked := materializeBlockedReason(job); blocked != nil {
return blocked
}
return reason("skill_package_pending")
}
}
if skipAgentCollectFailure {
return reason("skill_package_pending")
}
if cmd, err := s.latestCollectPackageFailure(skill.ID); err == nil && cmd != nil {
return reason("skill_package_collect_failed")
}
return reason("skill_package_pending")
}
if strings.EqualFold(strings.TrimSpace(blob.ScanStatus), "failed") {
return reason("skill_scan_failed")
}
if !strings.EqualFold(strings.TrimSpace(blob.ScanStatus), "completed") {
return reason("skill_not_scanned")
}
risk := strings.ToLower(strings.TrimSpace(blob.RiskLevel))
if risk != skillRiskNone && risk != skillRiskLow {
return reason("skill_risk_blocked")
}
return nil
}
func (s *skillService) validateHubTagSelection(actorRole string, tagIDs []int) error {
if len(tagIDs) == 0 {
return fmt.Errorf("skill_tags_required")
}
hasPublicTag := false
for _, tagID := range tagIDs {
tag, err := s.repo.GetHubTagByID(tagID)
if err != nil {
return err
}
if tag == nil {
return fmt.Errorf("skill hub tag not found")
}
if tag.AdminOnly && !isAdminRole(actorRole) {
return fmt.Errorf("access denied")
}
if !tag.AdminOnly {
hasPublicTag = true
}
}
if !hasPublicTag {
return fmt.Errorf("skill_tags_required")
}
return nil
}
func (s *skillService) ListHubTags(actorRole string) ([]SkillHubTagPayload, error) {
tags, err := s.repo.ListHubTags(isAdminRole(actorRole))
if err != nil {
return nil, err
}
return s.hubTagsToPayload(tags), nil
}
func (s *skillService) ListHubCatalog(_ int, _ string, query SkillHubCatalogQuery) (*SkillHubCatalogResponse, error) {
if query.Page <= 0 {
query.Page = 1
}
if query.PageSize <= 0 {
query.PageSize = 20
}
if query.PageSize > 1000 {
query.PageSize = 1000
}
items, err := s.repo.ListPublicHubSkills()
if err != nil {
return nil, err
}
tagKeySet := map[string]struct{}{}
for _, key := range query.TagKeys {
key = strings.TrimSpace(key)
if key != "" {
tagKeySet[key] = struct{}{}
}
}
search := strings.ToLower(strings.TrimSpace(query.Search))
filtered := make([]SkillPayload, 0, len(items))
for _, item := range items {
blob, blobErr := s.skillBlobForPublish(&item)
if blobErr != nil || !s.isHubPublishable(&item, blob) {
continue
}
if len(tagKeySet) > 0 {
tags, err := s.repo.ListHubTagsBySkillID(item.ID)
if err != nil {
return nil, err
}
matched := false
for _, tag := range tags {
if _, ok := tagKeySet[tag.TagKey]; ok {
matched = true
break
}
}
if !matched {
continue
}
}
if search != "" {
haystack := strings.ToLower(strings.Join([]string{item.Name, item.SkillKey, derefString(item.Description)}, " "))
if !strings.Contains(haystack, search) {
continue
}
}
payload, err := s.toSkillPayload(item)
if err != nil {
return nil, err
}
if err := s.enrichSkillPayload(payload, item, nil); err != nil {
return nil, err
}
filtered = append(filtered, *payload)
}
total := len(filtered)
start := (query.Page - 1) * query.PageSize
if start > total {
start = total
}
end := start + query.PageSize
if end > total {
end = total
}
pageItems := filtered[start:end]
totalPages := total / query.PageSize
if total%query.PageSize != 0 {
totalPages++
}
if totalPages == 0 {
totalPages = 1
}
return &SkillHubCatalogResponse{
Items: pageItems,
Total: total,
Page: query.Page,
PageSize: query.PageSize,
TotalPages: totalPages,
}, nil
}
func (s *skillService) ListMyHubSkills(userID int) ([]SkillPayload, error) {
items, err := s.repo.ListSkillsByUser(userID)
if err != nil {
return nil, err
}
filtered := make([]models.Skill, 0, len(items))
for _, item := range items {
if isDeletedSkill(&item) {
continue
}
if isUserManagedSkill(item) {
filtered = append(filtered, item)
}
}
result := make([]SkillPayload, 0, len(filtered))
for _, item := range filtered {
payload, err := s.toSkillPayload(item)
if err != nil {
return nil, err
}
if err := s.enrichSkillPayload(payload, item, nil); err != nil {
return nil, err
}
result = append(result, *payload)
}
return result, nil
}
func (s *skillService) ListAllHubSkillsAdmin() ([]SkillPayload, error) {
items, err := s.repo.ListSkillsForHubAdmin()
if err != nil {
return nil, err
}
result := make([]SkillPayload, 0, len(items))
for _, item := range items {
if isDeletedSkill(&item) {
continue
}
payload, err := s.toSkillPayload(item)
if err != nil {
return nil, err
}
if err := s.enrichSkillPayload(payload, item, nil); err != nil {
return nil, err
}
result = append(result, *payload)
}
return result, nil
}
func (s *skillService) GetSkillHubDetail(actorUserID int, actorRole string, skillID int) (*SkillPayload, error) {
skill, err := s.repo.GetSkillByID(skillID)
if err != nil {
return nil, err
}
if skill == nil || !s.CanViewSkill(actorUserID, actorRole, skill) {
return nil, fmt.Errorf("skill not found")
}
payload, err := s.toSkillPayload(*skill)
if err != nil {
return nil, err
}
if err := s.enrichSkillPayload(payload, *skill, nil); err != nil {
return nil, err
}
return payload, nil
}
func (s *skillService) PublishToHub(actorUserID int, actorRole string, skillID int, tagIDs []int) (*SkillPayload, error) {
skill, err := s.repo.GetSkillByID(skillID)
if err != nil {
return nil, err
}
if skill == nil || isDeletedSkill(skill) {
return nil, fmt.Errorf("skill not found")
}
if skill.UserID != actorUserID && !isAdminRole(actorRole) {
return nil, fmt.Errorf("skill not found")
}
if err := s.validateHubTagSelection(actorRole, tagIDs); err != nil {
return nil, err
}
blob, err := s.skillBlobForPublish(skill)
if err != nil {
return nil, err
}
if strings.TrimSpace(blob.ObjectKey) == "" {
return nil, fmt.Errorf("skill_package_pending")
}
if !strings.EqualFold(strings.TrimSpace(blob.ScanStatus), "completed") {
return nil, fmt.Errorf("skill_not_scanned")
}
if !isHubPublishableBlob(blob) {
return nil, fmt.Errorf("skill_risk_blocked")
}
if strings.EqualFold(skill.SourceType, skillSourceDiscovered) {
skill.SourceType = skillSourceUploaded
}
if err := s.repo.ReplaceSkillTagAssignments(skillID, tagIDs); err != nil {
return nil, err
}
now := time.Now().UTC()
skill.Visibility = skillVisibilityPublic
skill.PublishedAt = &now
skill.PublishedBy = &actorUserID
skill.UpdatedAt = now
if err := s.repo.UpdateSkill(skill); err != nil {
return nil, err
}
return s.GetSkillHubDetail(actorUserID, actorRole, skillID)
}
func (s *skillService) UnpublishFromHub(actorUserID int, actorRole string, skillID int) (*SkillPayload, error) {
skill, err := s.repo.GetSkillByID(skillID)
if err != nil {
return nil, err
}
if skill == nil {
return nil, fmt.Errorf("skill not found")
}
if skill.UserID != actorUserID && !isAdminRole(actorRole) {
return nil, fmt.Errorf("skill not found")
}
skill.Visibility = skillVisibilityPrivate
skill.UpdatedAt = time.Now().UTC()
if err := s.repo.UpdateSkill(skill); err != nil {
return nil, err
}
return s.GetSkillHubDetail(actorUserID, actorRole, skillID)
}
func (s *skillService) UpdateHubTags(actorUserID int, actorRole string, skillID int, tagIDs []int) (*SkillPayload, error) {
skill, err := s.repo.GetSkillByID(skillID)
if err != nil {
return nil, err
}
if skill == nil {
return nil, fmt.Errorf("skill not found")
}
if skill.UserID != actorUserID && !isAdminRole(actorRole) {
return nil, fmt.Errorf("skill not found")
}
if !strings.EqualFold(strings.TrimSpace(skill.Visibility), skillVisibilityPublic) {
return nil, fmt.Errorf("skill is not published to hub")
}
if err := s.validateHubTagSelection(actorRole, tagIDs); err != nil {
return nil, err
}
if err := s.repo.ReplaceSkillTagAssignments(skillID, tagIDs); err != nil {
return nil, err
}
skill.UpdatedAt = time.Now().UTC()
if err := s.repo.UpdateSkill(skill); err != nil {
return nil, err
}
return s.GetSkillHubDetail(actorUserID, actorRole, skillID)
}
func (s *skillService) InstallHubSkill(actorUserID int, actorRole string, skillID, instanceID int) (*InstanceSkillPayload, error) {
return s.AttachSkillToInstance(actorUserID, actorRole, instanceID, skillID)
}
func (s *skillService) ImportInstanceSkillToLibrary(actorUserID int, actorRole string, instanceID, skillID int) (*SkillPayload, error) {
instance, err := s.instanceRepo.GetByID(instanceID)
if err != nil {
return nil, err
}
if instance == nil {
return nil, fmt.Errorf("instance not found")
}
if !isAdminRole(actorRole) && instance.UserID != actorUserID {
return nil, fmt.Errorf("access denied")
}
skill, err := s.repo.GetSkillByID(skillID)
if err != nil {
return nil, err
}
if skill == nil || skill.UserID != instance.UserID || isDeletedSkill(skill) {
return nil, fmt.Errorf("skill not found")
}
instanceSkill, err := s.repo.GetInstanceSkill(instanceID, skillID)
if err != nil {
return nil, err
}
if instanceSkill == nil || instanceSkill.Status == "removed" {
return nil, fmt.Errorf("skill not found on instance")
}
if isUserManagedSkill(*skill) {
blob, blobErr := s.skillBlobForPublish(skill)
if blobErr == nil && strings.TrimSpace(blob.ObjectKey) != "" && strings.EqualFold(strings.TrimSpace(blob.ScanStatus), "completed") {
return s.GetSkillHubDetail(actorUserID, actorRole, skillID)
}
}
if err := s.requestSkillPackageCollection(instanceID, skill, instanceSkill, fmt.Sprintf("import-%d-%d", instanceID, skillID)); err != nil {
return nil, err
}
blob, err := s.skillBlobForPublish(skill)
if err != nil {
return nil, err
}
content, err := s.storage.GetObject(context.Background(), blob.ObjectKey)
if err != nil {
return nil, err
}
if err := s.ensureBlobObject(context.Background(), blob, content); err != nil {
return nil, err
}
if blob.LastScanResultID == nil || !strings.EqualFold(strings.TrimSpace(blob.ScanStatus), "completed") {
if err := s.recordScanFromStoredBlob(blob); err != nil {
return nil, err
}
}
skill, err = s.repo.GetSkillByID(skillID)
if err != nil {
return nil, err
}
if skill == nil {
return nil, fmt.Errorf("skill not found")
}
blob, err = s.repo.GetBlobByID(blob.ID)
if err != nil {
return nil, err
}
if blob != nil {
skill.RiskLevel = blob.RiskLevel
skill.LastScannedAt = blob.LastScannedAt
skill.LastScanResultID = blob.LastScanResultID
}
if err := s.promoteSkillToUploadedLibrary(skill); err != nil {
return nil, err
}
return s.GetSkillHubDetail(actorUserID, actorRole, skillID)
}
func (s *skillService) RetrySkillPackageCollection(actorUserID int, actorRole string, instanceID, skillID int) error {
instance, err := s.instanceRepo.GetByID(instanceID)
if err != nil {
return err
}
if instance == nil {
return fmt.Errorf("instance not found")
}
if !isAdminRole(actorRole) && instance.UserID != actorUserID {
return fmt.Errorf("access denied")
}
skill, err := s.repo.GetSkillByID(skillID)
if err != nil {
return err
}
if skill == nil || skill.UserID != instance.UserID || isDeletedSkill(skill) {
return fmt.Errorf("skill not found")
}
instanceSkill, err := s.repo.GetInstanceSkill(instanceID, skillID)
if err != nil {
return err
}
if instanceSkill == nil || instanceSkill.Status == "removed" {
return fmt.Errorf("skill not found on instance")
}
blob, blobErr := s.skillBlobForPublish(skill)
if blobErr == nil && strings.TrimSpace(blob.ObjectKey) != "" && strings.EqualFold(strings.TrimSpace(blob.ScanStatus), "completed") {
return nil
}
if isLiteRuntimeInstance(instance) && s.materializeService != nil {
if job, findErr := s.materializeService.FindLatestBySkillID(skillID); findErr == nil && job != nil && strings.EqualFold(strings.TrimSpace(job.Status), MaterializeJobStatusFailed) {
_ = s.materializeService.RetryJob(skillID)
}
}
return s.requestSkillPackageCollection(instanceID, skill, instanceSkill, fmt.Sprintf("retry-%d-%d-%d", instanceID, skillID, time.Now().Unix()))
}
func (s *skillService) PublishFromInstance(actorUserID int, actorRole string, instanceID, skillID int, tagIDs []int) (*SkillPayload, error) {
instance, err := s.instanceRepo.GetByID(instanceID)
if err != nil {
return nil, err
}
if instance == nil {
return nil, fmt.Errorf("instance not found")
}
if !isAdminRole(actorRole) && instance.UserID != actorUserID {
return nil, fmt.Errorf("access denied")
}
skill, err := s.repo.GetSkillByID(skillID)
if err != nil {
return nil, err
}
if skill == nil || skill.UserID != instance.UserID {
return nil, fmt.Errorf("skill not found")
}
if !isUserManagedSkill(*skill) {
return nil, fmt.Errorf("skill_not_in_library")
}
instanceSkill, err := s.repo.GetInstanceSkill(instanceID, skillID)
if err != nil {
return nil, err
}
if instanceSkill == nil || instanceSkill.Status == "removed" {
return nil, fmt.Errorf("skill not found on instance")
}
if err := s.requestSkillPackageCollection(instanceID, skill, instanceSkill, fmt.Sprintf("publish-%d-%d", instanceID, skillID)); err != nil {
return nil, err
}
return s.PublishToHub(actorUserID, actorRole, skillID, tagIDs)
}
func (s *skillService) ListAttachableSkills(actorUserID int, actorRole string) ([]SkillPayload, error) {
result := make([]SkillPayload, 0)
seen := map[int]struct{}{}
mine, err := s.ListMyHubSkills(actorUserID)
if err != nil {
return nil, err
}
for _, item := range mine {
if strings.EqualFold(item.SourceType, skillSourceDiscovered) {
continue
}
if item.Status != "active" || item.RiskLevel == skillRiskMedium || item.RiskLevel == skillRiskHigh {
continue
}
if _, ok := seen[item.ID]; ok {
continue
}
seen[item.ID] = struct{}{}
result = append(result, item)
}
catalog, err := s.ListHubCatalog(actorUserID, actorRole, SkillHubCatalogQuery{Page: 1, PageSize: 1000})
if err != nil {
return nil, err
}
for _, item := range catalog.Items {
if item.UserID == actorUserID {
continue
}
if _, ok := seen[item.ID]; ok {
continue
}
seen[item.ID] = struct{}{}
result = append(result, item)
}
return result, nil
}
func (s *skillService) ImportHubArchive(ctx context.Context, userID int, fileHeader *multipart.FileHeader) ([]SkillPayload, error) {
items, err := s.ImportHubArchiveWithDecisions(ctx, userID, fileHeader, nil)
if err != nil {
return nil, err
}
results := make([]SkillPayload, 0, len(items))
for _, item := range items {
results = append(results, item.Skill)
}
return results, nil
}
backend/internal/services/skill_hub_service_test.go
+957
View File
@@ -0,0 +1,957 @@
package services
import (
"context"
"fmt"
"strings"
"testing"
"time"
"clawreef/internal/models"
)
func TestIsHubPublishableBlob(t *testing.T) {
tests := []struct {
name string
blob *models.SkillBlob
want bool
}{
{
name: "completed none risk with object key",
blob: &models.SkillBlob{ScanStatus: "completed", RiskLevel: skillRiskNone, ObjectKey: "user/demo/hash.zip"},
want: true,
},
{
name: "completed low risk",
blob: &models.SkillBlob{ScanStatus: "completed", RiskLevel: skillRiskLow, ObjectKey: "key"},
want: true,
},
{
name: "medium risk blocked",
blob: &models.SkillBlob{ScanStatus: "completed", RiskLevel: skillRiskMedium, ObjectKey: "key"},
want: false,
},
{
name: "pending scan blocked",
blob: &models.SkillBlob{ScanStatus: "pending", RiskLevel: skillRiskNone, ObjectKey: "key"},
want: false,
},
{
name: "missing object key blocked",
blob: &models.SkillBlob{ScanStatus: "completed", RiskLevel: skillRiskNone, ObjectKey: ""},
want: false,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if got := isHubPublishableBlob(tc.blob); got != tc.want {
t.Fatalf("isHubPublishableBlob() = %v, want %v", got, tc.want)
}
})
}
}
func TestListMyHubSkillsExcludesDiscoveredSkills(t *testing.T) {
svc := &skillService{
repo: &skillRepoStub{
skills: map[int]*models.Skill{
1: {ID: 1, UserID: 1, SkillKey: "dogfood", Name: "dogfood", Status: skillStatusActive, SourceType: skillSourceUploaded},
2: {ID: 2, UserID: 1, SkillKey: "software-development-spike", Name: "software-development/spike", Status: skillStatusActive, SourceType: skillSourceDiscovered},
},
},
}
items, err := svc.ListMyHubSkills(1)
if err != nil {
t.Fatalf("ListMyHubSkills() error = %v", err)
}
if len(items) != 1 {
t.Fatalf("items = %d, want 1 uploaded skill only", len(items))
}
if items[0].SkillKey != "dogfood" {
t.Fatalf("SkillKey = %q, want dogfood", items[0].SkillKey)
}
}
func TestCanAttachSkillRules(t *testing.T) {
svc := &skillService{}
privateSkill := &models.Skill{UserID: 1, SourceType: skillSourceUploaded, Status: "active", Visibility: skillVisibilityPrivate, RiskLevel: skillRiskLow}
publicSkill := &models.Skill{UserID: 2, SourceType: skillSourceUploaded, Status: "active", Visibility: skillVisibilityPublic, RiskLevel: skillRiskLow}
ownInstance := &models.Instance{UserID: 1}
otherInstance := &models.Instance{UserID: 3}
if !svc.CanAttachSkill(1, "user", privateSkill, ownInstance) {
t.Fatal("owner should attach private skill to own instance")
}
if svc.CanAttachSkill(3, "user", privateSkill, otherInstance) {
t.Fatal("other user must not attach private skill")
}
if !svc.CanAttachSkill(3, "user", publicSkill, otherInstance) {
t.Fatal("user should attach public skill to own instance")
}
if svc.CanAttachSkill(3, "user", publicSkill, ownInstance) {
t.Fatal("user must not attach public skill to someone else's instance")
}
if !svc.CanAttachSkill(99, "admin", privateSkill, otherInstance) {
t.Fatal("admin should attach any skill to any instance")
}
}
func TestCanViewSkillRules(t *testing.T) {
svc := &skillService{}
privateSkill := &models.Skill{UserID: 1, SourceType: skillSourceUploaded, Visibility: skillVisibilityPrivate}
publicSkill := &models.Skill{UserID: 2, SourceType: skillSourceUploaded, Visibility: skillVisibilityPublic}
if !svc.CanViewSkill(1, "user", privateSkill) {
t.Fatal("owner should view private skill")
}
if svc.CanViewSkill(3, "user", privateSkill) {
t.Fatal("other user must not view private skill")
}
if !svc.CanViewSkill(3, "user", publicSkill) {
t.Fatal("user should view public skill")
}
if !svc.CanViewSkill(99, "admin", privateSkill) {
t.Fatal("admin should view private skill")
}
}
func TestCanDownloadSkillRules(t *testing.T) {
svc := &skillService{}
privateSkill := &models.Skill{UserID: 1, SourceType: skillSourceUploaded, Visibility: skillVisibilityPrivate}
publicSkill := &models.Skill{UserID: 2, SourceType: skillSourceUploaded, Visibility: skillVisibilityPublic}
discoveredSkill := &models.Skill{UserID: 1, SourceType: skillSourceDiscovered, Visibility: skillVisibilityPublic}
if !svc.CanDownloadSkill(1, "user", privateSkill) {
t.Fatal("owner should download private uploaded skill")
}
if svc.CanDownloadSkill(3, "user", privateSkill) {
t.Fatal("other user must not download private skill")
}
if !svc.CanDownloadSkill(3, "user", publicSkill) {
t.Fatal("user should download public skill")
}
if svc.CanDownloadSkill(1, "user", discoveredSkill) {
t.Fatal("discovered skill is not user-managed and must not download")
}
}
func TestIsHubPublishableSkill(t *testing.T) {
svc := &skillService{}
blob := &models.SkillBlob{ScanStatus: "completed", RiskLevel: skillRiskNone, ObjectKey: "user/demo/hash.zip"}
activeSkill := &models.Skill{SourceType: skillSourceUploaded, Status: "active"}
inactiveSkill := &models.Skill{SourceType: skillSourceUploaded, Status: "inactive"}
if !svc.isHubPublishable(activeSkill, blob) {
t.Fatal("active uploaded skill with clean blob should be publishable")
}
if svc.isHubPublishable(inactiveSkill, blob) {
t.Fatal("inactive skill must not be publishable")
}
if svc.isHubPublishable(activeSkill, &models.SkillBlob{ScanStatus: "pending", RiskLevel: skillRiskNone, ObjectKey: "key"}) {
t.Fatal("pending scan blob must block publish")
}
}
func TestValidateHubTagSelection(t *testing.T) {
svc := &skillService{repo: &hubTagRepoStub{
tags: map[int]*models.SkillHubTag{
1: {ID: 1, TagKey: "coding", Name: "Coding", AdminOnly: false},
2: {ID: 2, TagKey: "featured", Name: "Featured", AdminOnly: true},
},
}}
if err := svc.validateHubTagSelection("user", nil); err == nil || err.Error() != "skill_tags_required" {
t.Fatalf("empty tag list should require tags, got %v", err)
}
if err := svc.validateHubTagSelection("user", []int{2}); err == nil || err.Error() != "access denied" {
t.Fatalf("user must not select admin-only tag alone, got %v", err)
}
if err := svc.validateHubTagSelection("user", []int{1}); err != nil {
t.Fatalf("user with public tag should pass, got %v", err)
}
if err := svc.validateHubTagSelection("admin", []int{2}); err == nil || err.Error() != "skill_tags_required" {
t.Fatalf("admin-only tag alone should still require a public tag, got %v", err)
}
if err := svc.validateHubTagSelection("admin", []int{1, 2}); err != nil {
t.Fatalf("admin with mixed tags should pass, got %v", err)
}
}
func newPublishTestStub(blob *models.SkillBlob) (*skillService, *skillRepoStub) {
versionID := 10
blobID := 20
stub := &skillRepoStub{
skills: map[int]*models.Skill{
1: {
ID: 1, UserID: 1, SkillKey: "demo", Name: "Demo", Status: skillStatusActive,
SourceType: skillSourceUploaded, Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID,
},
},
versions: map[int]*models.SkillVersion{versionID: {ID: versionID, BlobID: blobID}},
blobs: map[int]*models.SkillBlob{blobID: blob},
tags: map[int]*models.SkillHubTag{
1: {ID: 1, TagKey: "coding", Name: "Coding", AdminOnly: false},
},
tagAssignments: map[int][]int{},
}
return &skillService{repo: stub}, stub
}
func TestDeleteSkillSoftPreservesInstanceSkills(t *testing.T) {
stub := &skillRepoStub{
skills: map[int]*models.Skill{
1: {ID: 1, UserID: 1, SkillKey: "demo", Name: "Demo", Status: skillStatusActive, SourceType: skillSourceUploaded, Visibility: skillVisibilityPublic},
},
tagAssignments: map[int][]int{1: {1}},
instanceSkillsBySkillID: map[int][]models.InstanceSkill{
1: {{ID: 99, InstanceID: 2, SkillID: 1, Status: "active"}},
},
}
svc := &skillService{repo: stub}
if err := svc.DeleteSkill(1, "user", 1); err != nil {
t.Fatalf("DeleteSkill() error = %v", err)
}
if stub.hardDeleteCalled {
t.Fatal("DeleteSkill must not hard-delete skill row")
}
if stub.skills[1].Status != skillStatusDeleted {
t.Fatalf("expected soft-deleted status, got %q", stub.skills[1].Status)
}
if len(stub.tagAssignments[1]) != 0 {
t.Fatalf("expected tag assignments cleared, got %v", stub.tagAssignments[1])
}
if len(stub.instanceSkillsBySkillID[1]) != 1 {
t.Fatal("instance_skills records must remain after soft delete")
}
}
func TestCanViewSkillDeletedHidden(t *testing.T) {
svc := &skillService{}
deletedSkill := &models.Skill{UserID: 1, SourceType: skillSourceUploaded, Visibility: skillVisibilityPublic, Status: skillStatusDeleted}
if svc.CanViewSkill(3, "user", deletedSkill) {
t.Fatal("deleted public skill must not be viewable")
}
}
func TestPublishToHubRejectsPendingScan(t *testing.T) {
svc, _ := newPublishTestStub(&models.SkillBlob{ScanStatus: "pending", RiskLevel: skillRiskNone, ObjectKey: "key.zip"})
_, err := svc.PublishToHub(1, "user", 1, []int{1})
if err == nil || err.Error() != "skill_not_scanned" {
t.Fatalf("expected skill_not_scanned, got %v", err)
}
}
func TestPublishToHubRejectsMediumRisk(t *testing.T) {
svc, _ := newPublishTestStub(&models.SkillBlob{ScanStatus: "completed", RiskLevel: skillRiskMedium, ObjectKey: "key.zip"})
_, err := svc.PublishToHub(1, "user", 1, []int{1})
if err == nil || err.Error() != "skill_risk_blocked" {
t.Fatalf("expected skill_risk_blocked, got %v", err)
}
}
func TestPublishToHubRejectsEmptyTags(t *testing.T) {
svc, _ := newPublishTestStub(&models.SkillBlob{ScanStatus: "completed", RiskLevel: skillRiskNone, ObjectKey: "key.zip"})
_, err := svc.PublishToHub(1, "user", 1, nil)
if err == nil || err.Error() != "skill_tags_required" {
t.Fatalf("expected skill_tags_required, got %v", err)
}
}
func TestPublishToHubAllowsAdminForOtherUsersSkill(t *testing.T) {
svc, _ := newPublishTestStub(&models.SkillBlob{ScanStatus: "completed", RiskLevel: skillRiskNone, ObjectKey: "key.zip"})
item, err := svc.PublishToHub(99, "admin", 1, []int{1})
if err != nil {
t.Fatalf("PublishToHub() error = %v", err)
}
if item == nil || !strings.EqualFold(item.Visibility, skillVisibilityPublic) {
t.Fatalf("expected admin publish to succeed, got %#v", item)
}
}
func TestUnpublishAllowsAdminForOtherUsersSkill(t *testing.T) {
svc, stub := newPublishTestStub(&models.SkillBlob{ScanStatus: "completed", RiskLevel: skillRiskNone, ObjectKey: "key.zip"})
stub.skills[1].Visibility = skillVisibilityPublic
item, err := svc.UnpublishFromHub(99, "admin", 1)
if err != nil {
t.Fatalf("UnpublishFromHub() error = %v", err)
}
if item == nil || !strings.EqualFold(item.Visibility, skillVisibilityPrivate) {
t.Fatalf("expected admin unpublish to succeed, got %#v", item)
}
}
func TestUnpublishRemovesFromPublicCatalog(t *testing.T) {
versionID := 10
blobID := 20
stub := &skillRepoStub{
skills: map[int]*models.Skill{
1: {
ID: 1, UserID: 1, SkillKey: "demo", Name: "Demo", Status: skillStatusActive,
SourceType: skillSourceUploaded, Visibility: skillVisibilityPublic, CurrentVersionID: &versionID,
},
},
versions: map[int]*models.SkillVersion{10: {ID: 10, BlobID: blobID}},
blobs: map[int]*models.SkillBlob{blobID: {ScanStatus: "completed", RiskLevel: skillRiskNone, ObjectKey: "key.zip"}},
}
svc := &skillService{repo: stub}
if _, err := svc.UnpublishFromHub(1, "user", 1); err != nil {
t.Fatalf("UnpublishFromHub() error = %v", err)
}
if stub.skills[1].Visibility != skillVisibilityPrivate {
t.Fatalf("expected private visibility after unpublish, got %q", stub.skills[1].Visibility)
}
publicSkills, err := stub.ListPublicHubSkills()
if err != nil {
t.Fatalf("ListPublicHubSkills() error = %v", err)
}
if len(publicSkills) != 0 {
t.Fatalf("catalog source should be empty after unpublish, got %d items", len(publicSkills))
}
}
func TestListAllHubSkillsAdminExcludesDeleted(t *testing.T) {
stub := &skillRepoStub{
skills: map[int]*models.Skill{
1: {ID: 1, UserID: 1, SkillKey: "demo", Name: "Demo", Status: skillStatusActive, SourceType: skillSourceUploaded, Visibility: skillVisibilityPublic},
2: {ID: 2, UserID: 1, SkillKey: "demo__deleted_2", Name: "Demo", Status: skillStatusDeleted, SourceType: skillSourceUploaded, Visibility: skillVisibilityPrivate},
},
tagAssignments: map[int][]int{},
}
svc := &skillService{repo: stub}
items, err := svc.ListAllHubSkillsAdmin()
if err != nil {
t.Fatalf("ListAllHubSkillsAdmin() error = %v", err)
}
if len(items) != 1 {
t.Fatalf("ListAllHubSkillsAdmin() len = %d, want 1", len(items))
}
if items[0].ID != 1 {
t.Fatalf("ListAllHubSkillsAdmin() id = %d, want 1", items[0].ID)
}
}
type importTestInstanceRepo struct {
instances map[int]*models.Instance
}
func (r *importTestInstanceRepo) Create(*models.Instance) error { panic("not used") }
func (r *importTestInstanceRepo) GetByID(id int) (*models.Instance, error) {
if inst, ok := r.instances[id]; ok {
copy := *inst
return &copy, nil
}
return nil, nil
}
func (r *importTestInstanceRepo) GetByAccessToken(string) (*models.Instance, error) { panic("not used") }
func (r *importTestInstanceRepo) GetByAgentBootstrapToken(string) (*models.Instance, error) {
panic("not used")
}
func (r *importTestInstanceRepo) GetAll(int, int) ([]models.Instance, error) {
items := make([]models.Instance, 0, len(r.instances))
for _, inst := range r.instances {
items = append(items, *inst)
}
return items, nil
}
func (r *importTestInstanceRepo) CountAll() (int, error) { panic("not used") }
func (r *importTestInstanceRepo) GetByUserID(int, int, int) ([]models.Instance, error) {
panic("not used")
}
func (r *importTestInstanceRepo) CountByUserID(int) (int, error) { panic("not used") }
func (r *importTestInstanceRepo) CountActiveByMode(context.Context, string) (int, error) {
panic("not used")
}
func (r *importTestInstanceRepo) ExistsByUserIDAndName(int, string) (bool, error) {
panic("not used")
}
func (r *importTestInstanceRepo) GetAllRunning() ([]models.Instance, error) { panic("not used") }
func (r *importTestInstanceRepo) GetV2DesiredRunning(context.Context, int) ([]models.Instance, error) {
panic("not used")
}
func (r *importTestInstanceRepo) GetV2Creating(context.Context, int) ([]models.Instance, error) {
panic("not used")
}
func (r *importTestInstanceRepo) UpdateRuntimeState(context.Context, int, string, int, *string) error {
panic("not used")
}
func (r *importTestInstanceRepo) SetWorkspacePath(context.Context, int, string) error {
panic("not used")
}
func (r *importTestInstanceRepo) UpdateWorkspaceUsage(context.Context, int, int64) error {
panic("not used")
}
func (r *importTestInstanceRepo) Update(*models.Instance) error { panic("not used") }
func (r *importTestInstanceRepo) Delete(int) error { panic("not used") }
type noopInstanceCommandService struct{}
func (n *noopInstanceCommandService) Create(int, *int, CreateInstanceCommandRequest) (*InstanceCommandPayload, error) {
return nil, nil
}
func (n *noopInstanceCommandService) GetNextForAgent(*AgentSession) (*AgentCommandEnvelope, error) {
panic("not used")
}
func (n *noopInstanceCommandService) MarkStarted(*AgentSession, int, *time.Time) error {
panic("not used")
}
func (n *noopInstanceCommandService) MarkFinished(*AgentSession, int, AgentCommandFinishRequest) error {
panic("not used")
}
func (n *noopInstanceCommandService) ListByInstanceID(int, int) ([]InstanceCommandPayload, error) {
panic("not used")
}
func TestPublishFromInstanceRejectsDiscoveredSkill(t *testing.T) {
versionID := 10
blobID := 20
stub := &skillRepoStub{
skills: map[int]*models.Skill{
1: {
ID: 1, UserID: 1, SkillKey: "demo", Name: "Demo", Status: skillStatusActive,
SourceType: skillSourceDiscovered, Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID,
},
},
versions: map[int]*models.SkillVersion{versionID: {ID: versionID, SkillID: 1, BlobID: blobID}},
blobs: map[int]*models.SkillBlob{blobID: {ID: blobID, ScanStatus: "completed", RiskLevel: skillRiskNone, ObjectKey: "key.zip"}},
instanceSkills: []models.InstanceSkill{{InstanceID: 1, SkillID: 1, Status: "active", SourceType: "discovered_in_instance"}},
}
instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{1: {ID: 1, UserID: 1}}}
svc := &skillService{repo: stub, instanceRepo: instRepo, commandService: &noopInstanceCommandService{}}
_, err := svc.PublishFromInstance(1, "user", 1, 1, []int{1})
if err == nil || err.Error() != "skill_not_in_library" {
t.Fatalf("expected skill_not_in_library, got %v", err)
}
}
func TestImportInstanceSkillToLibraryPendingPackage(t *testing.T) {
versionID := 10
blobID := 20
stub := &skillRepoStub{
skills: map[int]*models.Skill{
1: {
ID: 1, UserID: 1, SkillKey: "demo", Name: "Demo", Status: skillStatusActive,
SourceType: skillSourceDiscovered, Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID,
},
},
versions: map[int]*models.SkillVersion{versionID: {ID: versionID, SkillID: 1, BlobID: blobID}},
blobs: map[int]*models.SkillBlob{blobID: {ID: blobID, ScanStatus: "pending", RiskLevel: skillRiskUnknown, ObjectKey: ""}},
instanceSkills: []models.InstanceSkill{{InstanceID: 1, SkillID: 1, Status: "active", SourceType: "discovered_in_instance"}},
tagAssignments: map[int][]int{},
tags: map[int]*models.SkillHubTag{
1: {ID: 1, TagKey: "coding", Name: "Coding", AdminOnly: false},
},
}
instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{1: {ID: 1, UserID: 1}}}
svc := &skillService{repo: stub, instanceRepo: instRepo, commandService: &noopInstanceCommandService{}}
_, err := svc.ImportInstanceSkillToLibrary(1, "user", 1, 1)
if err == nil || err.Error() != "skill_package_pending" {
t.Fatalf("expected skill_package_pending, got %v", err)
}
}
func TestImportInstanceSkillToLibraryRejectsNonOwner(t *testing.T) {
versionID := 10
blobID := 20
stub := &skillRepoStub{
skills: map[int]*models.Skill{
1: {
ID: 1, UserID: 1, SkillKey: "demo", Name: "Demo", Status: skillStatusActive,
SourceType: skillSourceDiscovered, Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID,
},
},
versions: map[int]*models.SkillVersion{versionID: {ID: versionID, SkillID: 1, BlobID: blobID}},
blobs: map[int]*models.SkillBlob{blobID: {ID: blobID, ScanStatus: "pending", RiskLevel: skillRiskUnknown, ObjectKey: ""}},
instanceSkills: []models.InstanceSkill{{InstanceID: 1, SkillID: 1, Status: "active", SourceType: "discovered_in_instance"}},
tagAssignments: map[int][]int{},
}
instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{1: {ID: 1, UserID: 1}}}
svc := &skillService{repo: stub, instanceRepo: instRepo, commandService: &noopInstanceCommandService{}}
_, err := svc.ImportInstanceSkillToLibrary(2, "user", 1, 1)
if err == nil || err.Error() != "access denied" {
t.Fatalf("expected access denied, got %v", err)
}
}
func TestImportInstanceSkillToLibraryRejectsDeletedSkill(t *testing.T) {
versionID := 10
blobID := 20
stub := &skillRepoStub{
skills: map[int]*models.Skill{
1: {
ID: 1, UserID: 1, SkillKey: "demo", Name: "Demo", Status: skillStatusDeleted,
SourceType: skillSourceDiscovered, Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID,
},
},
versions: map[int]*models.SkillVersion{versionID: {ID: versionID, SkillID: 1, BlobID: blobID}},
blobs: map[int]*models.SkillBlob{blobID: {ID: blobID, ScanStatus: "completed", RiskLevel: skillRiskNone, ObjectKey: "discovered/1/demo.zip"}},
instanceSkills: []models.InstanceSkill{{InstanceID: 1, SkillID: 1, Status: "active", SourceType: "discovered_in_instance"}},
tagAssignments: map[int][]int{},
}
instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{1: {ID: 1, UserID: 1}}}
svc := &skillService{repo: stub, instanceRepo: instRepo, commandService: &noopInstanceCommandService{}}
_, err := svc.ImportInstanceSkillToLibrary(1, "user", 1, 1)
if err == nil || err.Error() != "skill not found" {
t.Fatalf("expected skill not found, got %v", err)
}
}
func TestImportInstanceSkillToLibraryPromotesDiscoveredSkill(t *testing.T) {
versionID := 10
blobID := 20
scanResultID := 99
objectKey := "discovered/1/demo.zip"
stub := &skillRepoStub{
skills: map[int]*models.Skill{
1: {
ID: 1, UserID: 1, SkillKey: "demo", Name: "Demo", Status: skillStatusActive,
SourceType: skillSourceDiscovered, Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID,
},
},
versions: map[int]*models.SkillVersion{versionID: {ID: versionID, SkillID: 1, BlobID: blobID, SourceType: skillSourceDiscovered}},
blobs: map[int]*models.SkillBlob{
blobID: {
ID: blobID, ScanStatus: "completed", RiskLevel: skillRiskNone, ObjectKey: objectKey,
LastScanResultID: &scanResultID,
},
},
instanceSkills: []models.InstanceSkill{{InstanceID: 1, SkillID: 1, Status: "active", SourceType: "discovered_in_instance"}},
tagAssignments: map[int][]int{},
}
instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{1: {ID: 1, UserID: 1}}}
storage := &importTestObjectStorage{objects: map[string][]byte{objectKey: []byte("fake-zip")}}
svc := &skillService{repo: stub, instanceRepo: instRepo, commandService: &noopInstanceCommandService{}, storage: storage}
_, err := svc.ImportInstanceSkillToLibrary(1, "user", 1, 1)
if err != nil {
t.Fatalf("ImportInstanceSkillToLibrary() error = %v", err)
}
if stub.skills[1].SourceType != skillSourceUploaded {
t.Fatalf("skill source_type = %q, want %q", stub.skills[1].SourceType, skillSourceUploaded)
}
if stub.versions[versionID].SourceType != skillSourceUploaded {
t.Fatalf("version source_type = %q, want %q", stub.versions[versionID].SourceType, skillSourceUploaded)
}
}
func TestImportInstanceSkillToLibraryLiteMaterializesPackage(t *testing.T) {
versionID := 10
blobID := 20
contentHash := "abc123def456789012345678901234"
workspaceDir := "yuanbao"
stub := &skillRepoStub{
skills: map[int]*models.Skill{
1: {
ID: 1, UserID: 1, SkillKey: "yuanbao", Name: "yuanbao", Status: skillStatusActive,
SourceType: skillSourceDiscovered, Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID,
},
},
versions: map[int]*models.SkillVersion{versionID: {ID: versionID, SkillID: 1, BlobID: blobID, SourceType: skillSourceDiscovered}},
blobs: map[int]*models.SkillBlob{
blobID: {ID: blobID, ContentHash: contentHash, ScanStatus: "pending", RiskLevel: skillRiskUnknown, ObjectKey: ""},
},
instanceSkills: []models.InstanceSkill{{
InstanceID: 1, SkillID: 1, Status: "active", SourceType: "discovered_in_instance",
WorkspaceDir: &workspaceDir,
}},
tagAssignments: map[int][]int{},
}
instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{
1: {ID: 1, UserID: 1, InstanceMode: InstanceModeLite, RuntimeType: RuntimeBackendGateway},
}}
storage := &importTestObjectStorage{objects: map[string][]byte{}}
cmdSvc := &capturingInstanceCommandService{}
matSvc := NewSkillPackageMaterializeService(
&materializeJobRepoStub{},
stub,
importLiteMaterializer{repo: stub, storage: storage, blobID: blobID},
)
svc := &skillService{
repo: stub, instanceRepo: instRepo, commandService: cmdSvc, storage: storage, materializeService: matSvc,
}
_, err := svc.ImportInstanceSkillToLibrary(1, "user", 1, 1)
if err != nil {
t.Fatalf("ImportInstanceSkillToLibrary() error = %v", err)
}
for _, req := range cmdSvc.created {
if req.CommandType == InstanceCommandTypeCollectSkillPackage {
t.Fatalf("unexpected collect_skill_package command: %#v", req)
}
}
if stub.skills[1].SourceType != skillSourceUploaded {
t.Fatalf("skill source_type = %q, want %q", stub.skills[1].SourceType, skillSourceUploaded)
}
blob := stub.blobs[blobID]
if blob == nil || strings.TrimSpace(blob.ObjectKey) == "" {
t.Fatalf("expected materialized object key, got %#v", blob)
}
}
func TestImportInstanceSkillToLibraryIdempotentForUploaded(t *testing.T) {
versionID := 10
blobID := 20
scanResultID := 99
objectKey := "user/1/demo.zip"
stub := &skillRepoStub{
skills: map[int]*models.Skill{
1: {
ID: 1, UserID: 1, SkillKey: "demo", Name: "Demo", Status: skillStatusActive,
SourceType: skillSourceUploaded, Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID,
},
},
versions: map[int]*models.SkillVersion{versionID: {ID: versionID, SkillID: 1, BlobID: blobID, SourceType: skillSourceUploaded}},
blobs: map[int]*models.SkillBlob{
blobID: {
ID: blobID, ScanStatus: "completed", RiskLevel: skillRiskNone, ObjectKey: objectKey,
LastScanResultID: &scanResultID,
},
},
instanceSkills: []models.InstanceSkill{{InstanceID: 1, SkillID: 1, Status: "active", SourceType: "discovered_in_instance"}},
tagAssignments: map[int][]int{},
}
instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{1: {ID: 1, UserID: 1}}}
storage := &importTestObjectStorage{objects: map[string][]byte{objectKey: []byte("fake-zip")}}
svc := &skillService{repo: stub, instanceRepo: instRepo, commandService: &noopInstanceCommandService{}, storage: storage}
payload, err := svc.ImportInstanceSkillToLibrary(1, "user", 1, 1)
if err != nil {
t.Fatalf("ImportInstanceSkillToLibrary() error = %v", err)
}
if stub.skills[1].SourceType != skillSourceUploaded {
t.Fatalf("skill source_type = %q, want %q", stub.skills[1].SourceType, skillSourceUploaded)
}
if payload == nil || payload.SourceType != skillSourceUploaded {
t.Fatalf("payload source_type = %v, want %q", payload, skillSourceUploaded)
}
}
func TestSyncAgentSkillsCreatesDiscoveredSkillWithPrivateVisibility(t *testing.T) {
stub := &capturingSkillRepoStub{
skillRepoStub: skillRepoStub{
skills: map[int]*models.Skill{},
blobs: map[int]*models.SkillBlob{},
versions: map[int]*models.SkillVersion{},
},
}
instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{1: {ID: 1, UserID: 1}}}
svc := &skillService{repo: stub, instanceRepo: instRepo, commandService: &noopInstanceCommandService{}}
err := svc.SyncAgentSkills(1, AgentSkillInventoryReportRequest{
Skills: []AgentSkillRecord{{
Identifier: "weather",
ContentMD5: "abc123def456789012345678901234",
Source: "discovered_in_instance",
}},
})
if err != nil {
t.Fatalf("SyncAgentSkills() error = %v", err)
}
if len(stub.createdSkills) != 1 {
t.Fatalf("created %d skills, want 1", len(stub.createdSkills))
}
if stub.createdSkills[0].Visibility != skillVisibilityPrivate {
t.Fatalf("visibility = %q, want %q", stub.createdSkills[0].Visibility, skillVisibilityPrivate)
}
}
type capturingSkillRepoStub struct {
skillRepoStub
createdSkills []*models.Skill
nextSkillID int
markMissingCalls int
lastMarkMissingActive []int
}
func (s *capturingSkillRepoStub) CreateSkill(skill *models.Skill) error {
s.nextSkillID++
skill.ID = s.nextSkillID
copy := *skill
s.createdSkills = append(s.createdSkills, &copy)
if s.skills == nil {
s.skills = map[int]*models.Skill{}
}
stored := *skill
s.skills[skill.ID] = &stored
return nil
}
func (s *capturingSkillRepoStub) CreateBlob(blob *models.SkillBlob) error {
if s.blobs == nil {
s.blobs = map[int]*models.SkillBlob{}
}
s.nextSkillID++
blob.ID = s.nextSkillID
stored := *blob
s.blobs[blob.ID] = &stored
return nil
}
func (s *capturingSkillRepoStub) CreateVersion(version *models.SkillVersion) error {
if s.versions == nil {
s.versions = map[int]*models.SkillVersion{}
}
s.nextSkillID++
version.ID = s.nextSkillID
stored := *version
s.versions[version.ID] = &stored
return nil
}
func (s *capturingSkillRepoStub) GetBlobByContentHash(hash string) (*models.SkillBlob, error) {
for _, blob := range s.blobs {
if blob != nil && blob.ContentHash == hash {
copy := *blob
return &copy, nil
}
}
return nil, nil
}
func (s *capturingSkillRepoStub) GetVersionBySkillAndBlob(skillID, blobID int) (*models.SkillVersion, error) {
for _, version := range s.versions {
if version != nil && version.SkillID == skillID && version.BlobID == blobID {
copy := *version
return &copy, nil
}
}
return nil, nil
}
func (s *capturingSkillRepoStub) UpsertInstanceSkill(item *models.InstanceSkill) error {
copy := *item
updated := false
for i, existing := range s.instanceSkills {
if existing.InstanceID == item.InstanceID && existing.SkillID == item.SkillID {
s.instanceSkills[i] = copy
updated = true
break
}
}
if !updated {
s.instanceSkills = append(s.instanceSkills, copy)
}
return nil
}
func (s *capturingSkillRepoStub) MarkMissingInstanceSkills(instanceID int, activeSkillIDs []int, observedAt time.Time) error {
s.markMissingCalls++
s.lastMarkMissingActive = append([]int(nil), activeSkillIDs...)
active := map[int]struct{}{}
for _, id := range activeSkillIDs {
active[id] = struct{}{}
}
for i := range s.instanceSkills {
item := &s.instanceSkills[i]
if item.InstanceID != instanceID {
continue
}
if _, ok := active[item.SkillID]; ok {
continue
}
if strings.EqualFold(item.Status, "removed") {
continue
}
item.Status = "missing"
item.RemovedAt = &observedAt
item.UpdatedAt = observedAt
}
return nil
}
type importLiteMaterializer struct {
repo *skillRepoStub
storage *importTestObjectStorage
blobID int
}
func (m importLiteMaterializer) materializeSkillPackageFromWorkspace(_ context.Context, instanceID int, workspaceDir, contentHash string, _ int) (*models.SkillBlob, error) {
objectKey := fmt.Sprintf("discovered/%d/%s/%s.zip", instanceID, workspaceDir, contentHash)
if m.storage.objects == nil {
m.storage.objects = map[string][]byte{}
}
m.storage.objects[objectKey] = []byte("fake-zip")
scanID := 99
blob := m.repo.blobs[m.blobID]
blob.ObjectKey = objectKey
blob.ScanStatus = "completed"
blob.RiskLevel = skillRiskNone
blob.LastScanResultID = &scanID
m.repo.blobs[m.blobID] = blob
return blob, nil
}
func (importLiteMaterializer) syncSkillRecordFromBlob(int, *models.SkillBlob) error { return nil }
type importTestObjectStorage struct {
objects map[string][]byte
}
func (s *importTestObjectStorage) PutObject(_ context.Context, objectKey string, body []byte, _ string) error {
if s.objects == nil {
s.objects = map[string][]byte{}
}
s.objects[objectKey] = body
return nil
}
func (s *importTestObjectStorage) GetObject(_ context.Context, objectKey string) ([]byte, error) {
if body, ok := s.objects[objectKey]; ok {
return body, nil
}
return nil, fmt.Errorf("object not found: %s", objectKey)
}
func TestDeleteSkillReleasesSkillKey(t *testing.T) {
stub := &skillRepoStub{
skills: map[int]*models.Skill{
1: {ID: 1, UserID: 1, SkillKey: "weather", Name: "Weather", Status: skillStatusActive, SourceType: skillSourceUploaded},
},
tagAssignments: map[int][]int{},
}
svc := &skillService{repo: stub}
if err := svc.DeleteSkill(1, "user", 1); err != nil {
t.Fatalf("DeleteSkill() error = %v", err)
}
want := deletedSkillKey("weather", 1)
if stub.skills[1].SkillKey != want {
t.Fatalf("skill_key = %q, want %q", stub.skills[1].SkillKey, want)
}
active, err := stub.GetSkillByUserKey(1, "weather")
if err != nil {
t.Fatalf("GetSkillByUserKey() error = %v", err)
}
if active != nil {
t.Fatal("original skill_key should be released for re-import")
}
}
func TestSyncAgentSkillsIncrementalDoesNotMarkMissing(t *testing.T) {
contentHash := "abc123def456789012345678901234"
versionID := 1
stub := &capturingSkillRepoStub{
skillRepoStub: skillRepoStub{
skills: map[int]*models.Skill{
10: {
ID: 10, UserID: 1, SkillKey: "weather", Name: "weather",
SourceType: skillSourceDiscovered, Status: skillStatusActive,
Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID,
},
},
blobs: map[int]*models.SkillBlob{
1: {ID: 1, ContentHash: contentHash, ObjectKey: "discovered/weather.zip", ScanStatus: "completed"},
},
versions: map[int]*models.SkillVersion{
1: {ID: 1, SkillID: 10, BlobID: 1, VersionNo: 1},
},
instanceSkills: []models.InstanceSkill{
{InstanceID: 1, SkillID: 10, Status: "active", SourceType: "discovered_in_instance"},
},
},
}
instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{1: {ID: 1, UserID: 1}}}
svc := &skillService{repo: stub, instanceRepo: instRepo, commandService: &noopInstanceCommandService{}}
if err := svc.SyncAgentSkills(1, AgentSkillInventoryReportRequest{Mode: "incremental", Skills: nil}); err != nil {
t.Fatalf("incremental SyncAgentSkills() error = %v", err)
}
if stub.markMissingCalls != 0 {
t.Fatalf("markMissingCalls = %d, want 0 for incremental", stub.markMissingCalls)
}
if stub.instanceSkills[0].Status != "active" {
t.Fatalf("status = %q, want active after incremental empty report", stub.instanceSkills[0].Status)
}
if err := svc.SyncAgentSkills(1, AgentSkillInventoryReportRequest{Mode: "full", Skills: nil}); err != nil {
t.Fatalf("full SyncAgentSkills() error = %v", err)
}
if stub.markMissingCalls != 1 {
t.Fatalf("markMissingCalls = %d, want 1 for full", stub.markMissingCalls)
}
if stub.instanceSkills[0].Status != "missing" {
t.Fatalf("status = %q, want missing after full empty report", stub.instanceSkills[0].Status)
}
}
func TestSyncAgentSkillsReactivatesMissingButNotRemoved(t *testing.T) {
contentHash := "abc123def456789012345678901234"
versionID := 1
removedAt := time.Now().UTC().Add(-time.Hour)
stub := &capturingSkillRepoStub{
skillRepoStub: skillRepoStub{
skills: map[int]*models.Skill{
10: {
ID: 10, UserID: 1, SkillKey: "weather", Name: "weather",
SourceType: skillSourceDiscovered, Status: skillStatusActive,
Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID,
},
11: {
ID: 11, UserID: 1, SkillKey: "calendar", Name: "calendar",
SourceType: skillSourceDiscovered, Status: skillStatusActive,
Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID,
},
},
blobs: map[int]*models.SkillBlob{
1: {ID: 1, ContentHash: contentHash, ObjectKey: "discovered/weather.zip", ScanStatus: "completed"},
2: {ID: 2, ContentHash: "def456abc123789012345678901234", ObjectKey: "discovered/calendar.zip", ScanStatus: "completed"},
},
versions: map[int]*models.SkillVersion{
1: {ID: 1, SkillID: 10, BlobID: 1, VersionNo: 1},
2: {ID: 2, SkillID: 11, BlobID: 2, VersionNo: 1},
},
instanceSkills: []models.InstanceSkill{
{InstanceID: 1, SkillID: 10, Status: "missing", SourceType: "discovered_in_instance", RemovedAt: &removedAt},
{InstanceID: 1, SkillID: 11, Status: "removed", SourceType: "discovered_in_instance", RemovedAt: &removedAt},
},
},
}
instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{1: {ID: 1, UserID: 1}}}
svc := &skillService{repo: stub, instanceRepo: instRepo, commandService: &noopInstanceCommandService{}}
err := svc.SyncAgentSkills(1, AgentSkillInventoryReportRequest{
Mode: "full",
Skills: []AgentSkillRecord{
{Identifier: "weather", ContentMD5: contentHash, Source: "discovered_in_instance"},
{Identifier: "calendar", ContentMD5: "def456abc123789012345678901234", Source: "discovered_in_instance"},
},
})
if err != nil {
t.Fatalf("SyncAgentSkills() error = %v", err)
}
var missing, removed *models.InstanceSkill
for i := range stub.instanceSkills {
item := &stub.instanceSkills[i]
switch item.SkillID {
case 10:
missing = item
case 11:
removed = item
}
}
if missing == nil || missing.Status != "active" || missing.RemovedAt != nil {
t.Fatalf("missing skill revive = %#v, want active with nil RemovedAt", missing)
}
if removed == nil || removed.Status != "removed" {
t.Fatalf("removed skill = %#v, want status removed", removed)
}
}
func TestDownloadSkillNilSafe(t *testing.T) {
missingVersionID := 999
versionID := 7
stub := &skillRepoStub{
skills: map[int]*models.Skill{
1: {ID: 1, UserID: 1, Status: skillStatusActive, Visibility: skillVisibilityPrivate, CurrentVersionID: &missingVersionID},
2: {ID: 2, UserID: 1, Status: skillStatusActive, Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID},
},
versions: map[int]*models.SkillVersion{
7: {ID: 7, SkillID: 2, BlobID: 99},
},
blobs: map[int]*models.SkillBlob{},
}
svc := &skillService{repo: stub, storage: &importTestObjectStorage{objects: map[string][]byte{}}}
if _, _, err := svc.DownloadSkill(1, "user", 1); err == nil {
t.Fatal("expected error when current version is missing")
}
if _, _, err := svc.DownloadSkill(1, "user", 2); err == nil {
t.Fatal("expected error when blob is missing")
}
}
backend/internal/services/skill_import.go
+439
View File
@@ -0,0 +1,439 @@
package services
import (
"context"
"encoding/json"
"fmt"
"io"
"mime/multipart"
"strings"
"time"
"clawreef/internal/models"
)
const (
skillImportConflictNone = "none"
skillImportConflictUnchanged = "unchanged"
skillImportConflictContentChanged = "content_changed"
skillImportActionAuto = "auto"
skillImportActionNewVersion = "new_version"
skillImportActionSaveAsNew = "save_as_new"
skillImportActionSkip = "skip"
skillImportResultCreated = "created"
skillImportResultVersioned = "versioned"
skillImportResultUnchanged = "unchanged"
skillImportResultSavedAsNew = "saved_as_new"
)
type SkillImportPreviewItem struct {
DirectoryName string `json:"directory_name"`
SkillKey string `json:"skill_key"`
ContentHash string `json:"content_hash"`
ConflictType string `json:"conflict_type"`
ExistingSkillID *int `json:"existing_skill_id,omitempty"`
ExistingName *string `json:"existing_name,omitempty"`
CurrentVersionNo *int `json:"current_version_no,omitempty"`
SuggestedSkillKey *string `json:"suggested_skill_key,omitempty"`
}
type SkillImportDecision struct {
DirectoryName string `json:"directory_name"`
Action string `json:"action"`
SkillKey *string `json:"skill_key,omitempty"`
}
type SkillImportResultItem struct {
Skill SkillPayload `json:"skill"`
Action string `json:"action"`
PreviousVersionNo *int `json:"previous_version_no,omitempty"`
DirectoryName string `json:"directory_name"`
}
type ImportDirectoryOptions struct {
Action string
OverrideSkillKey string
}
func (s *skillService) PreviewHubImport(ctx context.Context, userID int, fileHeader *multipart.FileHeader) ([]SkillImportPreviewItem, error) {
_ = ctx
directories, _, err := readSkillArchiveDirectories(fileHeader)
if err != nil {
return nil, err
}
items := make([]SkillImportPreviewItem, 0, len(directories))
for _, dir := range directories {
item, err := s.previewImportDirectory(userID, dir)
if err != nil {
return nil, err
}
items = append(items, item)
}
return items, nil
}
func (s *skillService) ImportHubArchiveWithDecisions(ctx context.Context, userID int, fileHeader *multipart.FileHeader, decisions []SkillImportDecision) ([]SkillImportResultItem, error) {
directories, filename, err := readSkillArchiveDirectories(fileHeader)
if err != nil {
return nil, err
}
decisionMap := mapSkillImportDecisions(decisions)
hasDecisions := len(decisions) > 0
results := make([]SkillImportResultItem, 0, len(directories))
for _, dir := range directories {
preview, err := s.previewImportDirectory(userID, dir)
if err != nil {
return nil, err
}
opts := resolveImportDecision(preview, decisionMap[dir.Name], hasDecisions)
if preview.ConflictType == skillImportConflictUnchanged || opts.Action == skillImportActionSkip {
if preview.ConflictType == skillImportConflictUnchanged && preview.ExistingSkillID != nil {
payload, err := s.loadSkillPayloadByID(*preview.ExistingSkillID)
if err != nil {
return nil, err
}
results = append(results, SkillImportResultItem{
Skill: *payload,
Action: skillImportResultUnchanged,
DirectoryName: dir.Name,
})
}
continue
}
result, err := s.importDirectoryWithOptions(ctx, userID, dir, filename, opts)
if err != nil {
return nil, err
}
result.DirectoryName = dir.Name
results = append(results, *result)
}
return results, nil
}
func readSkillArchiveDirectories(fileHeader *multipart.FileHeader) ([]extractedSkillDirectory, string, error) {
if !strings.HasSuffix(strings.ToLower(strings.TrimSpace(fileHeader.Filename)), ".zip") {
return nil, "", fmt.Errorf("only .zip skill archives are supported")
}
file, err := fileHeader.Open()
if err != nil {
return nil, "", fmt.Errorf("failed to open uploaded archive: %w", err)
}
defer file.Close()
raw, err := io.ReadAll(file)
if err != nil {
return nil, "", fmt.Errorf("failed to read uploaded archive: %w", err)
}
directories, err := extractSkillDirectories(fileHeader.Filename, raw)
if err != nil {
return nil, "", err
}
if len(directories) == 0 {
return nil, "", fmt.Errorf("no skill directories found in archive")
}
return directories, fileHeader.Filename, nil
}
func mapSkillImportDecisions(decisions []SkillImportDecision) map[string]SkillImportDecision {
result := make(map[string]SkillImportDecision, len(decisions))
for _, item := range decisions {
key := strings.TrimSpace(item.DirectoryName)
if key == "" {
continue
}
result[key] = item
}
return result
}
func resolveImportDecision(preview SkillImportPreviewItem, decision SkillImportDecision, hasDecision bool) ImportDirectoryOptions {
if preview.ConflictType == skillImportConflictUnchanged {
return ImportDirectoryOptions{Action: skillImportActionSkip}
}
if !hasDecision {
return ImportDirectoryOptions{Action: skillImportActionAuto}
}
if strings.TrimSpace(decision.DirectoryName) == "" {
switch preview.ConflictType {
case skillImportConflictNone:
return ImportDirectoryOptions{Action: skillImportActionNewVersion}
case skillImportConflictContentChanged:
return ImportDirectoryOptions{Action: skillImportActionNewVersion}
default:
return ImportDirectoryOptions{Action: skillImportActionSkip}
}
}
switch strings.TrimSpace(decision.Action) {
case skillImportActionSaveAsNew:
key := preview.SkillKey
if preview.SuggestedSkillKey != nil && strings.TrimSpace(*preview.SuggestedSkillKey) != "" {
key = *preview.SuggestedSkillKey
}
if decision.SkillKey != nil && strings.TrimSpace(*decision.SkillKey) != "" {
key = strings.TrimSpace(*decision.SkillKey)
}
return ImportDirectoryOptions{Action: skillImportActionSaveAsNew, OverrideSkillKey: key}
case skillImportActionSkip:
return ImportDirectoryOptions{Action: skillImportActionSkip}
default:
return ImportDirectoryOptions{Action: skillImportActionNewVersion}
}
}
func (s *skillService) previewImportDirectory(userID int, dir extractedSkillDirectory) (SkillImportPreviewItem, error) {
skillKey := sanitizeSkillKey(dir.Name)
if skillKey == "" {
return SkillImportPreviewItem{}, fmt.Errorf("skill directory name %q is invalid", dir.Name)
}
contentHash := hashDirectory(dir.Files)
item := SkillImportPreviewItem{
DirectoryName: dir.Name,
SkillKey: skillKey,
ContentHash: contentHash,
ConflictType: skillImportConflictNone,
}
skill, err := s.repo.GetSkillByUserKey(userID, skillKey)
if err != nil {
return SkillImportPreviewItem{}, err
}
if skill == nil {
return item, nil
}
existingName := skill.Name
item.ExistingSkillID = &skill.ID
item.ExistingName = &existingName
existingHash, versionNo, err := s.currentSkillContentHash(skill)
if err != nil {
return SkillImportPreviewItem{}, err
}
if versionNo != nil {
item.CurrentVersionNo = versionNo
}
if existingHash != "" && existingHash == contentHash {
item.ConflictType = skillImportConflictUnchanged
return item, nil
}
item.ConflictType = skillImportConflictContentChanged
suggested := s.nextUploadSkillKey(userID, skillKey)
item.SuggestedSkillKey = &suggested
return item, nil
}
func (s *skillService) currentSkillContentHash(skill *models.Skill) (string, *int, error) {
if skill == nil || skill.CurrentVersionID == nil {
return "", nil, nil
}
version, err := s.repo.GetVersionByID(*skill.CurrentVersionID)
if err != nil {
return "", nil, err
}
if version == nil {
return "", nil, nil
}
versionNo := version.VersionNo
blob, err := s.repo.GetBlobByID(version.BlobID)
if err != nil {
return "", nil, err
}
if blob == nil {
return "", &versionNo, nil
}
return blob.ContentHash, &versionNo, nil
}
func (s *skillService) loadSkillPayloadByID(skillID int) (*SkillPayload, error) {
skill, err := s.repo.GetSkillByID(skillID)
if err != nil {
return nil, err
}
if skill == nil {
return nil, fmt.Errorf("skill not found")
}
payload, err := s.toSkillPayload(*skill)
if err != nil {
return nil, err
}
if err := s.enrichSkillPayload(payload, *skill, nil); err != nil {
return nil, err
}
return payload, nil
}
func (s *skillService) nextUploadSkillKey(userID int, baseKey string) string {
candidate := strings.TrimSpace(baseKey)
if candidate == "" {
candidate = "skill"
}
for i := 2; i <= 99; i++ {
next := fmt.Sprintf("%s-%d", candidate, i)
existing, err := s.repo.GetSkillByUserKey(userID, next)
if err == nil && existing == nil {
return next
}
}
return fmt.Sprintf("%s-%d", candidate, time.Now().UTC().Unix())
}
func (s *skillService) importDirectory(ctx context.Context, userID int, dir extractedSkillDirectory, originalName string) (*SkillPayload, error) {
result, err := s.importDirectoryWithOptions(ctx, userID, dir, originalName, ImportDirectoryOptions{Action: skillImportActionAuto})
if err != nil {
return nil, err
}
return &result.Skill, nil
}
func (s *skillService) importDirectoryWithOptions(ctx context.Context, userID int, dir extractedSkillDirectory, originalName string, opts ImportDirectoryOptions) (*SkillImportResultItem, error) {
baseSkillKey := sanitizeSkillKey(dir.Name)
if baseSkillKey == "" {
return nil, fmt.Errorf("skill directory name %q is invalid", dir.Name)
}
targetSkillKey := baseSkillKey
isSaveAsNew := opts.Action == skillImportActionSaveAsNew
if isSaveAsNew {
targetSkillKey = sanitizeSkillKey(opts.OverrideSkillKey)
if targetSkillKey == "" {
return nil, fmt.Errorf("invalid skill key for save_as_new")
}
}
contentHash := hashDirectory(dir.Files)
archiveBytes, archiveHash, err := buildNormalizedZip(dir)
if err != nil {
return nil, err
}
blob, err := s.repo.GetBlobByContentHash(contentHash)
if err != nil {
return nil, err
}
if blob == nil {
blob = &models.SkillBlob{
ContentHash: contentHash, ArchiveHash: archiveHash,
ObjectKey: fmt.Sprintf("%d/%s/%s.zip", userID, targetSkillKey, contentHash),
FileName: fmt.Sprintf("%s.zip", targetSkillKey),
MediaType: "application/zip", SizeBytes: int64(len(archiveBytes)),
ScanStatus: "pending", RiskLevel: skillRiskUnknown,
}
if err := s.storage.PutObject(ctx, blob.ObjectKey, archiveBytes, blob.MediaType); err != nil {
return nil, err
}
if err := s.repo.CreateBlob(blob); err != nil {
return nil, err
}
if err := s.recordScan(blob, &dir); err != nil {
return nil, err
}
} else {
if err := s.ensureBlobObject(ctx, blob, archiveBytes); err != nil {
return nil, err
}
if blob.LastScanResultID == nil || blob.ScanStatus != "completed" {
if err := s.recordScan(blob, &dir); err != nil {
return nil, err
}
}
}
existingBefore, err := s.repo.GetSkillByUserKey(userID, targetSkillKey)
if err != nil {
return nil, err
}
var previousVersionNo *int
if existingBefore != nil {
_, versionNo, err := s.currentSkillContentHash(existingBefore)
if err != nil {
return nil, err
}
previousVersionNo = versionNo
}
if isSaveAsNew && existingBefore != nil {
return nil, fmt.Errorf("skill key %q already exists", targetSkillKey)
}
skill := existingBefore
created := false
if skill == nil {
description := fmt.Sprintf("Imported from %s", originalName)
skill = &models.Skill{
UserID: userID, SkillKey: targetSkillKey, Name: dir.Name, Description: &description,
SourceType: skillSourceUploaded, Status: "active", Visibility: skillVisibilityPrivate, RiskLevel: blob.RiskLevel,
LastScannedAt: blob.LastScannedAt, LastScanResultID: blob.LastScanResultID,
}
if err := s.repo.CreateSkill(skill); err != nil {
return nil, err
}
created = true
}
version, err := s.repo.GetVersionBySkillAndBlob(skill.ID, blob.ID)
if err != nil {
return nil, err
}
versionCreated := false
if version == nil {
latest, err := s.repo.GetLatestVersionBySkillID(skill.ID)
if err != nil {
return nil, err
}
versionNo := 1
if latest != nil {
versionNo = latest.VersionNo + 1
}
manifest, _ := json.Marshal(map[string]interface{}{"root_dir": dir.Name, "files": len(dir.Files)})
manifestJSON := string(manifest)
version = &models.SkillVersion{
SkillID: skill.ID, BlobID: blob.ID, VersionNo: versionNo, ManifestJSON: &manifestJSON, SourceType: skillSourceUploaded,
}
if err := s.repo.CreateVersion(version); err != nil {
return nil, err
}
versionCreated = true
}
skill.CurrentVersionID = &version.ID
skill.RiskLevel = blob.RiskLevel
skill.LastScannedAt = blob.LastScannedAt
skill.LastScanResultID = blob.LastScanResultID
skill.UpdatedAt = time.Now().UTC()
if err := s.repo.UpdateSkill(skill); err != nil {
return nil, err
}
payload, err := s.toSkillPayload(*skill)
if err != nil {
return nil, err
}
if err := s.enrichSkillPayload(payload, *skill, nil); err != nil {
return nil, err
}
action := skillImportResultVersioned
if created || isSaveAsNew {
action = skillImportResultCreated
if isSaveAsNew {
action = skillImportResultSavedAsNew
}
} else if !versionCreated {
action = skillImportResultUnchanged
}
result := &SkillImportResultItem{
Skill: *payload,
Action: action,
DirectoryName: dir.Name,
}
if action == skillImportResultVersioned && previousVersionNo != nil {
result.PreviousVersionNo = previousVersionNo
}
return result, nil
}
backend/internal/services/skill_import_test.go
+75
View File
@@ -0,0 +1,75 @@
package services
import (
"testing"
"clawreef/internal/models"
)
func TestPreviewImportDirectoryNone(t *testing.T) {
svc := &skillService{repo: &skillRepoStub{skills: map[int]*models.Skill{}}}
item, err := svc.previewImportDirectory(1, extractedSkillDirectory{
Name: "weather",
Files: map[string][]byte{"SKILL.md": []byte("# weather")},
})
if err != nil {
t.Fatalf("previewImportDirectory() error = %v", err)
}
if item.ConflictType != skillImportConflictNone {
t.Fatalf("conflict_type = %q, want %q", item.ConflictType, skillImportConflictNone)
}
}
func TestPreviewImportDirectoryUnchanged(t *testing.T) {
versionID := 10
blobID := 20
dir := extractedSkillDirectory{Name: "weather", Files: map[string][]byte{"SKILL.md": []byte("# weather")}}
contentHash := hashDirectory(dir.Files)
stub := &skillRepoStub{
skills: map[int]*models.Skill{
1: {
ID: 1, UserID: 1, SkillKey: "weather", Name: "Weather", Status: skillStatusActive,
SourceType: skillSourceUploaded, CurrentVersionID: &versionID,
},
},
versions: map[int]*models.SkillVersion{versionID: {ID: versionID, SkillID: 1, BlobID: blobID, VersionNo: 2}},
blobs: map[int]*models.SkillBlob{blobID: {ID: blobID, ContentHash: contentHash, ScanStatus: "completed"}},
}
svc := &skillService{repo: stub}
item, err := svc.previewImportDirectory(1, dir)
if err != nil {
t.Fatalf("previewImportDirectory() error = %v", err)
}
if item.ConflictType != skillImportConflictUnchanged {
t.Fatalf("conflict_type = %q, want %q", item.ConflictType, skillImportConflictUnchanged)
}
}
func TestPreviewImportDirectoryContentChanged(t *testing.T) {
versionID := 10
blobID := 20
stub := &skillRepoStub{
skills: map[int]*models.Skill{
1: {
ID: 1, UserID: 1, SkillKey: "weather", Name: "Weather", Status: skillStatusActive,
SourceType: skillSourceUploaded, CurrentVersionID: &versionID,
},
},
versions: map[int]*models.SkillVersion{versionID: {ID: versionID, SkillID: 1, BlobID: blobID, VersionNo: 2}},
blobs: map[int]*models.SkillBlob{blobID: {ID: blobID, ContentHash: "old-hash", ScanStatus: "completed"}},
}
svc := &skillService{repo: stub}
item, err := svc.previewImportDirectory(1, extractedSkillDirectory{
Name: "weather",
Files: map[string][]byte{"SKILL.md": []byte("# changed")},
})
if err != nil {
t.Fatalf("previewImportDirectory() error = %v", err)
}
if item.ConflictType != skillImportConflictContentChanged {
t.Fatalf("conflict_type = %q, want %q", item.ConflictType, skillImportConflictContentChanged)
}
if item.SuggestedSkillKey == nil || *item.SuggestedSkillKey != "weather-2" {
t.Fatalf("suggested skill key = %v, want weather-2", item.SuggestedSkillKey)
}
}
backend/internal/services/skill_lite_package_collect.go
+242
View File
@@ -0,0 +1,242 @@
package services
import (
"context"
"fmt"
"strings"
"time"
"clawreef/internal/models"
)
func loadLiteSkillDirectoryFromWorkspace(instance *models.Instance, workspaceDir string) (extractedSkillDirectory, string, error) {
workspaceDir = sanitizeWorkspaceRelativePath(strings.TrimSpace(workspaceDir))
if workspaceDir == "" {
return extractedSkillDirectory{}, "", fmt.Errorf("workspace skill directory is required")
}
root := runtimeSkillInstallRoot(instance)
if root == "" {
return extractedSkillDirectory{}, "", fmt.Errorf("runtime skill workspace root is not configured")
}
skillRoot, err := joinRuntimeSkillPath(root, workspaceDir)
if err != nil {
return extractedSkillDirectory{}, "", fmt.Errorf("workspace skill directory is invalid: %s", workspaceDir)
}
files, err := collectLiteSkillDirectoryFiles(skillRoot)
if err != nil {
return extractedSkillDirectory{}, "", err
}
if len(files) == 0 {
return extractedSkillDirectory{}, "", fmt.Errorf("lite skill directory not found: %s", workspaceDir)
}
dir := extractedSkillDirectory{Name: workspaceDir, Files: files}
return dir, hashDirectory(files), nil
}
func resolveLiteWorkspaceDir(instanceSkill *models.InstanceSkill, skill *models.Skill) string {
if instanceSkill != nil && instanceSkill.WorkspaceDir != nil && strings.TrimSpace(*instanceSkill.WorkspaceDir) != "" {
return sanitizeWorkspaceRelativePath(strings.TrimSpace(*instanceSkill.WorkspaceDir))
}
if instanceSkill != nil {
if key := skillKeyForRemoval(instanceSkill); key != "" && !strings.HasPrefix(key, "skill-") {
return sanitizeWorkspaceRelativePath(key)
}
}
if skill != nil && strings.TrimSpace(skill.Name) != "" {
return sanitizeWorkspaceRelativePath(strings.TrimSpace(skill.Name))
}
if skill != nil {
return sanitizeWorkspaceRelativePath(strings.TrimSpace(skill.SkillKey))
}
return ""
}
func (s *skillService) persistDiscoveredSkillPackage(ctx context.Context, instanceID int, dir extractedSkillDirectory, contentMD5 string, existingBlob *models.SkillBlob) (*models.SkillBlob, error) {
if s == nil || s.storage == nil {
return nil, fmt.Errorf("object storage is not configured")
}
contentMD5 = strings.TrimSpace(contentMD5)
if contentMD5 == "" {
return nil, fmt.Errorf("content hash is required")
}
archiveBytes, archiveHash, err := buildNormalizedZip(dir)
if err != nil {
return nil, err
}
blob := existingBlob
if blob == nil {
blob, err = s.repo.GetBlobByContentHash(contentMD5)
if err != nil {
return nil, err
}
}
if blob == nil {
blob = &models.SkillBlob{
ContentHash: contentMD5,
ArchiveHash: archiveHash,
ObjectKey: fmt.Sprintf("discovered/%d/%s/%s.zip", instanceID, sanitizeSkillKey(dir.Name), contentMD5),
FileName: fmt.Sprintf("%s.zip", sanitizeSkillKey(dir.Name)),
MediaType: "application/zip",
SizeBytes: int64(len(archiveBytes)),
ScanStatus: "pending",
RiskLevel: skillRiskUnknown,
}
if err := s.storage.PutObject(ctx, blob.ObjectKey, archiveBytes, blob.MediaType); err != nil {
return nil, err
}
if err := s.repo.CreateBlob(blob); err != nil {
return nil, err
}
} else if strings.TrimSpace(blob.ObjectKey) == "" {
blob.ObjectKey = fmt.Sprintf("discovered/%d/%s/%s.zip", instanceID, sanitizeSkillKey(dir.Name), contentMD5)
blob.FileName = fmt.Sprintf("%s.zip", sanitizeSkillKey(dir.Name))
blob.MediaType = "application/zip"
blob.SizeBytes = int64(len(archiveBytes))
blob.ArchiveHash = archiveHash
if err := s.storage.PutObject(ctx, blob.ObjectKey, archiveBytes, blob.MediaType); err != nil {
return nil, err
}
if err := s.repo.UpdateBlob(blob); err != nil {
return nil, err
}
}
if blob.LastScanResultID == nil || !strings.EqualFold(strings.TrimSpace(blob.ScanStatus), "completed") {
if err := s.recordScan(blob, &dir); err != nil {
blob.ScanStatus = "failed"
blob.UpdatedAt = timeNowUTC()
_ = s.repo.UpdateBlob(blob)
}
}
updated, err := s.repo.GetBlobByID(blob.ID)
if err != nil {
return nil, err
}
if updated != nil {
blob = updated
}
return blob, nil
}
func (s *skillService) materializeSkillPackageFromWorkspace(ctx context.Context, instanceID int, workspaceDir, expectedMD5 string, targetBlobID int) (*models.SkillBlob, error) {
instance, err := s.instanceRepo.GetByID(instanceID)
if err != nil {
return nil, err
}
if instance == nil {
return nil, fmt.Errorf("instance not found")
}
if !isLiteRuntimeInstance(instance) && !SupportsServerWorkspaceSkillScan(instance) {
return nil, fmt.Errorf("instance does not support workspace skill materialization")
}
dir, contentMD5, err := loadLiteSkillDirectoryFromWorkspace(instance, workspaceDir)
if err != nil {
return nil, err
}
var existingBlob *models.SkillBlob
if targetBlobID > 0 {
existingBlob, err = s.repo.GetBlobByID(targetBlobID)
if err != nil {
return nil, err
}
}
if existingBlob == nil {
existingBlob, err = s.repo.GetBlobByContentHash(contentMD5)
if err != nil {
return nil, err
}
}
if existingBlob != nil && !strings.EqualFold(strings.TrimSpace(existingBlob.ContentHash), contentMD5) {
existingBlob.ContentHash = contentMD5
existingBlob.ArchiveHash = contentMD5
}
if existingBlob != nil && strings.TrimSpace(existingBlob.ObjectKey) != "" && strings.EqualFold(strings.TrimSpace(existingBlob.ScanStatus), "completed") {
return existingBlob, nil
}
return s.persistDiscoveredSkillPackage(ctx, instanceID, dir, contentMD5, existingBlob)
}
func (s *skillService) reconcileLiteDiscoveredBlob(skill *models.Skill, contentHash string) (*models.SkillBlob, *models.SkillVersion, error) {
if s == nil || s.repo == nil || skill == nil || skill.CurrentVersionID == nil {
return nil, nil, nil
}
contentHash = strings.TrimSpace(contentHash)
if contentHash == "" {
return nil, nil, nil
}
version, err := s.repo.GetVersionByID(*skill.CurrentVersionID)
if err != nil {
return nil, nil, err
}
if version == nil {
return nil, nil, nil
}
blob, err := s.repo.GetBlobByID(version.BlobID)
if err != nil {
return nil, version, err
}
if blob == nil {
return nil, version, nil
}
if !strings.EqualFold(strings.TrimSpace(blob.ContentHash), contentHash) {
blob.ContentHash = contentHash
blob.ArchiveHash = contentHash
if err := s.repo.UpdateBlob(blob); err != nil {
return nil, version, err
}
}
return blob, version, nil
}
func liteInventoryUsesWorkspaceHash(instance *models.Instance) bool {
return isLiteRuntimeInstance(instance) || SupportsServerWorkspaceSkillScan(instance)
}
func workspaceContentHashForRecord(instance *models.Instance, record AgentSkillRecord) string {
if !runtimeInventoryUsesWorkspaceHash(instance) {
return strings.TrimSpace(record.ContentMD5)
}
workspaceDir := sanitizeWorkspaceRelativePath(strings.TrimSpace(record.Identifier))
if workspaceDir == "" {
return strings.TrimSpace(record.ContentMD5)
}
_, computed, err := loadLiteSkillDirectoryFromWorkspace(instance, workspaceDir)
if err != nil || strings.TrimSpace(computed) == "" {
return strings.TrimSpace(record.ContentMD5)
}
return computed
}
func workspaceContentHashForLiteRecord(instance *models.Instance, record AgentSkillRecord) string {
return workspaceContentHashForRecord(instance, record)
}
func runtimeInventoryUsesWorkspaceHash(instance *models.Instance) bool {
return liteInventoryUsesWorkspaceHash(instance)
}
func (s *skillService) syncSkillRecordFromBlob(skillID int, blob *models.SkillBlob) error {
if blob == nil {
return nil
}
skill, err := s.repo.GetSkillByID(skillID)
if err != nil {
return err
}
if skill == nil {
return nil
}
skill.RiskLevel = blob.RiskLevel
skill.LastScannedAt = blob.LastScannedAt
skill.LastScanResultID = blob.LastScanResultID
skill.UpdatedAt = timeNowUTC()
return s.repo.UpdateSkill(skill)
}
func timeNowUTC() time.Time {
return time.Now().UTC()
}
backend/internal/services/skill_lite_package_collect_test.go
+136
View File
@@ -0,0 +1,136 @@
package services
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"clawreef/internal/models"
)
func TestMaterializeSelfHealsStaleBlobContentHash(t *testing.T) {
root := t.TempDir()
workspace := filepath.Join(root, "hermes", "user-1", "instance-1")
skillRoot := filepath.Join(workspace, "home", ".hermes", "skills", "demo")
if err := os.MkdirAll(skillRoot, 0o750); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(skillRoot, "SKILL.md"), []byte("# demo\n"), 0o640); err != nil {
t.Fatal(err)
}
instance := &models.Instance{
ID: 1,
UserID: 1,
Type: RuntimeTypeHermes,
InstanceMode: InstanceModeLite,
RuntimeType: RuntimeBackendGateway,
WorkspacePath: strPtr(workspace),
}
_, contentHash, err := loadLiteSkillDirectoryFromWorkspace(instance, "demo")
if err != nil {
t.Fatalf("loadLiteSkillDirectoryFromWorkspace() error = %v", err)
}
staleHash := "deadbeefdeadbeefdeadbeefdeadbeef"
blobID := 20
stub := &skillRepoStub{
skills: map[int]*models.Skill{
1: {ID: 1, UserID: 1, SkillKey: "demo", Name: "Demo", Status: skillStatusActive, CurrentVersionID: intPtr(10)},
},
versions: map[int]*models.SkillVersion{10: {ID: 10, SkillID: 1, BlobID: blobID}},
blobs: map[int]*models.SkillBlob{
blobID: {ID: blobID, ContentHash: staleHash, ScanStatus: "pending", RiskLevel: skillRiskUnknown, ObjectKey: ""},
},
}
instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{1: instance}}
storage := &importTestObjectStorage{objects: map[string][]byte{}}
svc := &skillService{repo: stub, instanceRepo: instRepo, storage: storage, scanner: testSkillScanner{}}
blob, err := svc.materializeSkillPackageFromWorkspace(context.Background(), 1, "demo", staleHash, blobID)
if err != nil {
t.Fatalf("materializeSkillPackageFromWorkspace() error = %v", err)
}
if blob == nil || strings.TrimSpace(blob.ObjectKey) == "" {
t.Fatalf("expected materialized blob, got %#v", blob)
}
if !strings.EqualFold(strings.TrimSpace(stub.blobs[blobID].ContentHash), contentHash) {
t.Fatalf("blob content_hash = %q, want %q", stub.blobs[blobID].ContentHash, contentHash)
}
}
func TestWorkspaceContentHashForLiteRecordOverridesAgentMD5(t *testing.T) {
root := t.TempDir()
workspace := filepath.Join(root, "hermes", "user-1", "instance-1")
skillRoot := filepath.Join(workspace, "home", ".hermes", "skills", "yuanbao")
if err := os.MkdirAll(skillRoot, 0o750); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(skillRoot, "SKILL.md"), []byte("# yuanbao\n"), 0o640); err != nil {
t.Fatal(err)
}
instance := &models.Instance{
Type: RuntimeTypeHermes, InstanceMode: InstanceModeLite,
RuntimeType: RuntimeBackendGateway, WorkspacePath: strPtr(workspace),
}
got := workspaceContentHashForLiteRecord(instance, AgentSkillRecord{
Identifier: "yuanbao", ContentMD5: "deadbeefdeadbeefdeadbeefdeadbeef",
})
_, want, err := loadLiteSkillDirectoryFromWorkspace(instance, "yuanbao")
if err != nil {
t.Fatal(err)
}
if got != want {
t.Fatalf("workspaceContentHashForLiteRecord() = %q, want %q", got, want)
}
}
func TestResolveLiteWorkspaceDir(t *testing.T) {
workspace := "yuanbao"
instanceSkill := &models.InstanceSkill{
WorkspaceDir: &workspace,
InstallPath: strPtr("home/.hermes/skills/yuanbao"),
}
skill := &models.Skill{SkillKey: "yuanbao-deadbeef", Name: "yuanbao"}
if got := resolveLiteWorkspaceDir(instanceSkill, skill); got != "yuanbao" {
t.Fatalf("resolveLiteWorkspaceDir() = %q, want yuanbao", got)
}
}
func TestLoadLiteSkillDirectoryFromWorkspace(t *testing.T) {
root := t.TempDir()
workspace := filepath.Join(root, "hermes", "user-1", "instance-1")
skillRoot := filepath.Join(workspace, "home", ".hermes", "skills", "weather")
if err := os.MkdirAll(filepath.Join(skillRoot, "src"), 0o750); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(skillRoot, "SKILL.md"), []byte("# weather\n"), 0o640); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(skillRoot, "src", "main.py"), []byte("print('ok')\n"), 0o640); err != nil {
t.Fatal(err)
}
instance := &models.Instance{
Type: RuntimeTypeHermes,
InstanceMode: InstanceModeLite,
RuntimeType: RuntimeBackendGateway,
WorkspacePath: strPtr(workspace),
}
dir, md5, err := loadLiteSkillDirectoryFromWorkspace(instance, "weather")
if err != nil {
t.Fatalf("loadLiteSkillDirectoryFromWorkspace() error = %v", err)
}
if dir.Name != "weather" || len(dir.Files) != 2 {
t.Fatalf("unexpected dir: %#v", dir)
}
if md5 == "" {
t.Fatal("expected non-empty md5")
}
}
func strPtr(value string) *string {
return &value
}
backend/internal/services/skill_package_collect_test.go
+68
View File
@@ -0,0 +1,68 @@
package services
import (
"testing"
"time"
"clawreef/internal/models"
)
type stubCommandRepo struct {
failed *models.InstanceCommand
}
func (s *stubCommandRepo) Create(*models.InstanceCommand) error { panic("not used") }
func (s *stubCommandRepo) Update(*models.InstanceCommand) error { panic("not used") }
func (s *stubCommandRepo) GetByID(int) (*models.InstanceCommand, error) { panic("not used") }
func (s *stubCommandRepo) GetByInstanceIdempotencyKey(int, string) (*models.InstanceCommand, error) {
panic("not used")
}
func (s *stubCommandRepo) GetNextPendingByInstance(int) (*models.InstanceCommand, error) {
panic("not used")
}
func (s *stubCommandRepo) ListByInstanceID(int, int) ([]models.InstanceCommand, error) {
panic("not used")
}
func (s *stubCommandRepo) FindLatestFailedCollectSkillPackage(string) (*models.InstanceCommand, error) {
return s.failed, nil
}
func TestPublishBlockedReasonCollectFailed(t *testing.T) {
errMsg := `unexpected status 500: {"error":"skill package md5 mismatch: expected abc got def","success":false}`
svc := &skillService{
commandRepo: &stubCommandRepo{
failed: &models.InstanceCommand{
CommandType: "collect_skill_package",
Status: "failed",
ErrorMessage: &errMsg,
FinishedAt: ptrTime(time.Now()),
},
},
}
skill := &models.Skill{ID: 2, Status: skillStatusActive, SourceType: skillSourceDiscovered}
blob := &models.SkillBlob{ScanStatus: "pending", RiskLevel: skillRiskUnknown, ObjectKey: ""}
reason := svc.publishBlockedReasonForSkill(skill, blob, nil, false, false)
if reason == nil || *reason != "skill_package_collect_failed" {
t.Fatalf("expected skill_package_collect_failed, got %v", reason)
}
collectErr := svc.resolvePackageCollectError(skill.ID, blob, nil, false)
if collectErr == nil || *collectErr != errMsg {
t.Fatalf("expected package collect error summary, got %v", collectErr)
}
}
func TestPublishBlockedReasonScanFailed(t *testing.T) {
svc := &skillService{}
skill := &models.Skill{ID: 1, Status: skillStatusActive, SourceType: skillSourceUploaded}
blob := &models.SkillBlob{ScanStatus: "failed", RiskLevel: skillRiskUnknown, ObjectKey: "discovered/1/demo.zip"}
reason := svc.publishBlockedReasonForSkill(skill, blob, nil, false, false)
if reason == nil || *reason != "skill_scan_failed" {
t.Fatalf("expected skill_scan_failed, got %v", reason)
}
}
func ptrTime(value time.Time) *time.Time {
return &value
}
backend/internal/services/skill_package_materialize_service.go
+298
View File
@@ -0,0 +1,298 @@
package services
import (
"context"
"fmt"
"strings"
"clawreef/internal/models"
"clawreef/internal/repository"
)
const (
MaterializeJobStatusPending = "pending"
MaterializeJobStatusRunning = "running"
MaterializeJobStatusSucceeded = "succeeded"
MaterializeJobStatusFailed = "failed"
MaterializeJobStatusCancelled = "cancelled"
MaterializeTriggerSync = "sync"
MaterializeTriggerRetry = "retry"
MaterializeTriggerImport = "import"
MaterializeTriggerPublish = "publish"
MaterializeTriggerBackfill = "backfill"
)
type EnqueueMaterializeRequest struct {
InstanceID int
SkillID int
BlobID int
WorkspaceDir string
ContentHash string
TriggerSource string
IdempotencyKey string
}
type skillBlobReader interface {
GetBlobByID(id int) (*models.SkillBlob, error)
}
type SkillPackageMaterializeService struct {
jobRepo repository.SkillPackageMaterializeJobRepository
blobRepo skillBlobReader
worker skillPackageMaterializer
}
type skillPackageMaterializer interface {
materializeSkillPackageFromWorkspace(ctx context.Context, instanceID int, workspaceDir, contentHash string, targetBlobID int) (*models.SkillBlob, error)
syncSkillRecordFromBlob(skillID int, blob *models.SkillBlob) error
}
func SkillServiceAsMaterializer(service SkillService) skillPackageMaterializer {
if impl, ok := service.(*skillService); ok {
return impl
}
return nil
}
func NewSkillPackageMaterializeService(jobRepo repository.SkillPackageMaterializeJobRepository, blobRepo skillBlobReader, worker skillPackageMaterializer) *SkillPackageMaterializeService {
return &SkillPackageMaterializeService{jobRepo: jobRepo, blobRepo: blobRepo, worker: worker}
}
func (m *SkillPackageMaterializeService) Enqueue(ctx context.Context, req EnqueueMaterializeRequest) (*models.SkillPackageMaterializeJob, error) {
if m == nil || m.jobRepo == nil {
return nil, fmt.Errorf("skill package materialize service is not configured")
}
workspaceDir := sanitizeWorkspaceRelativePath(strings.TrimSpace(req.WorkspaceDir))
contentHash := strings.TrimSpace(req.ContentHash)
if req.InstanceID <= 0 || req.SkillID <= 0 || req.BlobID <= 0 || workspaceDir == "" || contentHash == "" {
return nil, fmt.Errorf("invalid materialize enqueue request")
}
idempotencyKey := strings.TrimSpace(req.IdempotencyKey)
if idempotencyKey == "" {
idempotencyKey = fmt.Sprintf("materialize-%d-%s", req.InstanceID, contentHash)
}
if existing, err := m.jobRepo.GetByIdempotencyKey(idempotencyKey); err != nil {
return nil, err
} else if existing != nil {
if strings.EqualFold(strings.TrimSpace(existing.Status), MaterializeJobStatusSucceeded) {
return existing, nil
}
if strings.EqualFold(strings.TrimSpace(existing.Status), MaterializeJobStatusFailed) {
if err := m.jobRepo.RequeueExisting(existing.ID, req.BlobID, contentHash, workspaceDir); err != nil {
return nil, err
}
return m.jobRepo.GetByID(existing.ID)
}
if strings.EqualFold(strings.TrimSpace(existing.Status), MaterializeJobStatusPending) ||
strings.EqualFold(strings.TrimSpace(existing.Status), MaterializeJobStatusRunning) {
return existing, nil
}
}
if m.blobRepo != nil {
blob, err := m.blobRepo.GetBlobByID(req.BlobID)
if err != nil {
return nil, err
}
if blob != nil && strings.TrimSpace(blob.ObjectKey) != "" {
if existing, err := m.jobRepo.GetByIdempotencyKey(idempotencyKey); err != nil {
return nil, err
} else if existing != nil {
if !strings.EqualFold(strings.TrimSpace(existing.Status), MaterializeJobStatusSucceeded) {
if err := m.jobRepo.MarkSucceeded(existing.ID); err != nil {
return nil, err
}
existing, err = m.jobRepo.GetByID(existing.ID)
if err != nil {
return nil, err
}
}
return existing, nil
}
job := &models.SkillPackageMaterializeJob{
InstanceID: req.InstanceID,
SkillID: req.SkillID,
BlobID: req.BlobID,
WorkspaceDir: workspaceDir,
ContentHash: contentHash,
Status: MaterializeJobStatusSucceeded,
MaxAttempts: 5,
IdempotencyKey: idempotencyKey,
TriggerSource: strings.TrimSpace(req.TriggerSource),
}
if job.TriggerSource == "" {
job.TriggerSource = MaterializeTriggerSync
}
if err := m.jobRepo.Create(job); err != nil {
return nil, err
}
if !strings.EqualFold(strings.TrimSpace(job.Status), MaterializeJobStatusSucceeded) {
if err := m.jobRepo.MarkSucceeded(job.ID); err != nil {
return nil, err
}
job, err = m.jobRepo.GetByID(job.ID)
if err != nil {
return nil, err
}
}
return job, nil
}
}
trigger := strings.TrimSpace(req.TriggerSource)
if trigger == "" {
trigger = MaterializeTriggerSync
}
job := &models.SkillPackageMaterializeJob{
InstanceID: req.InstanceID,
SkillID: req.SkillID,
BlobID: req.BlobID,
WorkspaceDir: workspaceDir,
ContentHash: contentHash,
Status: MaterializeJobStatusPending,
MaxAttempts: 5,
IdempotencyKey: idempotencyKey,
TriggerSource: trigger,
}
if err := m.jobRepo.Create(job); err != nil {
return nil, err
}
return job, nil
}
func (m *SkillPackageMaterializeService) ReleaseToPending(id int) error {
if m == nil || m.jobRepo == nil {
return fmt.Errorf("skill package materialize service is not configured")
}
return m.jobRepo.ReleaseToPending(id)
}
func (m *SkillPackageMaterializeService) ClaimNextPending(ctx context.Context, limit int) ([]models.SkillPackageMaterializeJob, error) {
if m == nil || m.jobRepo == nil {
return nil, fmt.Errorf("skill package materialize service is not configured")
}
return m.jobRepo.ClaimNextPending(ctx, limit)
}
func (m *SkillPackageMaterializeService) ProcessJob(ctx context.Context, jobID int) error {
if m == nil || m.jobRepo == nil || m.worker == nil {
return fmt.Errorf("skill package materialize service is not configured")
}
job, err := m.jobRepo.GetByID(jobID)
if err != nil {
return err
}
if job == nil {
return fmt.Errorf("materialize job not found")
}
if strings.EqualFold(strings.TrimSpace(job.Status), MaterializeJobStatusPending) {
if err := m.jobRepo.MarkRunning(jobID); err != nil {
return err
}
job, err = m.jobRepo.GetByID(jobID)
if err != nil {
return err
}
if job == nil {
return fmt.Errorf("materialize job not found")
}
}
blob, err := m.worker.materializeSkillPackageFromWorkspace(ctx, job.InstanceID, job.WorkspaceDir, job.ContentHash, job.BlobID)
if err != nil {
retryable := !strings.Contains(strings.ToLower(err.Error()), "md5 mismatch")
if markErr := m.jobRepo.MarkFailed(job.ID, err.Error(), retryable); markErr != nil {
return markErr
}
return err
}
if err := m.worker.syncSkillRecordFromBlob(job.SkillID, blob); err != nil {
if markErr := m.jobRepo.MarkFailed(job.ID, err.Error(), true); markErr != nil {
return markErr
}
return err
}
return m.jobRepo.MarkSucceeded(job.ID)
}
func (m *SkillPackageMaterializeService) RetryJob(skillID int) error {
if m == nil || m.jobRepo == nil {
return fmt.Errorf("skill package materialize service is not configured")
}
return m.jobRepo.ResetForRetry(skillID)
}
func (m *SkillPackageMaterializeService) BackfillOnce(ctx context.Context, limit int) (int, error) {
if m == nil || m.jobRepo == nil {
return 0, fmt.Errorf("skill package materialize service is not configured")
}
candidates, err := m.jobRepo.ListBackfillCandidates(limit)
if err != nil {
return 0, err
}
enqueued := 0
for _, candidate := range candidates {
_, err := m.Enqueue(ctx, EnqueueMaterializeRequest{
InstanceID: candidate.InstanceID,
SkillID: candidate.SkillID,
BlobID: candidate.BlobID,
WorkspaceDir: candidate.WorkspaceDir,
ContentHash: candidate.ContentHash,
TriggerSource: MaterializeTriggerBackfill,
IdempotencyKey: fmt.Sprintf("materialize-%d-%s", candidate.InstanceID, candidate.ContentHash),
})
if err != nil {
return enqueued, err
}
enqueued++
}
return enqueued, nil
}
func (m *SkillPackageMaterializeService) FindLatestBySkillID(skillID int) (*models.SkillPackageMaterializeJob, error) {
if m == nil || m.jobRepo == nil {
return nil, nil
}
return m.jobRepo.FindLatestBySkillID(skillID)
}
func (m *SkillPackageMaterializeService) GetObservedStatus(skillID int, blob *models.SkillBlob) (*string, *string) {
if m == nil || m.jobRepo == nil {
return nil, nil
}
if blob != nil && strings.TrimSpace(blob.ObjectKey) != "" {
return nil, nil
}
job, err := m.jobRepo.FindLatestBySkillID(skillID)
if err != nil || job == nil {
return nil, nil
}
status := strings.TrimSpace(job.Status)
if status == "" {
return nil, nil
}
var errSummary *string
if job.LastError != nil && strings.TrimSpace(*job.LastError) != "" {
summary := truncateCollectError(*job.LastError, 512)
if summary != "" {
errSummary = &summary
}
}
return &status, errSummary
}
func materializeBlockedReason(job *models.SkillPackageMaterializeJob) *string {
if job == nil {
return nil
}
reason := func(value string) *string { return &value }
switch strings.TrimSpace(job.Status) {
case MaterializeJobStatusRunning:
return reason("skill_package_materializing")
case MaterializeJobStatusPending:
return reason("skill_package_materializing")
case MaterializeJobStatusFailed:
return reason("skill_package_materialize_failed")
default:
return nil
}
}
backend/internal/services/skill_package_materialize_service_test.go
+445
View File
@@ -0,0 +1,445 @@
package services
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"time"
"clawreef/internal/models"
"clawreef/internal/repository"
)
func TestMaterializeBlockedReason(t *testing.T) {
running := "skill_package_materializing"
if got := materializeBlockedReason(&models.SkillPackageMaterializeJob{Status: MaterializeJobStatusRunning}); got == nil || *got != running {
t.Fatalf("running reason = %v, want %q", got, running)
}
failed := "skill_package_materialize_failed"
if got := materializeBlockedReason(&models.SkillPackageMaterializeJob{Status: MaterializeJobStatusFailed}); got == nil || *got != failed {
t.Fatalf("failed reason = %v, want %q", got, failed)
}
}
func TestPublishBlockedReasonUsesMaterializeJob(t *testing.T) {
svc := &skillService{
materializeService: NewSkillPackageMaterializeService(
&materializeJobRepoStub{latest: &models.SkillPackageMaterializeJob{Status: MaterializeJobStatusPending}},
nil,
nil,
),
}
skill := &models.Skill{ID: 1, Status: skillStatusActive, SourceType: skillSourceDiscovered}
blob := &models.SkillBlob{ScanStatus: "pending", RiskLevel: skillRiskUnknown, ObjectKey: ""}
reason := svc.publishBlockedReasonForSkill(skill, blob, nil, false, false)
if reason == nil || *reason != "skill_package_materializing" {
t.Fatalf("expected skill_package_materializing, got %v", reason)
}
}
func TestPublishBlockedReasonSkipsCollectWhenMaterializeJobSucceeded(t *testing.T) {
svc := &skillService{
materializeService: NewSkillPackageMaterializeService(
&materializeJobRepoStub{latest: &models.SkillPackageMaterializeJob{Status: MaterializeJobStatusSucceeded}},
nil,
nil,
),
commandRepo: &stubCommandRepo{
failed: &models.InstanceCommand{
CommandType: "collect_skill_package",
Status: "failed",
ErrorMessage: strPtr("agent failed"),
},
},
}
skill := &models.Skill{ID: 1, Status: skillStatusActive, SourceType: skillSourceDiscovered}
blob := &models.SkillBlob{ScanStatus: "pending", RiskLevel: skillRiskUnknown, ObjectKey: ""}
reason := svc.publishBlockedReasonForSkill(skill, blob, nil, false, false)
if reason == nil || *reason != "skill_package_pending" {
t.Fatalf("expected skill_package_pending, got %v", reason)
}
}
func TestPublishBlockedReasonLiteSkipsAgentCollectFailed(t *testing.T) {
svc := &skillService{
commandRepo: &stubCommandRepo{
failed: &models.InstanceCommand{
CommandType: "collect_skill_package",
Status: "failed",
ErrorMessage: strPtr("agent failed"),
},
},
}
skill := &models.Skill{ID: 1, Status: skillStatusActive, SourceType: skillSourceDiscovered}
blob := &models.SkillBlob{ScanStatus: "pending", RiskLevel: skillRiskUnknown, ObjectKey: ""}
reason := svc.publishBlockedReasonForSkill(skill, blob, nil, false, true)
if reason == nil || *reason != "skill_package_pending" {
t.Fatalf("expected skill_package_pending, got %v", reason)
}
collectErr := svc.resolvePackageCollectError(skill.ID, blob, nil, true)
if collectErr != nil {
t.Fatalf("expected nil collect error for lite, got %v", collectErr)
}
}
func TestListMyHubSkillsLiteAutoResolvesInstanceContext(t *testing.T) {
versionID := 10
blobID := 20
stub := &skillRepoStub{
skills: map[int]*models.Skill{
1: {
ID: 1, UserID: 1, SkillKey: "yuanbao", Name: "yuanbao", Status: skillStatusActive,
SourceType: skillSourceUploaded, Visibility: skillVisibilityPrivate, CurrentVersionID: &versionID,
},
},
versions: map[int]*models.SkillVersion{versionID: {ID: versionID, SkillID: 1, BlobID: blobID}},
blobs: map[int]*models.SkillBlob{
blobID: {ID: blobID, ScanStatus: "pending", RiskLevel: skillRiskUnknown, ObjectKey: ""},
},
instanceSkills: []models.InstanceSkill{{
InstanceID: 1, SkillID: 1, Status: "active", SourceType: "discovered_in_instance",
}},
tagAssignments: map[int][]int{},
}
instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{
1: {ID: 1, UserID: 1, InstanceMode: InstanceModeLite, RuntimeType: RuntimeBackendGateway},
}}
svc := &skillService{
repo: stub,
instanceRepo: instRepo,
commandRepo: &stubCommandRepo{
failed: &models.InstanceCommand{
CommandType: "collect_skill_package",
Status: "failed",
ErrorMessage: strPtr("agent failed"),
},
},
}
items, err := svc.ListMyHubSkills(1)
if err != nil {
t.Fatalf("ListMyHubSkills() error = %v", err)
}
if len(items) != 1 {
t.Fatalf("items = %d, want 1", len(items))
}
if items[0].PublishBlockedReason == nil || *items[0].PublishBlockedReason != "skill_package_pending" {
t.Fatalf("PublishBlockedReason = %v, want skill_package_pending", items[0].PublishBlockedReason)
}
if items[0].PackageCollectError != nil {
t.Fatalf("PackageCollectError = %v, want nil", items[0].PackageCollectError)
}
}
func TestSyncAgentSkillsLiteSkipsAgentEnqueue(t *testing.T) {
stub := &capturingSkillRepoStub{
skillRepoStub: skillRepoStub{
skills: map[int]*models.Skill{},
blobs: map[int]*models.SkillBlob{},
versions: map[int]*models.SkillVersion{},
},
}
instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{
1: {ID: 1, UserID: 1, InstanceMode: InstanceModeLite, RuntimeType: RuntimeBackendGateway},
}}
cmdSvc := &capturingInstanceCommandService{}
matSvc := NewSkillPackageMaterializeService(&materializeJobRepoStub{}, nil, nil)
svc := &skillService{
repo: stub,
instanceRepo: instRepo,
commandService: cmdSvc,
materializeService: matSvc,
}
err := svc.SyncAgentSkills(1, AgentSkillInventoryReportRequest{
Skills: []AgentSkillRecord{{
Identifier: "yuanbao",
ContentMD5: "abc123def456789012345678901234",
Source: "discovered_in_instance",
InstallPath: "home/.hermes/skills/yuanbao",
}},
})
if err != nil {
t.Fatalf("SyncAgentSkills() error = %v", err)
}
for _, req := range cmdSvc.created {
if req.CommandType == InstanceCommandTypeCollectSkillPackage {
t.Fatalf("unexpected collect_skill_package command: %#v", req)
}
}
}
func TestEnqueueSkipsWhenObjectKeyPresent(t *testing.T) {
objectKey := "discovered/1/demo/abc.zip"
service := NewSkillPackageMaterializeService(
&materializeJobRepoStub{},
&materializeBlobRepoStub{
blobs: map[int]*models.SkillBlob{
20: {ID: 20, ObjectKey: objectKey, ContentHash: "abc"},
},
},
nil,
)
job, err := service.Enqueue(context.Background(), EnqueueMaterializeRequest{
InstanceID: 1,
SkillID: 1,
BlobID: 20,
WorkspaceDir: "demo",
ContentHash: "abc",
TriggerSource: MaterializeTriggerSync,
IdempotencyKey: "materialize-1-abc",
})
if err != nil {
t.Fatalf("Enqueue() error = %v", err)
}
if job == nil || job.Status != MaterializeJobStatusSucceeded {
t.Fatalf("expected succeeded job, got %#v", job)
}
}
func TestEnqueueRequeuesFailedMaterializeJob(t *testing.T) {
failed := "skill package md5 mismatch"
job := &models.SkillPackageMaterializeJob{
ID: 7, InstanceID: 1, SkillID: 1, BlobID: 20, WorkspaceDir: "demo",
ContentHash: "stale", Status: MaterializeJobStatusFailed, LastError: &failed,
IdempotencyKey: "materialize-1-goodhash",
}
repo := &materializeJobRepoStub{
latest: job,
byKey: map[string]*models.SkillPackageMaterializeJob{"materialize-1-goodhash": job},
}
service := NewSkillPackageMaterializeService(repo, nil, nil)
updated, err := service.Enqueue(context.Background(), EnqueueMaterializeRequest{
InstanceID: 1,
SkillID: 1,
BlobID: 20,
WorkspaceDir: "demo",
ContentHash: "goodhash",
IdempotencyKey: "materialize-1-goodhash",
})
if err != nil {
t.Fatalf("Enqueue() error = %v", err)
}
if updated == nil || updated.Status != MaterializeJobStatusPending {
t.Fatalf("expected pending job, got %#v", updated)
}
if updated.ContentHash != "goodhash" {
t.Fatalf("content_hash = %q, want goodhash", updated.ContentHash)
}
}
func TestNewSkillPackageMaterializeWorkerDefaults(t *testing.T) {
worker := NewSkillPackageMaterializeWorker(nil, 0, 0, 0, 0, true)
if worker.perInstanceLimit != 2 {
t.Fatalf("perInstanceLimit = %d, want 2", worker.perInstanceLimit)
}
if worker.concurrency != 5 {
t.Fatalf("concurrency = %d, want 5", worker.concurrency)
}
}
type capturingInstanceCommandService struct {
created []CreateInstanceCommandRequest
}
func (c *capturingInstanceCommandService) Create(_ int, _ *int, req CreateInstanceCommandRequest) (*InstanceCommandPayload, error) {
c.created = append(c.created, req)
return &InstanceCommandPayload{CommandType: req.CommandType, Status: "pending"}, nil
}
func (c *capturingInstanceCommandService) GetNextForAgent(*AgentSession) (*AgentCommandEnvelope, error) {
return nil, nil
}
func (c *capturingInstanceCommandService) MarkStarted(*AgentSession, int, *time.Time) error { return nil }
func (c *capturingInstanceCommandService) MarkFinished(*AgentSession, int, AgentCommandFinishRequest) error {
return nil
}
func (c *capturingInstanceCommandService) ListByInstanceID(int, int) ([]InstanceCommandPayload, error) {
return nil, nil
}
type materializeJobRepoStub struct {
latest *models.SkillPackageMaterializeJob
created []*models.SkillPackageMaterializeJob
byKey map[string]*models.SkillPackageMaterializeJob
}
func (s *materializeJobRepoStub) Create(job *models.SkillPackageMaterializeJob) error {
s.created = append(s.created, job)
if job.ID == 0 {
job.ID = len(s.created)
}
return nil
}
func (s *materializeJobRepoStub) GetByID(id int) (*models.SkillPackageMaterializeJob, error) {
for _, job := range s.created {
if job.ID == id {
return job, nil
}
}
return s.latest, nil
}
func (s *materializeJobRepoStub) GetByIdempotencyKey(key string) (*models.SkillPackageMaterializeJob, error) {
if s.byKey != nil {
if job, ok := s.byKey[key]; ok {
return job, nil
}
}
return nil, nil
}
func (s *materializeJobRepoStub) ClaimNextPending(context.Context, int) ([]models.SkillPackageMaterializeJob, error) {
return nil, nil
}
func (s *materializeJobRepoStub) MarkSucceeded(id int) error {
for _, job := range s.created {
if job.ID == id {
job.Status = MaterializeJobStatusSucceeded
}
}
return nil
}
func (s *materializeJobRepoStub) MarkFailed(id int, msg string, _ bool) error {
for _, job := range s.created {
if job.ID == id {
job.Status = MaterializeJobStatusFailed
job.LastError = &msg
}
}
if s.latest != nil && s.latest.ID == id {
s.latest.Status = MaterializeJobStatusFailed
s.latest.LastError = &msg
}
return nil
}
func (s *materializeJobRepoStub) MarkRunning(id int) error {
for _, job := range s.created {
if job.ID == id {
job.Status = MaterializeJobStatusRunning
}
}
if s.latest != nil && s.latest.ID == id {
s.latest.Status = MaterializeJobStatusRunning
}
return nil
}
func (s *materializeJobRepoStub) ReleaseToPending(int) error { return nil }
func (s *materializeJobRepoStub) ResetForRetry(int) error { return nil }
func (s *materializeJobRepoStub) RequeueExisting(id, blobID int, contentHash, workspaceDir string) error {
for _, job := range s.created {
if job.ID == id {
job.Status = MaterializeJobStatusPending
job.BlobID = blobID
job.ContentHash = contentHash
job.WorkspaceDir = workspaceDir
job.LastError = nil
}
}
if s.latest != nil && s.latest.ID == id {
s.latest.Status = MaterializeJobStatusPending
s.latest.BlobID = blobID
s.latest.ContentHash = contentHash
s.latest.WorkspaceDir = workspaceDir
s.latest.LastError = nil
}
return nil
}
func (s *materializeJobRepoStub) FindLatestBySkillID(int) (*models.SkillPackageMaterializeJob, error) {
return s.latest, nil
}
func (s *materializeJobRepoStub) CountPendingByInstance(int) (int, error) { return 0, nil }
func (s *materializeJobRepoStub) ListBackfillCandidates(int) ([]repository.SkillPackageMaterializeBackfillCandidate, error) {
return nil, nil
}
type materializeBlobRepoStub struct {
blobs map[int]*models.SkillBlob
}
func (s *materializeBlobRepoStub) GetBlobByID(id int) (*models.SkillBlob, error) {
if s.blobs == nil {
return nil, nil
}
return s.blobs[id], nil
}
type testSkillScanner struct{}
func (testSkillScanner) ScanArchive(context.Context, string, []byte, map[string]string) (string, map[string]interface{}, string, error) {
return skillRiskNone, map[string]interface{}{}, "ok", nil
}
func (testSkillScanner) AvailableAnalyzers(context.Context) ([]string, error) { return nil, nil }
func TestProcessJobMaterializesFromWorkspaceWithStorage(t *testing.T) {
root := t.TempDir()
workspace := filepath.Join(root, "hermes", "user-1", "instance-1")
skillRoot := filepath.Join(workspace, "home", ".hermes", "skills", "demo")
if err := os.MkdirAll(filepath.Join(skillRoot, "src"), 0o750); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(skillRoot, "SKILL.md"), []byte("# demo\n"), 0o640); err != nil {
t.Fatal(err)
}
instance := &models.Instance{
ID: 1,
UserID: 1,
Type: RuntimeTypeHermes,
InstanceMode: InstanceModeLite,
RuntimeType: RuntimeBackendGateway,
WorkspacePath: strPtr(workspace),
}
_, contentHash, err := loadLiteSkillDirectoryFromWorkspace(instance, "demo")
if err != nil {
t.Fatalf("loadLiteSkillDirectoryFromWorkspace() error = %v", err)
}
blobID := 20
stub := &skillRepoStub{
skills: map[int]*models.Skill{
1: {
ID: 1, UserID: 1, SkillKey: "demo", Name: "Demo", Status: skillStatusActive,
SourceType: skillSourceDiscovered, CurrentVersionID: intPtr(10),
},
},
versions: map[int]*models.SkillVersion{10: {ID: 10, SkillID: 1, BlobID: blobID}},
blobs: map[int]*models.SkillBlob{
blobID: {ID: blobID, ContentHash: contentHash, ScanStatus: "pending", RiskLevel: skillRiskUnknown, ObjectKey: ""},
},
}
instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{1: instance}}
storage := &importTestObjectStorage{objects: map[string][]byte{}}
svc := &skillService{
repo: stub,
instanceRepo: instRepo,
storage: storage,
scanner: testSkillScanner{},
}
job := &models.SkillPackageMaterializeJob{
ID: 1, InstanceID: 1, SkillID: 1, BlobID: blobID,
WorkspaceDir: "demo", ContentHash: contentHash, Status: MaterializeJobStatusPending,
}
jobRepo := &materializeJobRepoStub{latest: job, created: []*models.SkillPackageMaterializeJob{job}}
matSvc := NewSkillPackageMaterializeService(jobRepo, stub, SkillServiceAsMaterializer(svc))
if err := matSvc.ProcessJob(context.Background(), job.ID); err != nil {
t.Fatalf("ProcessJob() error = %v", err)
}
if job.Status != MaterializeJobStatusSucceeded {
t.Fatalf("job status = %q, want %q", job.Status, MaterializeJobStatusSucceeded)
}
updated, err := stub.GetBlobByID(blobID)
if err != nil {
t.Fatal(err)
}
if updated == nil || strings.TrimSpace(updated.ObjectKey) == "" {
t.Fatalf("expected blob object key, got %#v", updated)
}
if !strings.EqualFold(strings.TrimSpace(updated.ScanStatus), "completed") {
t.Fatalf("blob scan_status = %q, want completed", updated.ScanStatus)
}
if _, ok := storage.objects[updated.ObjectKey]; !ok {
t.Fatalf("storage missing object %q", updated.ObjectKey)
}
}
backend/internal/services/skill_package_materialize_worker.go
+140
View File
@@ -0,0 +1,140 @@
package services
import (
"context"
"log"
"sync"
"time"
)
type SkillPackageMaterializeWorker struct {
service *SkillPackageMaterializeService
tick time.Duration
batchSize int
concurrency int
perInstanceLimit int
enabled bool
mu sync.Mutex
running bool
stopChan chan struct{}
}
func NewSkillPackageMaterializeWorker(service *SkillPackageMaterializeService, tick time.Duration, batchSize, concurrency, perInstanceLimit int, enabled bool) *SkillPackageMaterializeWorker {
if tick <= 0 {
tick = 2 * time.Second
}
if batchSize <= 0 {
batchSize = 5
}
if concurrency <= 0 {
concurrency = 5
}
if perInstanceLimit <= 0 {
perInstanceLimit = 2
}
return &SkillPackageMaterializeWorker{
service: service,
tick: tick,
batchSize: batchSize,
concurrency: concurrency,
perInstanceLimit: perInstanceLimit,
enabled: enabled,
}
}
func (w *SkillPackageMaterializeWorker) Start() {
if w == nil || !w.enabled || w.service == nil {
return
}
w.mu.Lock()
defer w.mu.Unlock()
if w.running {
return
}
w.stopChan = make(chan struct{})
w.running = true
go w.loop(w.stopChan)
}
func (w *SkillPackageMaterializeWorker) Stop() {
if w == nil {
return
}
w.mu.Lock()
defer w.mu.Unlock()
if !w.running {
return
}
close(w.stopChan)
w.running = false
}
func (w *SkillPackageMaterializeWorker) loop(stop <-chan struct{}) {
ctx := context.Background()
if count, err := w.service.BackfillOnce(ctx, 500); err != nil {
log.Printf("skill package materialize backfill failed: %v", err)
} else if count > 0 {
log.Printf("skill package materialize backfill enqueued %d jobs", count)
}
ticker := time.NewTicker(w.tick)
defer ticker.Stop()
for {
select {
case <-stop:
return
case <-ticker.C:
w.processBatch(context.Background())
}
}
}
func (w *SkillPackageMaterializeWorker) processBatch(ctx context.Context) {
jobs, err := w.service.ClaimNextPending(ctx, w.batchSize)
if err != nil {
log.Printf("skill package materialize claim failed: %v", err)
return
}
if len(jobs) == 0 {
return
}
sem := make(chan struct{}, w.concurrency)
instanceActive := make(map[int]int)
var instanceMu sync.Mutex
var wg sync.WaitGroup
for _, job := range jobs {
instanceMu.Lock()
if instanceActive[job.InstanceID] >= w.perInstanceLimit {
instanceMu.Unlock()
if err := w.service.ReleaseToPending(job.ID); err != nil {
log.Printf("skill package materialize release job %d failed: %v", job.ID, err)
}
continue
}
instanceActive[job.InstanceID]++
instanceMu.Unlock()
job := job
wg.Add(1)
sem <- struct{}{}
go func() {
defer wg.Done()
defer func() { <-sem }()
defer func() {
instanceMu.Lock()
instanceActive[job.InstanceID]--
if instanceActive[job.InstanceID] <= 0 {
delete(instanceActive, job.InstanceID)
}
instanceMu.Unlock()
}()
if err := w.service.ProcessJob(ctx, job.ID); err != nil {
log.Printf("skill package materialize job %d failed: %v", job.ID, err)
}
}()
}
wg.Wait()
}
backend/internal/services/skill_package_materialize_worker_test.go
+76
View File
@@ -0,0 +1,76 @@
package services
import (
"context"
"testing"
"time"
"clawreef/internal/models"
"clawreef/internal/repository"
)
func TestSkillPackageMaterializeWorkerReleasesWhenInstanceLimitReached(t *testing.T) {
repo := &workerMaterializeJobRepoStub{
jobs: []models.SkillPackageMaterializeJob{
{ID: 1, InstanceID: 9, Status: MaterializeJobStatusRunning},
{ID: 2, InstanceID: 9, Status: MaterializeJobStatusRunning},
{ID: 3, InstanceID: 9, Status: MaterializeJobStatusRunning},
},
}
service := NewSkillPackageMaterializeService(repo, nil, stubMaterializer{})
worker := NewSkillPackageMaterializeWorker(service, time.Second, 3, 5, 2, true)
worker.processBatch(context.Background())
if repo.released != 1 {
t.Fatalf("released = %d, want 1", repo.released)
}
}
type stubMaterializer struct{}
func (stubMaterializer) materializeSkillPackageFromWorkspace(context.Context, int, string, string, int) (*models.SkillBlob, error) {
return &models.SkillBlob{ObjectKey: "discovered/1/demo/hash.zip", ScanStatus: "completed"}, nil
}
func (stubMaterializer) syncSkillRecordFromBlob(int, *models.SkillBlob) error { return nil }
type workerMaterializeJobRepoStub struct {
jobs []models.SkillPackageMaterializeJob
released int
}
func (s *workerMaterializeJobRepoStub) Create(*models.SkillPackageMaterializeJob) error { return nil }
func (s *workerMaterializeJobRepoStub) GetByID(id int) (*models.SkillPackageMaterializeJob, error) {
for i := range s.jobs {
if s.jobs[i].ID == id {
job := s.jobs[i]
return &job, nil
}
}
return nil, nil
}
func (s *workerMaterializeJobRepoStub) GetByIdempotencyKey(string) (*models.SkillPackageMaterializeJob, error) {
return nil, nil
}
func (s *workerMaterializeJobRepoStub) ClaimNextPending(context.Context, int) ([]models.SkillPackageMaterializeJob, error) {
return s.jobs, nil
}
func (s *workerMaterializeJobRepoStub) MarkSucceeded(int) error { return nil }
func (s *workerMaterializeJobRepoStub) MarkFailed(int, string, bool) error { return nil }
func (s *workerMaterializeJobRepoStub) MarkRunning(int) error { return nil }
func (s *workerMaterializeJobRepoStub) ReleaseToPending(int) error {
s.released++
return nil
}
func (s *workerMaterializeJobRepoStub) ResetForRetry(int) error { return nil }
func (s *workerMaterializeJobRepoStub) RequeueExisting(int, int, string, string) error {
return nil
}
func (s *workerMaterializeJobRepoStub) FindLatestBySkillID(int) (*models.SkillPackageMaterializeJob, error) {
return nil, nil
}
func (s *workerMaterializeJobRepoStub) CountPendingByInstance(int) (int, error) { return 0, nil }
func (s *workerMaterializeJobRepoStub) ListBackfillCandidates(int) ([]repository.SkillPackageMaterializeBackfillCandidate, error) {
return nil, nil
}
backend/internal/services/skill_runtime_paths.go
+171
View File
@@ -0,0 +1,171 @@
package services
import (
"os"
"path/filepath"
"strings"
"clawreef/internal/models"
)
const runtimeSkillDiscoveryMaxDepth = 2
type runtimeSkillDiscovery struct {
RelativePath string
SkillRoot string
}
func runtimeSkillInstallRoot(instance *models.Instance) string {
if instance == nil || instance.WorkspacePath == nil || strings.TrimSpace(*instance.WorkspacePath) == "" {
return ""
}
workspacePath := filepath.Clean(strings.TrimSpace(*instance.WorkspacePath))
if isLiteRuntimeInstance(instance) {
if strings.EqualFold(strings.TrimSpace(instance.Type), RuntimeTypeHermes) {
return filepath.Join(workspacePath, "home", ".hermes", "skills")
}
return filepath.Join(workspacePath, "home", ".openclaw", "workspace", "skills")
}
if strings.EqualFold(strings.TrimSpace(instance.Type), RuntimeTypeHermes) {
return filepath.Join(workspacePath, ".hermes", "skills")
}
return filepath.Join(workspacePath, "home", ".openclaw", "workspace", "skills")
}
func liteSkillInstallRoot(instance *models.Instance) string {
return runtimeSkillInstallRoot(instance)
}
func sanitizeWorkspaceRelativePath(value string) string {
value = strings.TrimSpace(value)
value = strings.ReplaceAll(value, "\\", "/")
value = strings.Trim(value, "/")
if value == "" || strings.Contains(value, "..") {
return ""
}
parts := make([]string, 0, strings.Count(value, "/")+1)
for _, part := range strings.Split(value, "/") {
part = strings.TrimSpace(part)
if part == "" || part == "." || part == ".." || strings.HasPrefix(part, ".") {
return ""
}
parts = append(parts, part)
}
if len(parts) == 0 || len(parts) > runtimeSkillDiscoveryMaxDepth {
return ""
}
return strings.Join(parts, "/")
}
func skillKeyFromRelativePath(relativePath string) string {
relativePath = sanitizeWorkspaceRelativePath(relativePath)
if relativePath == "" {
return ""
}
parts := strings.Split(relativePath, "/")
return sanitizeSkillKey(parts[len(parts)-1])
}
func runtimeSkillInstallRelativePath(instance *models.Instance, relativePath string) string {
relativePath = sanitizeWorkspaceRelativePath(relativePath)
if relativePath == "" {
return ""
}
if instance == nil || instance.WorkspacePath == nil {
return relativePath
}
workspacePath := filepath.Clean(strings.TrimSpace(*instance.WorkspacePath))
target := filepath.Join(runtimeSkillInstallRoot(instance), filepath.FromSlash(relativePath))
rel, err := filepath.Rel(workspacePath, target)
if err != nil {
return filepath.ToSlash(relativePath)
}
return filepath.ToSlash(rel)
}
func joinRuntimeSkillPath(root, relativePath string) (string, error) {
root = filepath.Clean(strings.TrimSpace(root))
relativePath = sanitizeWorkspaceRelativePath(relativePath)
if root == "" || relativePath == "" {
return "", filepath.ErrBadPattern
}
target := filepath.Join(root, filepath.FromSlash(relativePath))
if !isPathWithin(root, target) {
return "", filepath.ErrBadPattern
}
return target, nil
}
func discoverRuntimeSkillDirectories(root string, maxDepth int) ([]runtimeSkillDiscovery, error) {
root = filepath.Clean(strings.TrimSpace(root))
if root == "" {
return nil, nil
}
if maxDepth <= 0 {
maxDepth = runtimeSkillDiscoveryMaxDepth
}
result := make([]runtimeSkillDiscovery, 0)
var walk func(currentRoot, relativePrefix string, depth int) error
walk = func(currentRoot, relativePrefix string, depth int) error {
entries, err := readDirNames(currentRoot)
if err != nil {
return err
}
for _, entry := range entries {
name := strings.TrimSpace(entry.Name)
if name == "" || strings.HasPrefix(name, ".") || name == ".tmp" {
continue
}
if !entry.IsDir {
continue
}
skillRoot := filepath.Join(currentRoot, name)
relativePath := name
if relativePrefix != "" {
relativePath = relativePrefix + "/" + name
}
relativePath = sanitizeWorkspaceRelativePath(relativePath)
if relativePath == "" {
continue
}
files, err := collectLiteSkillDirectoryFiles(skillRoot)
if err != nil {
return err
}
if len(files) > 0 {
result = append(result, runtimeSkillDiscovery{
RelativePath: relativePath,
SkillRoot: skillRoot,
})
continue
}
if depth < maxDepth {
if err := walk(skillRoot, relativePath, depth+1); err != nil {
return err
}
}
}
return nil
}
if err := walk(root, "", 1); err != nil {
return nil, err
}
return result, nil
}
func readDirNames(path string) ([]dirEntryName, error) {
entries, err := os.ReadDir(path)
if err != nil {
return nil, err
}
result := make([]dirEntryName, 0, len(entries))
for _, entry := range entries {
result = append(result, dirEntryName{Name: entry.Name(), IsDir: entry.IsDir()})
}
return result, nil
}
type dirEntryName struct {
Name string
IsDir bool
}
backend/internal/services/skill_runtime_sync.go
+356
View File
@@ -0,0 +1,356 @@
package services
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"time"
"clawreef/internal/models"
"clawreef/internal/repository"
)
type runtimeSkillSyncDeps struct {
bindingRepo repository.InstanceRuntimeBindingRepository
runtimePodRepo repository.RuntimePodRepository
runtimeAgentClient RuntimeAgentClient
}
func (s *skillService) ConfigureRuntimeSkillSync(bindingRepo repository.InstanceRuntimeBindingRepository, runtimePodRepo repository.RuntimePodRepository, agentClient RuntimeAgentClient) {
if s == nil {
return
}
s.runtimeSkillSync = &runtimeSkillSyncDeps{
bindingRepo: bindingRepo,
runtimePodRepo: runtimePodRepo,
runtimeAgentClient: agentClient,
}
}
func ConfigureSkillRuntimeSync(service SkillService, bindingRepo repository.InstanceRuntimeBindingRepository, runtimePodRepo repository.RuntimePodRepository, agentClient RuntimeAgentClient) {
if impl, ok := service.(*skillService); ok {
impl.ConfigureRuntimeSkillSync(bindingRepo, runtimePodRepo, agentClient)
}
}
func (s *skillService) SyncRuntimeAgentSkillsReport(payload map[string]any) error {
if s == nil {
return fmt.Errorf("skill service is not configured")
}
reports, mode, reportedAt, err := parseRuntimeAgentSkillsReport(payload)
if err != nil {
return err
}
for _, report := range reports {
if report.InstanceID <= 0 {
continue
}
skills := make([]AgentSkillRecord, 0, len(report.Skills))
for _, record := range report.Skills {
record.Source = normalizeRuntimeSkillSource(record.Source)
skills = append(skills, record)
}
req := AgentSkillInventoryReportRequest{
AgentID: fmt.Sprintf("runtime-instance-%d", report.InstanceID),
ReportedAt: reportedAt,
Mode: mode,
Trigger: "runtime_agent_report",
Skills: skills,
}
if err := s.SyncAgentSkills(report.InstanceID, req); err != nil {
return err
}
s.completePendingSkillInventorySync(report.InstanceID)
}
return nil
}
func (s *skillService) RequestLiteSkillInventorySync(instanceID int) error {
if s == nil || s.instanceRepo == nil {
return fmt.Errorf("skill service is not configured")
}
instance, err := s.instanceRepo.GetByID(instanceID)
if err != nil {
return err
}
if instance == nil {
return fmt.Errorf("instance not found")
}
if err := EnsureInstanceWorkspacePathForServerScan(context.Background(), s.instanceRepo, instance); err != nil {
return err
}
if !isLiteRuntimeInstance(instance) && !SupportsServerWorkspaceSkillScan(instance) {
return fmt.Errorf("instance does not support workspace skill inventory sync")
}
if SupportsServerWorkspaceSkillScan(instance) {
workspaceMode := "full"
willResyncAgent := isLiteRuntimeInstance(instance) && s.runtimeSkillSync != nil
if willResyncAgent {
workspaceMode = "incremental"
}
if err := s.syncRuntimeSkillsFromWorkspace(instanceID, workspaceMode); err != nil {
return err
}
if !willResyncAgent {
s.completePendingSkillInventorySync(instanceID)
}
}
if !isLiteRuntimeInstance(instance) || s.runtimeSkillSync == nil {
return nil
}
deps := s.runtimeSkillSync
if deps.bindingRepo == nil || deps.runtimePodRepo == nil || deps.runtimeAgentClient == nil {
return nil
}
ctx := context.Background()
binding, err := deps.bindingRepo.GetRunningByInstanceID(ctx, instanceID)
if err != nil {
return fmt.Errorf("failed to resolve runtime binding: %w", err)
}
if binding == nil {
binding, err = deps.bindingRepo.GetByInstanceID(ctx, instanceID)
if err != nil {
return fmt.Errorf("failed to resolve runtime binding: %w", err)
}
}
if binding == nil || binding.Generation != instance.RuntimeGeneration {
return nil
}
runtimePod, err := deps.runtimePodRepo.GetByID(ctx, binding.RuntimePodID)
if err != nil {
return fmt.Errorf("failed to resolve runtime pod: %w", err)
}
if runtimePod != nil && runtimePod.AgentEndpoint != nil && strings.TrimSpace(*runtimePod.AgentEndpoint) != "" {
if err := deps.runtimeAgentClient.ResyncInstanceSkills(ctx, strings.TrimSpace(*runtimePod.AgentEndpoint), instanceID, "full"); err != nil {
return fmt.Errorf("failed to request runtime skill inventory resync: %w", err)
}
}
return nil
}
func (s *skillService) syncRuntimeSkillsFromWorkspace(instanceID int, mode string) error {
if s == nil || s.instanceRepo == nil {
return fmt.Errorf("skill service is not configured")
}
instance, err := s.instanceRepo.GetByID(instanceID)
if err != nil {
return err
}
if instance == nil {
return fmt.Errorf("instance not found")
}
if err := EnsureInstanceWorkspacePathForServerScan(context.Background(), s.instanceRepo, instance); err != nil {
return err
}
if !isLiteRuntimeInstance(instance) && !SupportsServerWorkspaceSkillScan(instance) {
return fmt.Errorf("instance does not support workspace skill inventory sync")
}
root := runtimeSkillInstallRoot(instance)
if root == "" {
return fmt.Errorf("runtime skill workspace root is not configured")
}
records := make([]AgentSkillRecord, 0)
if _, err := os.Stat(root); err != nil {
if os.IsNotExist(err) {
return s.SyncAgentSkills(instanceID, runtimeWorkspaceSkillInventoryRequest(instanceID, mode, records))
}
return fmt.Errorf("failed to inspect runtime skill directory: %w", err)
}
discoveries, err := discoverRuntimeSkillDirectories(root, runtimeSkillDiscoveryMaxDepth)
if err != nil {
return fmt.Errorf("failed to scan runtime skill directory: %w", err)
}
for _, discovery := range discoveries {
files, err := collectLiteSkillDirectoryFiles(discovery.SkillRoot)
if err != nil {
return fmt.Errorf("failed to scan runtime skill %q: %w", discovery.RelativePath, err)
}
if len(files) == 0 {
continue
}
records = append(records, AgentSkillRecord{
Identifier: discovery.RelativePath,
InstallPath: runtimeSkillInstallRelativePath(instance, discovery.RelativePath),
ContentMD5: hashDirectory(files),
Source: "discovered_in_instance",
Type: "agent-skill",
})
}
return s.SyncAgentSkills(instanceID, runtimeWorkspaceSkillInventoryRequest(instanceID, mode, records))
}
func (s *skillService) syncLiteSkillsFromWorkspace(instanceID int) error {
return s.syncRuntimeSkillsFromWorkspace(instanceID, "full")
}
func runtimeWorkspaceSkillInventoryRequest(instanceID int, mode string, records []AgentSkillRecord) AgentSkillInventoryReportRequest {
now := time.Now().UTC()
normalizedMode := strings.TrimSpace(mode)
if normalizedMode == "" {
normalizedMode = "full"
}
return AgentSkillInventoryReportRequest{
AgentID: fmt.Sprintf("workspace-scan-instance-%d", instanceID),
ReportedAt: &now,
Mode: normalizedMode,
Trigger: "runtime_workspace_scan",
Skills: records,
}
}
func liteWorkspaceSkillInventoryRequest(instanceID int, records []AgentSkillRecord) AgentSkillInventoryReportRequest {
return runtimeWorkspaceSkillInventoryRequest(instanceID, "full", records)
}
func liteSkillInstallRelativePath(instance *models.Instance, skillName string) string {
return runtimeSkillInstallRelativePath(instance, skillName)
}
func collectLiteSkillDirectoryFiles(skillRoot string) (map[string][]byte, error) {
manifestPath := filepath.Join(skillRoot, "SKILL.md")
info, err := os.Stat(manifestPath)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
if info.IsDir() {
return nil, nil
}
files := map[string][]byte{}
err = filepath.WalkDir(skillRoot, func(current string, entry os.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if entry.IsDir() {
return nil
}
if entry.Type()&os.ModeSymlink != 0 {
return nil
}
rel, err := filepath.Rel(skillRoot, current)
if err != nil {
return err
}
rel = normalizeSkillRelPath(filepath.ToSlash(rel))
if rel == "" || hasHiddenPathSegment(rel) {
return nil
}
body, err := os.ReadFile(current)
if err != nil {
return err
}
files[rel] = body
return nil
})
if err != nil {
return nil, err
}
return files, nil
}
type runtimeAgentInstanceSkillReport struct {
InstanceID int
Skills []AgentSkillRecord
}
func parseRuntimeAgentSkillsReport(payload map[string]any) ([]runtimeAgentInstanceSkillReport, string, *time.Time, error) {
if payload == nil {
return nil, "", nil, fmt.Errorf("skills report payload is required")
}
raw, err := json.Marshal(payload)
if err != nil {
return nil, "", nil, fmt.Errorf("failed to encode skills report payload: %w", err)
}
var decoded struct {
Mode string `json:"mode"`
ReportedAt *time.Time `json:"reported_at"`
Instances []struct {
InstanceID int `json:"instance_id"`
Skills []AgentSkillRecord `json:"skills"`
} `json:"instances"`
}
if err := json.Unmarshal(raw, &decoded); err != nil {
return nil, "", nil, fmt.Errorf("failed to decode skills report payload: %w", err)
}
mode := strings.TrimSpace(decoded.Mode)
if mode == "" {
mode = "full"
}
reports := make([]runtimeAgentInstanceSkillReport, 0, len(decoded.Instances))
for _, item := range decoded.Instances {
reports = append(reports, runtimeAgentInstanceSkillReport{
InstanceID: item.InstanceID,
Skills: item.Skills,
})
}
return reports, mode, decoded.ReportedAt, nil
}
func normalizeRuntimeSkillSource(value string) string {
switch strings.ToLower(strings.TrimSpace(value)) {
case "", "runtime", "discovered", "agent-skill", "agent_skill":
return "discovered_in_instance"
default:
return normalizeSkillSource(value)
}
}
func (s *skillService) CompletePendingSkillInventorySync(instanceID int) {
s.completePendingSkillInventorySync(instanceID)
}
func (s *skillService) completePendingSkillInventorySync(instanceID int) {
if s == nil || s.commandRepo == nil || instanceID <= 0 {
return
}
commands, err := s.commandRepo.ListByInstanceID(instanceID, 20)
if err != nil {
return
}
now := time.Now().UTC()
for _, command := range commands {
if command.CommandType != InstanceCommandTypeSyncSkillInventory {
continue
}
switch strings.TrimSpace(command.Status) {
case instanceCommandStatusPending, instanceCommandStatusDispatched, instanceCommandStatusRunning:
command.Status = instanceCommandStatusSucceeded
command.FinishedAt = &now
command.UpdatedAt = now
_ = s.commandRepo.Update(&command)
return
}
}
}
func SupportsServerWorkspaceSkillScan(instance *models.Instance) bool {
if instance == nil || instance.WorkspacePath == nil {
return false
}
if strings.TrimSpace(*instance.WorkspacePath) == "" {
return false
}
switch strings.ToLower(strings.TrimSpace(instance.Type)) {
case "hermes", "openclaw":
return true
default:
return false
}
}
func IsLiteRuntimeInstance(instance *models.Instance) bool {
return isLiteRuntimeInstance(instance)
}
backend/internal/services/skill_runtime_sync_test.go
+472
View File
@@ -0,0 +1,472 @@
package services
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"clawreef/internal/models"
)
func TestParseRuntimeAgentSkillsReport(t *testing.T) {
payload := map[string]any{
"mode": "full",
"instances": []map[string]any{
{
"instance_id": 12,
"skills": []map[string]any{
{
"identifier": "weather",
"content_md5": "abc123",
"source": "runtime",
},
},
},
},
}
reports, mode, _, err := parseRuntimeAgentSkillsReport(payload)
if err != nil {
t.Fatalf("parseRuntimeAgentSkillsReport() error = %v", err)
}
if mode != "full" {
t.Fatalf("mode = %q, want full", mode)
}
if len(reports) != 1 || reports[0].InstanceID != 12 {
t.Fatalf("unexpected reports: %#v", reports)
}
if len(reports[0].Skills) != 1 || reports[0].Skills[0].Identifier != "weather" {
t.Fatalf("unexpected skills: %#v", reports[0].Skills)
}
}
func TestNormalizeRuntimeSkillSource(t *testing.T) {
if got := normalizeRuntimeSkillSource("runtime"); got != "discovered_in_instance" {
t.Fatalf("normalizeRuntimeSkillSource(runtime) = %q", got)
}
if got := normalizeRuntimeSkillSource("injected_by_clawmanager"); got != "injected_by_clawmanager" {
t.Fatalf("normalizeRuntimeSkillSource(injected) = %q", got)
}
}
func TestCollectLiteSkillDirectoryFiles(t *testing.T) {
root := t.TempDir()
skillRoot := filepath.Join(root, "weather")
if err := os.MkdirAll(filepath.Join(skillRoot, "src"), 0o750); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(skillRoot, "SKILL.md"), []byte("# weather\n"), 0o640); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(skillRoot, "src", "main.py"), []byte("print('ok')\n"), 0o640); err != nil {
t.Fatal(err)
}
files, err := collectLiteSkillDirectoryFiles(skillRoot)
if err != nil {
t.Fatalf("collectLiteSkillDirectoryFiles() error = %v", err)
}
if len(files) != 2 {
t.Fatalf("files = %#v, want 2 entries", files)
}
if got := hashDirectory(files); got == "" {
t.Fatal("expected non-empty content md5")
}
}
func TestCollectLiteSkillDirectoryFilesSkipsWithoutManifest(t *testing.T) {
root := t.TempDir()
skillRoot := filepath.Join(root, "orphan")
if err := os.MkdirAll(skillRoot, 0o750); err != nil {
t.Fatal(err)
}
files, err := collectLiteSkillDirectoryFiles(skillRoot)
if err != nil {
t.Fatalf("collectLiteSkillDirectoryFiles() error = %v", err)
}
if files != nil {
t.Fatalf("files = %#v, want nil", files)
}
}
func TestDiscoverRuntimeSkillDirectoriesNestedCategory(t *testing.T) {
root := t.TempDir()
categoryRoot := filepath.Join(root, "productivity")
skillRoot := filepath.Join(categoryRoot, "my-skill")
for _, dir := range []string{skillRoot, filepath.Join(skillRoot, "src")} {
if err := os.MkdirAll(dir, 0o750); err != nil {
t.Fatal(err)
}
}
if err := os.WriteFile(filepath.Join(skillRoot, "SKILL.md"), []byte("# my skill\n"), 0o640); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(skillRoot, "src", "main.py"), []byte("print('ok')\n"), 0o640); err != nil {
t.Fatal(err)
}
discoveries, err := discoverRuntimeSkillDirectories(root, runtimeSkillDiscoveryMaxDepth)
if err != nil {
t.Fatalf("discoverRuntimeSkillDirectories() error = %v", err)
}
if len(discoveries) != 1 {
t.Fatalf("discoveries = %#v, want 1 nested skill", discoveries)
}
if discoveries[0].RelativePath != "productivity/my-skill" {
t.Fatalf("RelativePath = %q, want productivity/my-skill", discoveries[0].RelativePath)
}
}
func TestDiscoverRuntimeSkillDirectoriesFlatAndNested(t *testing.T) {
root := t.TempDir()
flatRoot := filepath.Join(root, "weather")
if err := os.MkdirAll(flatRoot, 0o750); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(flatRoot, "SKILL.md"), []byte("# weather\n"), 0o640); err != nil {
t.Fatal(err)
}
discoveries, err := discoverRuntimeSkillDirectories(root, runtimeSkillDiscoveryMaxDepth)
if err != nil {
t.Fatalf("discoverRuntimeSkillDirectories() error = %v", err)
}
if len(discoveries) != 1 || discoveries[0].RelativePath != "weather" {
t.Fatalf("discoveries = %#v, want flat weather skill", discoveries)
}
}
func TestRuntimeSkillInstallRootOpenClawAndHermes(t *testing.T) {
workspace := "/workspaces/demo/instance-1"
hermes := &models.Instance{Type: RuntimeTypeHermes, WorkspacePath: &workspace}
openclaw := &models.Instance{Type: RuntimeTypeOpenClaw, WorkspacePath: &workspace}
hermesRoot := runtimeSkillInstallRoot(hermes)
openclawRoot := runtimeSkillInstallRoot(openclaw)
if hermesRoot != filepath.Join(workspace, ".hermes", "skills") {
t.Fatalf("hermes root = %q", hermesRoot)
}
if openclawRoot != filepath.Join(workspace, "home", ".openclaw", "workspace", "skills") {
t.Fatalf("openclaw root = %q", openclawRoot)
}
}
func TestResolveInstanceSkillSourceTypePreservesInjected(t *testing.T) {
existing := &models.InstanceSkill{SourceType: "injected_by_clawmanager"}
skill := &models.Skill{SourceType: skillSourceUploaded}
got := resolveInstanceSkillSourceType(existing, "discovered_in_instance", skill)
if got != "injected_by_clawmanager" {
t.Fatalf("resolveInstanceSkillSourceType() = %q, want injected_by_clawmanager", got)
}
}
func TestSanitizeWorkspaceRelativePath(t *testing.T) {
if got := sanitizeWorkspaceRelativePath("productivity/my-skill"); got != "productivity/my-skill" {
t.Fatalf("sanitizeWorkspaceRelativePath() = %q", got)
}
if got := sanitizeWorkspaceRelativePath("../escape"); got != "" {
t.Fatalf("sanitizeWorkspaceRelativePath(../escape) = %q, want empty", got)
}
}
func TestIsLiteRuntimeInstanceOpenClawVariants(t *testing.T) {
liteGateway := &models.Instance{InstanceMode: InstanceModeLite, RuntimeType: RuntimeBackendGateway, Type: RuntimeTypeOpenClaw}
if !IsLiteRuntimeInstance(liteGateway) {
t.Fatal("expected gateway lite instance")
}
proDesktop := &models.Instance{InstanceMode: InstanceModePro, RuntimeType: RuntimeBackendDesktop, Type: RuntimeTypeOpenClaw}
if IsLiteRuntimeInstance(proDesktop) {
t.Fatal("expected pro desktop instance to be non-lite")
}
shellPod := &models.Instance{InstanceMode: InstanceModePro, RuntimeType: RuntimeBackendShell, Type: RuntimeTypeOpenClaw}
if IsLiteRuntimeInstance(shellPod) {
t.Fatal("expected shell pod instance to use pro agent path")
}
}
type provenanceCaptureRepoStub struct {
capturingSkillRepoStub
upserted []*models.InstanceSkill
}
func (s *provenanceCaptureRepoStub) GetBlobByContentHash(hash string) (*models.SkillBlob, error) {
for _, blob := range s.blobs {
if strings.EqualFold(strings.TrimSpace(blob.ContentHash), strings.TrimSpace(hash)) {
copy := *blob
return &copy, nil
}
}
return nil, nil
}
func (s *provenanceCaptureRepoStub) GetVersionBySkillAndBlob(skillID, blobID int) (*models.SkillVersion, error) {
for _, version := range s.versions {
if version.SkillID == skillID && version.BlobID == blobID {
copy := *version
return &copy, nil
}
}
return nil, nil
}
func (s *provenanceCaptureRepoStub) UpsertInstanceSkill(item *models.InstanceSkill) error {
copy := *item
s.upserted = append(s.upserted, &copy)
updated := false
for i, existing := range s.instanceSkills {
if existing.InstanceID == item.InstanceID && existing.SkillID == item.SkillID {
s.instanceSkills[i] = copy
updated = true
break
}
}
if !updated {
s.instanceSkills = append(s.instanceSkills, copy)
}
return nil
}
func TestSyncAgentSkillsPreservesInjectedProvenanceAfterWorkspaceScan(t *testing.T) {
contentHash := "abc123def456789012345678901234"
versionID := 1
stub := &provenanceCaptureRepoStub{
capturingSkillRepoStub: capturingSkillRepoStub{
skillRepoStub: skillRepoStub{
skills: map[int]*models.Skill{
10: {
ID: 10, UserID: 1, SkillKey: "ppt-1-0-0", Name: "ppt-1.0.0",
SourceType: skillSourceUploaded, Status: skillStatusActive,
Visibility: skillVisibilityPublic, CurrentVersionID: &versionID,
},
},
blobs: map[int]*models.SkillBlob{
1: {ID: 1, ContentHash: contentHash, ObjectKey: "hub/ppt.zip", ScanStatus: "completed"},
},
versions: map[int]*models.SkillVersion{
1: {ID: 1, SkillID: 10, BlobID: 1, VersionNo: 1},
},
instanceSkills: []models.InstanceSkill{
{InstanceID: 1, SkillID: 10, SourceType: "injected_by_clawmanager", Status: "active"},
},
},
},
}
instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{
1: {ID: 1, UserID: 1, Type: RuntimeTypeHermes, InstanceMode: InstanceModePro, RuntimeType: RuntimeBackendDesktop},
}}
svc := &skillService{repo: stub, instanceRepo: instRepo, commandService: &noopInstanceCommandService{}}
err := svc.SyncAgentSkills(1, AgentSkillInventoryReportRequest{
Mode: "full",
Skills: []AgentSkillRecord{{
Identifier: "ppt-1-0-0",
ContentMD5: contentHash,
Source: "discovered_in_instance",
InstallPath: "home/.hermes/skills/ppt-1-0-0",
}},
})
if err != nil {
t.Fatalf("SyncAgentSkills() error = %v", err)
}
if len(stub.upserted) != 1 {
t.Fatalf("upserted %d instance skills, want 1", len(stub.upserted))
}
if stub.upserted[0].SourceType != "injected_by_clawmanager" {
t.Fatalf("SourceType = %q, want injected_by_clawmanager", stub.upserted[0].SourceType)
}
}
func TestSyncAgentSkillsReusesUploadedSkillOnWorkspaceScan(t *testing.T) {
contentHash := "abc123def456789012345678901234"
versionID := 1
stub := &provenanceCaptureRepoStub{
capturingSkillRepoStub: capturingSkillRepoStub{
skillRepoStub: skillRepoStub{
skills: map[int]*models.Skill{
10: {
ID: 10, UserID: 1, SkillKey: "ppt-1-0-0", Name: "ppt-1.0.0",
SourceType: skillSourceUploaded, Status: skillStatusActive,
Visibility: skillVisibilityPublic, CurrentVersionID: &versionID,
},
},
blobs: map[int]*models.SkillBlob{
1: {ID: 1, ContentHash: contentHash, ObjectKey: "hub/ppt.zip", ScanStatus: "completed"},
},
versions: map[int]*models.SkillVersion{
1: {ID: 1, SkillID: 10, BlobID: 1, VersionNo: 1},
},
},
},
}
instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{
1: {ID: 1, UserID: 1, Type: RuntimeTypeHermes, InstanceMode: InstanceModeLite, RuntimeType: RuntimeBackendGateway},
}}
svc := &skillService{repo: stub, instanceRepo: instRepo, commandService: &noopInstanceCommandService{}}
err := svc.SyncAgentSkills(1, AgentSkillInventoryReportRequest{
Mode: "full",
Skills: []AgentSkillRecord{{
Identifier: "ppt-1-0-0",
ContentMD5: contentHash,
Source: "discovered_in_instance",
InstallPath: "home/.hermes/skills/ppt-1-0-0",
}},
})
if err != nil {
t.Fatalf("SyncAgentSkills() error = %v", err)
}
if len(stub.createdSkills) != 0 {
t.Fatalf("created %d discovered skills, want 0 reuse of uploaded skill", len(stub.createdSkills))
}
if len(stub.upserted) != 1 || stub.upserted[0].SkillID != 10 {
t.Fatalf("upserted = %#v, want instance skill for uploaded skill id 10", stub.upserted)
}
}
type recordingSkillResyncAgentClient struct {
fakeRuntimeAgentClient
calls []struct {
instanceID int
mode string
}
err error
}
func (c *recordingSkillResyncAgentClient) ResyncInstanceSkills(_ context.Context, _ string, instanceID int, mode string) error {
c.calls = append(c.calls, struct {
instanceID int
mode string
}{instanceID: instanceID, mode: mode})
return c.err
}
type inventorySyncCommandRepo struct {
commands []models.InstanceCommand
}
func (r *inventorySyncCommandRepo) Create(*models.InstanceCommand) error { return nil }
func (r *inventorySyncCommandRepo) Update(command *models.InstanceCommand) error {
for i := range r.commands {
if r.commands[i].ID == command.ID {
r.commands[i] = *command
return nil
}
}
r.commands = append(r.commands, *command)
return nil
}
func (r *inventorySyncCommandRepo) GetByID(int) (*models.InstanceCommand, error) { return nil, nil }
func (r *inventorySyncCommandRepo) GetByInstanceIdempotencyKey(int, string) (*models.InstanceCommand, error) {
return nil, nil
}
func (r *inventorySyncCommandRepo) GetNextPendingByInstance(int) (*models.InstanceCommand, error) {
return nil, nil
}
func (r *inventorySyncCommandRepo) ListByInstanceID(int, int) ([]models.InstanceCommand, error) {
return append([]models.InstanceCommand(nil), r.commands...), nil
}
func (r *inventorySyncCommandRepo) FindLatestFailedCollectSkillPackage(string) (*models.InstanceCommand, error) {
return nil, nil
}
func TestRequestLiteSkillInventorySyncUsesIncrementalWhenAgentResyncFollows(t *testing.T) {
workspace := t.TempDir()
skillsRoot := filepath.Join(workspace, "home", ".hermes", "skills")
if err := os.MkdirAll(skillsRoot, 0o750); err != nil {
t.Fatal(err)
}
stub := &capturingSkillRepoStub{
skillRepoStub: skillRepoStub{
skills: map[int]*models.Skill{
10: {ID: 10, UserID: 1, SkillKey: "weather", Name: "weather", SourceType: skillSourceDiscovered, Status: skillStatusActive},
},
blobs: map[int]*models.SkillBlob{},
versions: map[int]*models.SkillVersion{},
instanceSkills: []models.InstanceSkill{
{InstanceID: 1, SkillID: 10, Status: "active", SourceType: "discovered_in_instance"},
},
},
}
instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{
1: {
ID: 1, UserID: 1, Type: RuntimeTypeHermes, InstanceMode: InstanceModeLite,
RuntimeType: RuntimeBackendGateway, WorkspacePath: &workspace, RuntimeGeneration: 1,
},
}}
cmdRepo := &inventorySyncCommandRepo{commands: []models.InstanceCommand{{
ID: 44, InstanceID: 1, CommandType: InstanceCommandTypeSyncSkillInventory, Status: instanceCommandStatusPending,
}}}
agent := &recordingSkillResyncAgentClient{}
endpoint := "http://runtime-agent"
bindingRepo := newFakeRuntimeBindingRepo()
bindingRepo.bindings[1] = &models.InstanceRuntimeBinding{
InstanceID: 1, RuntimePodID: 9, State: "running", Generation: 1,
}
podRepo := &fakeRuntimePodRepo{pods: map[int64]*models.RuntimePod{
9: {ID: 9, AgentEndpoint: &endpoint},
}}
svc := &skillService{
repo: stub,
instanceRepo: instRepo,
commandRepo: cmdRepo,
commandService: &noopInstanceCommandService{},
}
svc.ConfigureRuntimeSkillSync(bindingRepo, podRepo, agent)
if err := svc.RequestLiteSkillInventorySync(1); err != nil {
t.Fatalf("RequestLiteSkillInventorySync() error = %v", err)
}
if stub.markMissingCalls != 0 {
t.Fatalf("markMissingCalls = %d, want 0 when workspace sync is incremental", stub.markMissingCalls)
}
if stub.instanceSkills[0].Status != "active" {
t.Fatalf("status = %q, want active", stub.instanceSkills[0].Status)
}
if len(agent.calls) != 1 || agent.calls[0].mode != "full" {
t.Fatalf("resync calls = %#v, want one full resync", agent.calls)
}
if cmdRepo.commands[0].Status != instanceCommandStatusPending {
t.Fatalf("command status = %q, want pending until agent inventory arrives", cmdRepo.commands[0].Status)
}
}
func TestRequestLiteSkillInventorySyncPropagatesResyncError(t *testing.T) {
workspace := t.TempDir()
skillsRoot := filepath.Join(workspace, "home", ".hermes", "skills")
if err := os.MkdirAll(skillsRoot, 0o750); err != nil {
t.Fatal(err)
}
stub := &capturingSkillRepoStub{
skillRepoStub: skillRepoStub{
skills: map[int]*models.Skill{},
blobs: map[int]*models.SkillBlob{},
versions: map[int]*models.SkillVersion{},
instanceSkills: nil,
},
}
instRepo := &importTestInstanceRepo{instances: map[int]*models.Instance{
1: {
ID: 1, UserID: 1, Type: RuntimeTypeHermes, InstanceMode: InstanceModeLite,
RuntimeType: RuntimeBackendGateway, WorkspacePath: &workspace, RuntimeGeneration: 1,
},
}}
agent := &recordingSkillResyncAgentClient{err: errors.New("resync failed")}
endpoint := "http://runtime-agent"
bindingRepo := newFakeRuntimeBindingRepo()
bindingRepo.bindings[1] = &models.InstanceRuntimeBinding{
InstanceID: 1, RuntimePodID: 9, State: "running", Generation: 1,
}
podRepo := &fakeRuntimePodRepo{pods: map[int64]*models.RuntimePod{
9: {ID: 9, AgentEndpoint: &endpoint},
}}
svc := &skillService{repo: stub, instanceRepo: instRepo, commandService: &noopInstanceCommandService{}}
svc.ConfigureRuntimeSkillSync(bindingRepo, podRepo, agent)
err := svc.RequestLiteSkillInventorySync(1)
if err == nil || !strings.Contains(err.Error(), "resync") {
t.Fatalf("error = %v, want resync failure", err)
}
}
backend/internal/services/skill_service.go
+437 -275
View File
File diff suppressed because it is too large Load Diff
backend/internal/services/skill_service_test.go
+13
View File
@@ -294,6 +294,19 @@ func TestChownRuntimePathReportsRootPermissionDenied(t *testing.T) {
t.Fatalf("chownRuntimePath() error = %v, want owner error", err)
}
}
func TestWriteSkillDirectoryAtomicallyNestedCategoryPath(t *testing.T) {
targetRoot := t.TempDir()
err := writeSkillDirectoryAtomically(targetRoot, "productivity/my-skill", map[string][]byte{
"SKILL.md": []byte("# Nested Skill\n"),
})
if err != nil {
t.Fatalf("writeSkillDirectoryAtomically() error = %v", err)
}
target := filepath.Join(targetRoot, "productivity", "my-skill", "SKILL.md")
if _, err := os.Stat(target); err != nil {
t.Fatalf("expected nested skill directory, stat err = %v", err)
}
}
func TestWriteSkillDirectoryAtomicallyUsesNestedTempRoot(t *testing.T) {
targetRoot := t.TempDir()
err := writeSkillDirectoryAtomically(targetRoot, "marker-pdf-ingest", map[string][]byte{
backend/internal/utils/hub_errors.go
+54
View File
@@ -0,0 +1,54 @@
package utils
import (
"errors"
"net/http"
"github.com/gin-gonic/gin"
)
type HubError struct {
Code string
Message string
Details map[string]string
}
func (e *HubError) Error() string {
if e == nil {
return ""
}
if e.Message != "" {
return e.Message
}
return e.Code
}
func NewHubError(code, message string, details map[string]string) *HubError {
return &HubError{Code: code, Message: message, Details: details}
}
// HandleHubError maps skill hub domain errors to HTTP responses.
func HandleHubError(c *gin.Context, err error) {
var hubErr *HubError
if errors.As(err, &hubErr) {
switch hubErr.Code {
case "skill_package_md5_mismatch":
Error(c, http.StatusBadRequest, hubErr.Code)
default:
Error(c, http.StatusBadRequest, hubErr.Error())
}
return
}
switch err.Error() {
case "skill_not_scanned", "skill_risk_blocked", "skill_tags_required", "skill_not_in_library", "skill is not published to hub":
Error(c, http.StatusBadRequest, err.Error())
case "skill_package_pending":
Error(c, http.StatusConflict, err.Error())
case "skill_package_materialize_failed", "skill_package_materializing":
Error(c, http.StatusConflict, err.Error())
case "skill_attach_forbidden", "access denied":
Error(c, http.StatusForbidden, err.Error())
default:
HandleError(c, err)
}
}
backend/internal/utils/response.go
+6 -2
View File
@@ -72,12 +72,16 @@ func HandleError(c *gin.Context, err error) {
Error(c, http.StatusForbidden, errStr)
case "invalid username or password", "account is disabled", "invalid or expired agent session token":
Error(c, http.StatusUnauthorized, errStr)
case "agent registration is only supported for openclaw instances", "agent registration is only supported for openclaw or hermes instances", "agent id does not match session", "access denied":
case "agent registration is only supported for openclaw instances", "agent registration is only supported for openclaw or hermes instances", "agent id does not match session", "access denied", "skill_attach_forbidden":
Error(c, http.StatusForbidden, errStr)
case "current password is incorrect":
Error(c, http.StatusBadRequest, errStr)
case "user not found", "model not found":
case "user not found", "model not found", "skill not found", "skill hub tag not found":
Error(c, http.StatusNotFound, errStr)
case "skill_not_scanned", "skill_risk_blocked", "skill_tags_required", "skill is not published to hub":
Error(c, http.StatusBadRequest, errStr)
case "skill_package_pending", "skill_package_materialize_failed", "skill_package_materializing":
Error(c, http.StatusConflict, err.Error())
default:
// For development, show actual error; for production, hide details
Error(c, http.StatusInternalServerError, errStr)
backend/internal/utils/session_format.go
+43
View File
@@ -0,0 +1,43 @@
package utils
import "strings"
// FormatOpenClawSessionKey extracts the display session key from a stored session ID.
func FormatOpenClawSessionKey(sessionID string) string {
sessionID = strings.TrimSpace(sessionID)
for _, prefix := range []string{"agent:openclaw:", "agent:hermes:"} {
if strings.HasPrefix(sessionID, prefix) {
return strings.TrimPrefix(sessionID, prefix)
}
}
if strings.HasPrefix(sessionID, "agent:") {
parts := strings.SplitN(sessionID, ":", 3)
if len(parts) == 3 && strings.TrimSpace(parts[2]) != "" {
return parts[2]
}
}
return sessionID
}
// NormalizeOpenClawSessionID maps a runtime session key to the canonical stored session ID.
func NormalizeOpenClawSessionID(sessionKey string, runtimeType string) string {
sessionKey = strings.TrimSpace(sessionKey)
if sessionKey == "" {
return sessionKey
}
if strings.HasPrefix(sessionKey, "agent:") {
return sessionKey
}
switch strings.ToLower(strings.TrimSpace(runtimeType)) {
case "hermes":
return "agent:hermes:" + sessionKey
default:
return "agent:openclaw:" + sessionKey
}
}
// IsTraceFallbackSessionID reports whether a session ID was generated per trace.
func IsTraceFallbackSessionID(sessionID string) bool {
sessionID = strings.TrimSpace(sessionID)
return strings.HasPrefix(sessionID, "sess_")
}
backend/internal/utils/session_format_test.go
+36
View File
@@ -0,0 +1,36 @@
package utils
import "testing"
func TestFormatOpenClawSessionKey(t *testing.T) {
if got := FormatOpenClawSessionKey("agent:openclaw:main"); got != "main" {
t.Fatalf("expected main, got %q", got)
}
if got := FormatOpenClawSessionKey("agent:hermes:work"); got != "work" {
t.Fatalf("expected work, got %q", got)
}
if got := FormatOpenClawSessionKey("sess_trc_123"); got != "sess_trc_123" {
t.Fatalf("expected passthrough, got %q", got)
}
}
func TestNormalizeOpenClawSessionID(t *testing.T) {
if got := NormalizeOpenClawSessionID("main", "openclaw"); got != "agent:openclaw:main" {
t.Fatalf("expected agent:openclaw:main, got %q", got)
}
if got := NormalizeOpenClawSessionID("main", "hermes"); got != "agent:hermes:main" {
t.Fatalf("expected agent:hermes:main, got %q", got)
}
if got := NormalizeOpenClawSessionID("agent:openclaw:main", "openclaw"); got != "agent:openclaw:main" {
t.Fatalf("expected unchanged, got %q", got)
}
}
func TestIsTraceFallbackSessionID(t *testing.T) {
if !IsTraceFallbackSessionID("sess_trc_abc") {
t.Fatal("expected trace fallback session")
}
if IsTraceFallbackSessionID("agent:openclaw:main") {
t.Fatal("expected stable session")
}
}
deployments/k8s/cluster/instance-egress-networkpolicy.yaml
+40
View File
@@ -0,0 +1,40 @@
# Optional cluster-wide reference policy for managed OpenClaw/Hermes runtimes.
# Enable per-instance enforcement by setting CLAWMANAGER_INSTANCE_NETWORK_LOCK=true
# on the ClawManager backend deployment.
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: clawmanager-managed-runtime-egress
namespace: clawmanager-system
labels:
app: clawmanager
policy-role: managed-runtime-egress-reference
spec:
podSelector:
matchLabels:
clawmanager.io/managed-runtime: "true"
policyTypes:
- Egress
egress:
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
ports:
- protocol: UDP
port: 53
- protocol: TCP
port: 53
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: clawmanager-system
ports:
- protocol: TCP
port: 80
- protocol: TCP
port: 443
- protocol: TCP
port: 9001
- protocol: TCP
port: 3128
deployments/k8s/single-node/clawmanager.yaml
+4 -2
View File
@@ -22,9 +22,9 @@ stringData:
runtime-agent-report-token: change-me-runtime-report-token
openclaw-gateway-token: change-me-openclaw-gateway-token
minio-root-user: minioadmin
minio-root-password: minioadmin123
minio-root-password: minioadmin@123
minio-access-key: minioadmin
minio-secret-key: minioadmin123
minio-secret-key: minioadmin@123
---
apiVersion: v1
kind: ConfigMap
@@ -906,6 +906,7 @@ spec:
spec:
containers:
- name: mysql
#image: mysql:8.4.8
image: mysql:8.4.8
imagePullPolicy: IfNotPresent
args:
@@ -1158,6 +1159,7 @@ subjects:
name: clawmanager-app
namespace: clawmanager-system
---
---
# Explicit lease permissions for control-plane leader election. The
# cluster-admin binding above already covers this; this Role documents the
# requirement and keeps leader election working if RBAC is tightened later.
deployments/k8s/single-node/instance-egress-networkpolicy.yaml
+43
View File
@@ -0,0 +1,43 @@
# Optional cluster-wide reference policy for managed OpenClaw/Hermes runtimes.
# Enable per-instance enforcement by setting CLAWMANAGER_INSTANCE_NETWORK_LOCK=true
# on the ClawManager backend deployment.
#
# Per-instance policies are created automatically for managed runtimes when the
# lock is enabled. This manifest documents the intended egress shape.
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: clawmanager-managed-runtime-egress
namespace: clawmanager-system
labels:
app: clawmanager
policy-role: managed-runtime-egress-reference
spec:
podSelector:
matchLabels:
clawmanager.io/managed-runtime: "true"
policyTypes:
- Egress
egress:
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
ports:
- protocol: UDP
port: 53
- protocol: TCP
port: 53
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: clawmanager-system
ports:
- protocol: TCP
port: 80
- protocol: TCP
port: 443
- protocol: TCP
port: 9001
- protocol: TCP
port: 3128
docs/runtime-agent-integration-guide.md
+39 -1
View File
@@ -93,6 +93,44 @@ Agent 启动时如果 `CLAWMANAGER_AGENT_ENABLED` 不是 `true`，应进入空
Runtime 内的应用和 agent 如果需要调用模型，优先使用这些变量，不要让用户在镜像内手工写入 provider key。
### LLM Session 归因
托管 runtime 的每次 LLM 请求应携带稳定 session 标识，供平台按会话统计 token：
| 项 | 要求 |
| --- | --- |
| Header | `x-openclaw-session-key: {sessionKey}` |
| 示例 | `main` 会归一化为 `agent:openclaw:main` |
| 托管实例 Gateway Token 默认 | 未显式传 key 时，ClawManager 默认使用 `main` |
| 备选 | 请求体 `session_id` 或 OpenAI `user` 字段 |
| 禁止 | 长期依赖 Gateway 自动生成的 `sess_{traceID}`（用户 JWT 等非托管调用仍会 fallback） |
Agent state report 可选上报 LLM 配置指纹，便于平台检测配置漂移：
```json
{
"runtime": {
"llm_config_status": "gateway",
"llm_provider_base_url": "http://clawmanager-gateway.../api/v1/gateway/llm",
"llm_config_fingerprint": "sha256..."
}
}
```
### Egress 代理与实例归因
托管 runtime 实例会注入 egress 代理环境变量，并将实例 ID 写入 `CLAWMANAGER_EGRESS_INSTANCE_ID`。当 egress 拦截直连 LLM 提供商域名时，平台会把该事件记为 `egress.llm.blocked` 审计。
| 变量 / Header | 说明 |
| --- | --- |
| `HTTP_PROXY` / `HTTPS_PROXY` | 指向 ClawManager egress proxy |
| `CLAWMANAGER_EGRESS_INSTANCE_ID` | 当前实例 ID，供代理客户端上报 |
| `X-ClawManager-Instance-Id` | egress 请求应携带的实例 ID header（与 `X-ClawManager-Egress-Instance-Id` 等价） |
建议：任何从实例内主动发起的 egress CONNECT/HTTP 代理请求（包括自定义脚本、sidecar、调试工具）在可行时读取 `CLAWMANAGER_EGRESS_INSTANCE_ID` 并设置 `X-ClawManager-Instance-Id`，以便平台将 bypass 尝试关联到具体实例。
可选网络加固：在 ClawManager 后端设置 `CLAWMANAGER_INSTANCE_NETWORK_LOCK=true` 后，新建的 **Pro（独立 Pod）** OpenClaw/Hermes 实例会自动创建 egress NetworkPolicy。**Lite（gateway 池）** 实例共享 runtime Pod，不适用按实例 NetworkPolicy。参考 `deployments/k8s/single-node/instance-egress-networkpolicy.yaml`。
## Agent 生命周期
推荐主循环：
@@ -342,7 +380,7 @@ Authorization: Bearer {session_token}
| `collect_system_info` | 立即采样，发送 state report，并在 finish result 中带上同一份摘要 |
| `health_check` | 检查主进程、桌面入口、agent、metrics collector，并发送 state report |
| `sync_skill_inventory` | 扫描 skill 目录并上报完整 inventory |
| `refresh_skill_inventory` | 重新扫描 skill 目录并上报完整 inventory |
| `refresh_skill_inventory` | **已废弃**。请使用 `sync_skill_inventory` |
| `collect_skill_package` | 打包指定 skill 并上传 |
| `install_skill` | 下载并安装平台指定 skill version |
| `update_skill` | 更新已安装 skill |
docs/session-token-usage.md
+176
View File
@@ -0,0 +1,176 @@
# Session Token Usage
Instance-level reporting for LLM calls routed through the platform AI Gateway.
## Data flow
1. Runtime or user calls `POST /api/v1/gateway/llm/chat/completions`.
2. AI Gateway persists:
- `model_invocations` (tokens, model, status, `instance_id`, `session_id`)
- `cost_records` (estimated cost per trace)
- `audit_events` (including `gateway.session.fallback` when session key is missing)
- `chat_sessions` (optional title from first user message)
3. Instance detail UI calls:
- `GET /api/v1/instances/:id/session-usage`
- `GET /api/v1/instances/:id/session-usage/detail?session_id=...`
4. Admin overview UI calls:
- `GET /api/v1/admin/session-usage/overview`
## Session ID rules
| Source | Stored `session_id` |
|--------|---------------------|
| Header `x-openclaw-session-key: main` on OpenClaw | `agent:openclaw:main` |
| Header on Hermes | `agent:hermes:{key}` |
| Missing stable key (user JWT or non-managed callers) | `sess_trc_{traceId}` (fallback) |
| Missing key on **instance gateway token** for OpenClaw/Hermes | `agent:{type}:main` (managed default) |
Display keys are derived via `FormatOpenClawSessionKey` (e.g. `main`).
## API
### List session usage
`GET /api/v1/instances/:id/session-usage?page=1&limit=20&search=main&since=2026-07-01T00:00:00Z`
Query parameters:
- `since` / `until`: optional RFC3339 timestamps (`until` must be after `since`); filter on invocation `created_at` (cost aggregates join non-blocked invocations on the same window)
- `search`: filters session rows by session id/key/title (summary totals ignore search)
Response highlights:
- `summary`: totals across all sessions on the instance
- `compliance`: fallback session count and recent fallback audit events
- `items`: paginated per-session rows
### Session detail
`GET /api/v1/instances/:id/session-usage/detail?session_id=agent:openclaw:main&since=2026-07-01T00:00:00Z`
Accepts the same optional `since` / `until` bounds as the list endpoint. Detail rows, model breakdown, and recent traces respect the time window and exclude blocked invocations.
Returns model breakdown (tokens + cost) and recent traces for one session.
### Admin cross-instance overview
`GET /api/v1/admin/session-usage/overview?page=1&limit=20&search=openclaw&since=2026-07-01T00:00:00Z`
Returns managed OpenClaw/Hermes running instances sorted by total tokens, with per-instance summary and global totals.
## UI features
- **Time range presets**: all time, 24h, 7d, 30d (instance panel and admin overview)
- **Auto refresh**: optional 15s polling
- **CSV export**: instance panel exports all filtered session rows; admin page exports instance summary rows
## Limits
- **Gateway only**: direct external LLM calls that bypass the platform gateway are not included.
- **Blocked invocations** are excluded from token aggregates.
- Supported instance types in UI: `openclaw`, `hermes`.
## Database indexes
Migration `038_add_session_usage_indexes.sql` adds:
- `cost_records(instance_id)`
- `cost_records(session_id)`
- `model_invocations(instance_id, session_id, created_at)`
## Local verification
1. Apply migrations (including `038`).
2. Open an OpenClaw or Hermes instance detail page (Lite or Pro).
3. Send a gateway chat completion with `x-openclaw-session-key: main`.
4. Refresh the **Session Token Usage** panel and confirm token totals increase.
5. Open **Admin → AI Gateway → Session Usage** for the cross-instance overview.
Optional E2E:
```bash
cd e2e
npx playwright test tests/instances/session-token-tracking.spec.ts
```
Requires a running stack, configured gateway models, and DB access for `fixtures/dbClient.ts`.
## E2E coverage
| Spec | Scope |
|------|-------|
| `session-token-tracking.spec.ts` | Instance session usage API, gateway aggregation, fallback compliance, instance gateway token |
| `session-usage-admin.spec.ts` | Admin overview API, `since` query validation, non-admin 403 |
Run all session usage specs:
```bash
cd e2e
npx playwright test tests/instances/session-token-tracking.spec.ts tests/instances/session-usage-admin.spec.ts
```
## Pre-commit checklist
When preparing the standalone session-usage PR:
1. Branch: `feat/session-token-usage` (from Skill Hub baseline)
2. Include migration `038_add_session_usage_indexes.sql`
3. Exclude unrelated WIP: egress policy, local deployment yaml, debug `_*.json` artifacts
4. Suggested commit split:
- `feat(session-usage): add session usage APIs, indexes, and admin overview`
- `feat(session-usage): add instance/admin UI with filters, refresh, and CSV export`
- `test(session-usage): add handler, service, repository, and e2e coverage`
- `docs(session-usage): add session token usage guide`
5. Verify locally:
- `go test ./internal/services/... ./internal/handlers/... ./internal/repository/... -run SessionUsage`
- Playwright P1 specs above (P0 gateway aggregation may skip when upstream LLM unavailable)
## Phase 9 staging file list
Include (session usage only):
**Backend**
- `backend/cmd/server/main.go` (session-usage routes only — review diff before staging)
- `backend/internal/db/migrations/038_add_session_usage_indexes.sql`
- `backend/internal/db/migrations_test.go` (038 test)
- `backend/internal/handlers/session_usage_query.go`
- `backend/internal/handlers/session_usage_query_test.go`
- `backend/internal/handlers/instance_handler.go`
- `backend/internal/handlers/instance_handler_test.go`
- `backend/internal/handlers/ai_observability_handler.go`
- `backend/internal/handlers/ai_observability_handler_test.go`
- `backend/internal/repository/session_usage_filter.go`
- `backend/internal/repository/session_usage_filter_test.go`
- `backend/internal/repository/model_invocation_repository.go`
- `backend/internal/repository/cost_record_repository.go`
- `backend/internal/services/ai_observability_service.go`
- `backend/internal/services/ai_observability_session_usage_test.go`
**Frontend**
- `frontend/src/components/InstanceSessionUsagePanel.tsx`
- `frontend/src/pages/admin/SessionUsageOverviewPage.tsx`
- `frontend/src/pages/instances/InstanceDetailPage.tsx`
- `frontend/src/components/AdminLayout.tsx`
- `frontend/src/pages/admin/AIGatewayPage.tsx`
- `frontend/src/router/index.tsx`
- `frontend/src/services/instanceService.ts`
- `frontend/src/services/adminService.ts`
- `frontend/src/types/instance.ts`
- `frontend/src/utils/sessionUsageExport.ts`
- `frontend/src/lib/i18n.ts`
**E2E & docs**
- `e2e/fixtures/apiClient.ts`
- `e2e/fixtures/dbClient.ts`
- `e2e/tests/instances/session-token-tracking.spec.ts`
- `e2e/tests/instances/session-usage-admin.spec.ts`
- `docs/session-token-usage.md`
Exclude (do not stage for session-usage PR):
- `backend/internal/egresspolicy/**`
- `backend/internal/handlers/egress_proxy_handler*.go`
- `deployments/k8s/**/instance-egress-networkpolicy.yaml`
- `deployments/scripts/**`
- `e2e/tests/instances/llm-governance.spec.ts` (unless bundled intentionally)
- Root `_*.json`, `pr138.patch`, debug artifacts
docs/skill-content-md5-spec.md
+10
View File
@@ -132,3 +132,13 @@ weather/
Hermes agent 在本地计算 MD5 时应对目录 `/config/.hermes/skills/weather` 调用 `skill_content_md5()`。不要对 zip 文件调用 MD5。
如果 agent 需要在上传前自检，可以先把 zip 解开，确认去掉 `weather/` 后得到的文件列表与本地计算使用的相对路径一致。
## 常见 MD5 不匹配排查
| 现象 | 可能原因 | 处理 |
|------|----------|------|
| `expected abc got def` 且 expected 来自 inventory | Agent inventory 与 collect 使用了不同目录快照 | 确保 collect 前目录未变化，且两次调用同一 `skill_content_md5()` |
| got 值每次不同 | 对 zip 文件 bytes 做 MD5，或 zip 内文件顺序/元数据参与计算 | 只对 skill 目录内容做规范化 MD5，见上文算法 |
| expected 含顶层目录名 | inventory 多剥了一层 skill 根目录 | 基准目录应为 `/config/.openclaw/workspace/skills/{name}/` 内部 |
| 隐藏文件导致偏差 | `.git`、`.cache` 等被计入或遗漏不一致 | 跳过任意以 `.` 开头的路径段 |
| collect 命令 succeeded 但 Hub 仍 unknown | skill-scanner 未部署或扫描失败 | 检查 `skill-scanner` Pod 与 blob `scan_status` |
docs/skill-lite-materialize.md
+57
View File
@@ -0,0 +1,57 @@
# Lite Skill Package Materialization
Lite (gateway / Hermes) instances discover skills from the shared workspace instead of
using the instance agent `collect_skill_package` command.
## Lifecycle
1. **Inventory** — `syncLiteSkillsFromWorkspace` or runtime agent report calls
`SyncAgentSkills`, which upserts skills and writes `instance_skills.workspace_dir`.
2. **Enqueue** — For Lite instances with empty `skill_blobs.object_key`, ClawManager
inserts a row into `skill_package_materialize_jobs` (never `collect_skill_package`).
3. **Materialize** — The leader-only `SkillPackageMaterializeWorker` reads workspace
directories, builds a normalized ZIP, uploads to MinIO, and runs skill-scanner.
4. **Publish** — Once `object_key` is set and scan completes, skills can be imported
to the library and published to Skill Hub.
## Paths
| Runtime | Workspace skill root |
|---------|---------------------|
| Hermes Lite | `{workspace}/home/.hermes/skills/{name}` |
| OpenClaw Lite | `{workspace}/home/.openclaw/workspace/skills/{name}` |
The authoritative directory name is stored in `instance_skills.workspace_dir`.
## Configuration
| Environment variable | Default | Description |
|------------------------|---------|-------------|
| `SKILL_MATERIALIZE_WORKER_ENABLED` | `true` | Enable background worker |
| `SKILL_MATERIALIZE_TICK_MS` | `2000` | Worker poll interval |
| `SKILL_MATERIALIZE_BATCH_SIZE` | `5` | Jobs claimed per tick |
| `SKILL_MATERIALIZE_CONCURRENCY` | `5` | Global worker concurrency |
| `SKILL_MATERIALIZE_PER_INSTANCE_CONCURRENCY` | `2` | Max parallel jobs per instance |
## Agent commands
Pro and Shell instances still use `collect_skill_package` via the instance agent.
Lite instances **do not**; package collection is server-side only.
For Lite inventory, ClawManager treats the shared workspace scan as the authoritative
`content_md5` source. Runtime agent reports may differ; server-side materialize always
recomputes from workspace and self-heals stale blob hashes instead of failing with
`skill package md5 mismatch`.
## Backfill
On worker start, pending Lite blobs with `workspace_dir` set are enqueued automatically.
Migration `039_add_skill_package_materialize.sql` also cancels stale Lite
`collect_skill_package` commands and backfills `workspace_dir` from `install_path`.
## Hub UI blocked reasons
When enriching Skill Hub payloads without an explicit instance (catalog, "My Skills",
detail pages), ClawManager resolves a Lite instance from active `instance_skills` rows
for that skill. This prevents stale Pro-only `collect_skill_package` failures from
showing as `skill_package_collect_failed` on Lite-discovered skills.
e2e/fixtures/apiClient.ts
+136
View File
@@ -396,3 +396,139 @@ export async function disableExternalAccess(
});
await expectOkEnvelope<null>(response);
}
export interface GatewayModelSummary {
id: number | string;
display_name?: string;
}
export async function listGatewayModels(
request: APIRequestContext,
accessToken: string,
): Promise<GatewayModelSummary[]> {
const response = await request.get(`${env.backendUrl}/gateway/llm/models`, {
headers: bearer(accessToken),
});
const body = await expectOkEnvelope<{ items: GatewayModelSummary[] }>(response);
return body.items ?? [];
}
export async function gatewayChatCompletion(
request: APIRequestContext,
accessToken: string,
payload: {
model: string;
instance_id?: number;
messages: Array<{ role: string; content: string }>;
stream?: boolean;
},
extraHeaders: Record<string, string> = {},
) {
return request.post(`${env.backendUrl}/gateway/llm/chat/completions`, {
headers: {
...bearer(accessToken),
...extraHeaders,
},
data: payload,
});
}
export interface LLMGovernanceOverview {
total_managed_instances: number;
non_compliant_count: number;
external_config_count: number;
high_fallback_count: number;
items: Array<{
instance_id: number;
instance_name: string;
is_compliant: boolean;
}>;
}
export async function getLLMGovernanceOverview(
request: APIRequestContext,
accessToken: string,
): Promise<LLMGovernanceOverview> {
const response = await request.get(`${env.backendUrl}/admin/llm-governance/overview`, {
headers: bearer(accessToken),
});
return expectOkEnvelope<LLMGovernanceOverview>(response);
}
export interface InstanceSessionUsageResult {
summary: {
total_prompt_tokens: number;
total_completion_tokens: number;
total_tokens: number;
total_estimated_cost: number;
currency: string;
session_count: number;
};
compliance: {
fallback_session_count: number;
has_fallback_sessions: boolean;
recent_fallback_audit_count: number;
};
items: Array<{
session_id: string;
session_key: string;
total_tokens: number;
invocation_count: number;
}>;
total: number;
page: number;
limit: number;
}
export interface SessionUsageOverviewResult {
summary: {
total_tokens: number;
total_estimated_cost: number;
currency: string;
session_count: number;
};
items: Array<{
instance_id: number;
instance_name: string;
instance_type: string;
user_id: number;
summary: {
total_tokens: number;
session_count: number;
total_estimated_cost: number;
currency: string;
};
compliance: {
fallback_session_count: number;
has_fallback_sessions: boolean;
};
}>;
total: number;
page: number;
limit: number;
}
export async function getInstanceSessionUsage(
request: APIRequestContext,
accessToken: string,
instanceId: number,
params?: { page?: number; limit?: number; search?: string; since?: string; until?: string },
): Promise<InstanceSessionUsageResult> {
const response = await request.get(`${env.backendUrl}/instances/${instanceId}/session-usage`, {
headers: bearer(accessToken),
params,
});
return expectOkEnvelope<InstanceSessionUsageResult>(response);
}
export async function getAdminSessionUsageOverview(
request: APIRequestContext,
accessToken: string,
params?: { page?: number; limit?: number; search?: string; since?: string; until?: string },
): Promise<SessionUsageOverviewResult> {
const response = await request.get(`${env.backendUrl}/admin/session-usage/overview`, {
headers: bearer(accessToken),
params,
});
return expectOkEnvelope<SessionUsageOverviewResult>(response);
}
e2e/fixtures/dbClient.ts
+23
View File
@@ -0,0 +1,23 @@
import mysql from "mysql2/promise";
import { env } from "./env.js";
export async function getInstanceGatewayToken(instanceId: number): Promise<string | null> {
const connection = await mysql.createConnection({
host: env.db.host,
port: env.db.port,
user: env.db.user,
password: env.db.password,
database: env.db.database,
});
try {
const [rows] = await connection.query<{ access_token: string | null }[]>(
"SELECT access_token FROM instances WHERE id = ? LIMIT 1",
[instanceId],
);
const row = rows[0];
const token = row?.access_token?.trim();
return token ? token : null;
} finally {
await connection.end();
}
}
e2e/tests/instances/llm-governance.spec.ts
+104
View File
@@ -0,0 +1,104 @@
import { expect, test } from "../../fixtures/test.js";
import { env } from "../../fixtures/env.js";
import { login, getLLMGovernanceOverview } from "../../fixtures/apiClient.js";
import { users } from "../../fixtures/users.js";
import { execFileSync } from "node:child_process";
interface ApiEnvelope<T> {
success: boolean;
data?: T;
error?: string;
}
function egressProxyOrigin(): string {
return env.backendUrl.replace(/\/api\/v1\/?$/, "");
}
test("@p2 create openclaw instance rejects protected env override", async ({ request }) => {
const tokens = await login(request, users.admin);
const suffix = Date.now();
const response = await request.post(`${env.backendUrl}/instances`, {
headers: { Authorization: `Bearer ${tokens.access_token}` },
data: {
name: `e2e-governance-${suffix}`,
type: "openclaw",
mode: "lite",
cpu_cores: 1,
memory_gb: 2,
disk_gb: 20,
gpu_enabled: false,
gpu_count: 0,
os_type: "openclaw",
os_version: "latest",
environment_overrides: {
OPENAI_BASE_URL: "https://api.openai.com/v1",
},
},
});
expect(response.status()).toBeGreaterThanOrEqual(400);
const body = (await response.json()) as ApiEnvelope<unknown>;
expect(body.success).toBe(false);
expect(body.error ?? "").toMatch(/managed by the platform/i);
});
test("@p2 batch lite create rejects protected env override", async ({ request }) => {
const tokens = await login(request, users.admin);
const suffix = Date.now();
const response = await request.post(`${env.backendUrl}/instances/batch/lite`, {
headers: { Authorization: `Bearer ${tokens.access_token}` },
data: {
name_prefix: `e2e-batch-gov-${suffix}`,
count: 1,
template: {
type: "openclaw",
environment_overrides: {
OPENAI_API_KEY: "sk-test",
},
},
},
});
expect(response.status()).toBeGreaterThanOrEqual(400);
const body = (await response.json()) as ApiEnvelope<unknown>;
expect(body.success).toBe(false);
expect(body.error ?? "").toMatch(/managed by the platform/i);
});
test("@p2 admin llm governance overview returns managed runtime summary", async ({ request }) => {
const tokens = await login(request, users.admin);
const overview = await getLLMGovernanceOverview(request, tokens.access_token);
expect(typeof overview.total_managed_instances).toBe("number");
expect(Array.isArray(overview.items)).toBe(true);
});
test("@p2 @local-only egress proxy blocks direct openai connect", async () => {
let statusCode = "";
try {
statusCode = execFileSync(
"curl",
[
"-x",
egressProxyOrigin(),
"-H",
"X-ClawManager-Egress-Instance-Id: 1",
"-m",
"5",
"-s",
"-o",
process.platform === "win32" ? "NUL" : "/dev/null",
"-w",
"%{http_code}",
"https://api.openai.com",
],
{ encoding: "utf8" },
).trim();
} catch {
test.skip(true, "curl unavailable or egress proxy not reachable");
}
expect(statusCode).toBe("403");
});
e2e/tests/instances/session-token-tracking.spec.ts
+167
View File
@@ -0,0 +1,167 @@
import { expect, test } from "../../fixtures/test.js";
import { env } from "../../fixtures/env.js";
import {
gatewayChatCompletion,
getInstanceSessionUsage,
listGatewayModels,
listInstances,
login,
} from "../../fixtures/apiClient.js";
import { getInstanceGatewayToken } from "../../fixtures/dbClient.js";
import { users } from "../../fixtures/users.js";
function firstOpenClawInstance(instances: Awaited<ReturnType<typeof listInstances>>) {
return instances.instances.find(
(instance) =>
instance.type === "openclaw" &&
instance.status !== "deleting",
);
}
async function fetchSessionUsage(
request: Parameters<typeof login>[0],
accessToken: string,
instanceId: number,
) {
return getInstanceSessionUsage(request, accessToken, instanceId, { page: 1, limit: 50 });
}
function isGatewaySuccessStatus(status: number): boolean {
return status === 200 || status === 201;
}
test("@p1 session usage endpoint returns structured payload for openclaw instance", async ({
request,
}) => {
const accessToken = (await login(request, users.admin)).access_token;
const instances = await listInstances(request, accessToken, { limit: 100 });
const instance = firstOpenClawInstance(instances);
test.skip(!instance, "No openclaw instance available for session usage test");
const data = await fetchSessionUsage(request, accessToken, instance!.id);
expect(data.summary).toBeTruthy();
expect(Array.isArray(data.items)).toBe(true);
expect(typeof data.total).toBe("number");
expect(typeof data.compliance.recent_fallback_audit_count).toBe("number");
});
test("@p1 session usage detail requires session_id", async ({ request }) => {
const accessToken = (await login(request, users.admin)).access_token;
const instances = await listInstances(request, accessToken, { limit: 100 });
const instance = firstOpenClawInstance(instances);
test.skip(!instance, "No openclaw instance available for session usage detail test");
const response = await request.get(
`${env.backendUrl}/instances/${instance!.id}/session-usage/detail`,
{
headers: { Authorization: `Bearer ${accessToken}` },
},
);
expect(response.status()).toBe(400);
});
test("@p0 gateway calls aggregate tokens by openclaw session key", async ({ request }) => {
const accessToken = (await login(request, users.admin)).access_token;
const instances = await listInstances(request, accessToken, { limit: 100 });
const instance = firstOpenClawInstance(instances);
test.skip(!instance, "No openclaw instance available for gateway aggregation test");
const models = await listGatewayModels(request, accessToken);
test.skip(models.length === 0, "No gateway models configured for session aggregation test");
const baseline = await fetchSessionUsage(request, accessToken, instance!.id);
const baselineMain = baseline.items.find((item) => item.session_key === "main");
let successfulCalls = 0;
for (let attempt = 0; attempt < 3; attempt += 1) {
const response = await gatewayChatCompletion(request, accessToken, {
model: "auto",
instance_id: instance!.id,
messages: [{ role: "user", content: `session aggregation probe ${Date.now()}-${attempt}` }],
}, {
"x-openclaw-session-key": "main",
"x-openclaw-run-id": `e2e-session-${Date.now()}-${attempt}`,
});
if (isGatewaySuccessStatus(response.status())) {
successfulCalls += 1;
}
}
test.skip(successfulCalls === 0, "Gateway upstream unavailable; skipping token aggregation assertion");
await expect
.poll(async () => {
const latest = await fetchSessionUsage(request, accessToken, instance!.id);
const main = latest.items.find((item) => item.session_key === "main");
return main?.invocation_count ?? 0;
}, { timeout: 20_000 })
.toBeGreaterThan(baselineMain?.invocation_count ?? 0);
await expect
.poll(async () => {
const latest = await fetchSessionUsage(request, accessToken, instance!.id);
const main = latest.items.find((item) => item.session_key === "main");
return main?.total_tokens ?? 0;
}, { timeout: 20_000 })
.toBeGreaterThan(baselineMain?.total_tokens ?? 0);
});
test("@p0 gateway missing session key surfaces fallback compliance", async ({ request }) => {
const accessToken = (await login(request, users.admin)).access_token;
const instances = await listInstances(request, accessToken, { limit: 100 });
const instance = firstOpenClawInstance(instances);
test.skip(!instance, "No openclaw instance available for fallback compliance test");
const models = await listGatewayModels(request, accessToken);
test.skip(models.length === 0, "No gateway models configured for fallback compliance test");
const baseline = await fetchSessionUsage(request, accessToken, instance!.id);
const response = await gatewayChatCompletion(request, accessToken, {
model: "auto",
instance_id: instance!.id,
messages: [{ role: "user", content: `fallback probe ${Date.now()}` }],
}, {
"x-openclaw-run-id": `e2e-fallback-${Date.now()}`,
});
test.skip(!isGatewaySuccessStatus(response.status()), "Gateway upstream unavailable; skipping fallback assertion");
await expect
.poll(async () => {
const latest = await fetchSessionUsage(request, accessToken, instance!.id);
return latest.compliance.has_fallback_sessions;
}, { timeout: 20_000 })
.toBe(true);
await expect
.poll(async () => {
const latest = await fetchSessionUsage(request, accessToken, instance!.id);
return latest.compliance.recent_fallback_audit_count;
}, { timeout: 20_000 })
.toBeGreaterThan(baseline.compliance.recent_fallback_audit_count);
});
test("@p0 instance gateway token can call chat completions", async ({ request }) => {
const accessToken = (await login(request, users.admin)).access_token;
const instances = await listInstances(request, accessToken, { limit: 100 });
const instance = firstOpenClawInstance(instances);
test.skip(!instance, "No openclaw instance available for instance gateway token test");
let gatewayToken: string | null = null;
try {
gatewayToken = await getInstanceGatewayToken(instance!.id);
} catch {
test.skip(true, "E2E database unavailable for instance gateway token lookup");
}
test.skip(!gatewayToken, "Instance gateway token not provisioned");
const models = await listGatewayModels(request, gatewayToken);
test.skip(models.length === 0, "No gateway models configured for instance token test");
const response = await gatewayChatCompletion(request, gatewayToken, {
model: "auto",
messages: [{ role: "user", content: `instance token probe ${Date.now()}` }],
}, {
"x-openclaw-session-key": "main",
"x-openclaw-run-id": `e2e-instance-token-${Date.now()}`,
});
expect([200, 201]).toContain(response.status());
});
e2e/tests/instances/session-usage-admin.spec.ts
+95
View File
@@ -0,0 +1,95 @@
import { expect, test } from "../../fixtures/test.js";
import { env } from "../../fixtures/env.js";
import {
getAdminSessionUsageOverview,
getInstanceSessionUsage,
listInstances,
login,
registerUser,
} from "../../fixtures/apiClient.js";
import { users } from "../../fixtures/users.js";
function firstManagedRuntimeInstance(instances: Awaited<ReturnType<typeof listInstances>>) {
return instances.instances.find(
(instance) =>
(instance.type === "openclaw" || instance.type === "hermes") &&
instance.status !== "deleting",
);
}
test("@p1 admin session usage overview returns structured payload", async ({ request }) => {
const accessToken = (await login(request, users.admin)).access_token;
const overview = await getAdminSessionUsageOverview(request, accessToken, {
page: 1,
limit: 20,
});
expect(overview.summary).toBeTruthy();
expect(Array.isArray(overview.items)).toBe(true);
expect(typeof overview.total).toBe("number");
expect(typeof overview.page).toBe("number");
expect(typeof overview.limit).toBe("number");
});
test("@p1 admin session usage overview rejects invalid since timestamp", async ({ request }) => {
const accessToken = (await login(request, users.admin)).access_token;
const response = await request.get(`${env.backendUrl}/admin/session-usage/overview`, {
headers: { Authorization: `Bearer ${accessToken}` },
params: { since: "not-a-date" },
});
expect(response.status()).toBe(400);
});
test("@p1 instance session usage accepts since query parameter", async ({ request }) => {
const accessToken = (await login(request, users.admin)).access_token;
const instances = await listInstances(request, accessToken, { limit: 100 });
const instance = firstManagedRuntimeInstance(instances);
test.skip(!instance, "No openclaw/hermes instance available for since-filter test");
const since = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString();
const data = await getInstanceSessionUsage(request, accessToken, instance!.id, {
page: 1,
limit: 20,
since,
});
expect(data.summary).toBeTruthy();
expect(Array.isArray(data.items)).toBe(true);
});
test("@p1 instance session usage rejects invalid since timestamp", async ({ request }) => {
const accessToken = (await login(request, users.admin)).access_token;
const instances = await listInstances(request, accessToken, { limit: 100 });
const instance = firstManagedRuntimeInstance(instances);
test.skip(!instance, "No openclaw/hermes instance available for invalid since test");
const response = await request.get(
`${env.backendUrl}/instances/${instance!.id}/session-usage`,
{
headers: { Authorization: `Bearer ${accessToken}` },
params: { since: "bad-timestamp" },
},
);
expect(response.status()).toBe(400);
});
test("@p2 non-admin cannot access session usage overview", async ({ request }) => {
await registerUser(request, users.user);
const accessToken = (await login(request, users.user)).access_token;
const response = await request.get(`${env.backendUrl}/admin/session-usage/overview`, {
headers: { Authorization: `Bearer ${accessToken}` },
});
expect(response.status()).toBe(403);
});
test("@p1 session usage rejects until before since", async ({ request }) => {
const accessToken = (await login(request, users.admin)).access_token;
const response = await request.get(`${env.backendUrl}/admin/session-usage/overview`, {
headers: { Authorization: `Bearer ${accessToken}` },
params: {
since: "2026-07-10T00:00:00Z",
until: "2026-07-01T00:00:00Z",
},
});
expect(response.status()).toBe(400);
});
frontend/src/components/AdminLayout.tsx
+1 -1
View File
@@ -74,7 +74,7 @@ const AdminLayout: React.FC<AdminLayoutProps> = ({ children, title = '' }) => {
path: '/admin/ai-gateway',
label: t('nav.aiGateway'),
icon: Bot,
matchPaths: ['/admin/models', '/admin/ai-audit', '/admin/costs', '/admin/risk-rules'],
matchPaths: ['/admin/models', '/admin/ai-audit', '/admin/costs', '/admin/risk-rules', '/admin/session-usage'],
},
{ path: '/admin/settings', label: t('nav.settings'), icon: Settings },
];
frontend/src/components/InstanceCollapsiblePanel.tsx
+106
View File
@@ -0,0 +1,106 @@
import React, { useEffect, useState } from "react";
import { ChevronDown, ChevronUp } from "lucide-react";
import { useI18n } from "../contexts/I18nContext";
type Props = {
storageKey: string;
title: string;
icon: React.ReactNode;
defaultCollapsed?: boolean;
summary?: React.ReactNode;
headerActions?: React.ReactNode;
onExpandedChange?: (expanded: boolean) => void;
contentClassName?: string;
children: React.ReactNode;
};
function readStoredCollapsed(storageKey: string, defaultCollapsed: boolean): boolean {
try {
const stored = localStorage.getItem(storageKey);
if (stored === "true") {
return true;
}
if (stored === "false") {
return false;
}
} catch {
// ignore storage failures
}
return defaultCollapsed;
}
export default function InstanceCollapsiblePanel({
storageKey,
title,
icon,
defaultCollapsed = true,
summary,
headerActions,
onExpandedChange,
contentClassName,
children,
}: Props) {
const { t } = useI18n();
const [collapsed, setCollapsed] = useState(() => readStoredCollapsed(storageKey, defaultCollapsed));
const toggle = () => {
setCollapsed((current) => {
const next = !current;
onExpandedChange?.(!next);
return next;
});
};
useEffect(() => {
onExpandedChange?.(!collapsed);
}, [collapsed, onExpandedChange]);
useEffect(() => {
try {
localStorage.setItem(storageKey, String(collapsed));
} catch {
// ignore storage failures
}
}, [collapsed, storageKey]);
return (
<section className={`cm-surface shrink-0 px-4 ${collapsed ? "py-2" : "py-3"}`}>
<div className="flex items-start justify-between gap-3">
<button
type="button"
onClick={toggle}
className="flex min-w-0 flex-1 items-start gap-2 text-left"
aria-expanded={!collapsed}
>
<span className="mt-0.5 shrink-0">{icon}</span>
<span className="min-w-0">
<span className="flex flex-wrap items-center gap-2">
<h2 className="text-sm font-semibold text-slate-950">{title}</h2>
{collapsed ? (
<ChevronDown className="h-4 w-4 text-slate-400" aria-hidden />
) : (
<ChevronUp className="h-4 w-4 text-slate-400" aria-hidden />
)}
</span>
{collapsed && summary ? (
<span className="mt-1 block text-xs text-slate-500">{summary}</span>
) : null}
</span>
</button>
<div className="flex shrink-0 items-center gap-2">
{!collapsed ? headerActions : null}
<button
type="button"
onClick={toggle}
className="rounded-md border border-slate-200 px-2 py-1 text-xs text-slate-600 hover:bg-slate-50"
>
{collapsed ? t("instances.panelExpand") : t("instances.panelCollapse")}
</button>
</div>
</div>
{!collapsed ? (
<div className={contentClassName ?? "mt-3"}>{children}</div>
) : null}
</section>
);
}

Some files were not shown because too many files have changed in this diff Show More