package services

import (
	"context"
	"errors"
	"fmt"
	"log"
	"strconv"
	"strings"
	"sync"
	"time"

	"clawreef/internal/models"
	"clawreef/internal/repository"
	"clawreef/internal/services/k8s"
)

type RuntimeScheduler struct {
	instanceRepo repository.InstanceRepository
	podRepo      repository.RuntimePodRepository
	bindingRepo  repository.InstanceRuntimeBindingRepository
	rolloutRepo  repository.RuntimeRolloutRepository
	agentClient  RuntimeAgentClient
	events       RuntimeEventService
	leader       RuntimeLeaderService
	deployments  k8s.RuntimeDeploymentService
	envBuilder   RuntimeGatewayEnvBuilder
	tick         time.Duration

	workspaceRoot             string
	runtimeNamespace          string
	gatewayPortStart          int
	gatewayPortEnd            int
	heartbeatTimeout          time.Duration
	maxGatewaysPerPod         int
	gatewayStartInFlightLimit int

	gatewayCreateLocksMu sync.Mutex
	gatewayCreateLocks   map[int64]*sync.Mutex
}

var (
	errRuntimeScaleOutPending     = errors.New("runtime scale-out pending")
	errRuntimeGatewayStartPending = errors.New("runtime gateway start pending")
)

const (
	runtimeRolloutStaleWindowMultiplier     = 3
	defaultRuntimeGatewayStartInFlightLimit = 32
	runtimeSchedulerBatchLimit              = 1000
)

type RuntimeGatewayEnvBuilder func(*models.Instance) (map[string]string, error)

type RuntimeSchedulerOption func(*RuntimeScheduler)

func WithRuntimeSchedulerWorkspaceRoot(root string) RuntimeSchedulerOption {
	return func(s *RuntimeScheduler) {
		if strings.TrimSpace(root) != "" {
			s.workspaceRoot = strings.TrimSpace(root)
		}
	}
}

func WithRuntimeSchedulerGatewayPortRange(start, end int) RuntimeSchedulerOption {
	return func(s *RuntimeScheduler) {
		if start > 0 {
			s.gatewayPortStart = start
		}
		if end > 0 {
			s.gatewayPortEnd = end
		}
		if s.gatewayPortEnd < s.gatewayPortStart {
			s.gatewayPortEnd = s.gatewayPortStart
		}
	}
}

func WithRuntimeSchedulerHeartbeatTimeout(timeout time.Duration) RuntimeSchedulerOption {
	return func(s *RuntimeScheduler) {
		s.heartbeatTimeout = timeout
	}
}

func WithRuntimeSchedulerNamespace(namespace string) RuntimeSchedulerOption {
	return func(s *RuntimeScheduler) {
		if strings.TrimSpace(namespace) != "" {
			s.runtimeNamespace = strings.TrimSpace(namespace)
		}
	}
}

func WithRuntimeSchedulerMaxGatewaysPerPod(capacity int) RuntimeSchedulerOption {
	return func(s *RuntimeScheduler) {
		if capacity > 0 {
			s.maxGatewaysPerPod = capacity
		}
	}
}

func WithRuntimeSchedulerGatewayStartInFlightLimit(limit int) RuntimeSchedulerOption {
	return func(s *RuntimeScheduler) {
		if limit > 0 {
			s.gatewayStartInFlightLimit = limit
		}
	}
}

func WithRuntimeSchedulerGatewayEnvBuilder(builder RuntimeGatewayEnvBuilder) RuntimeSchedulerOption {
	return func(s *RuntimeScheduler) {
		s.envBuilder = builder
	}
}

func NewRuntimeScheduler(
	instanceRepo repository.InstanceRepository,
	podRepo repository.RuntimePodRepository,
	bindingRepo repository.InstanceRuntimeBindingRepository,
	rolloutRepo repository.RuntimeRolloutRepository,
	agentClient RuntimeAgentClient,
	events RuntimeEventService,
	leader RuntimeLeaderService,
	deployments k8s.RuntimeDeploymentService,
	tick time.Duration,
	opts ...RuntimeSchedulerOption,
) *RuntimeScheduler {
	if tick <= 0 {
		tick = 2 * time.Second
	}
	s := &RuntimeScheduler{
		instanceRepo:              instanceRepo,
		podRepo:                   podRepo,
		bindingRepo:               bindingRepo,
		rolloutRepo:               rolloutRepo,
		agentClient:               agentClient,
		events:                    events,
		leader:                    leader,
		deployments:               deployments,
		tick:                      tick,
		workspaceRoot:             "/workspaces",
		runtimeNamespace:          "clawmanager-system",
		gatewayPortStart:          RuntimeGatewayPortStart,
		gatewayPortEnd:            RuntimeGatewayPortEnd,
		heartbeatTimeout:          10 * time.Second,
		maxGatewaysPerPod:         RuntimePodCapacity,
		gatewayStartInFlightLimit: defaultRuntimeGatewayStartInFlightLimit,
		gatewayCreateLocks:        map[int64]*sync.Mutex{},
	}
	for _, opt := range opts {
		opt(s)
	}
	return s
}

func (s *RuntimeScheduler) Start(ctx context.Context) {
	if s == nil {
		return
	}
	go func() {
		ticker := time.NewTicker(s.tick)
		defer ticker.Stop()

		s.reconcileIfLeader(ctx)
		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				s.reconcileIfLeader(ctx)
			}
		}
	}()
}

func (s *RuntimeScheduler) HeartbeatTimeout() time.Duration {
	if s == nil {
		return 0
	}
	return s.heartbeatTimeout
}

func (s *RuntimeScheduler) DrainPod(ctx context.Context, podID int64) error {
	if s == nil || s.podRepo == nil {
		return fmt.Errorf("runtime scheduler pod repository is not configured")
	}
	pod, err := s.podRepo.GetByID(ctx, podID)
	if err != nil {
		return err
	}
	if pod == nil {
		return fmt.Errorf("runtime pod %d not found", podID)
	}
	if pod.AgentEndpoint != nil && strings.TrimSpace(*pod.AgentEndpoint) != "" && s.agentClient != nil {
		if err := s.agentClient.Drain(ctx, strings.TrimSpace(*pod.AgentEndpoint)); err != nil {
			return err
		}
	}
	return s.podRepo.MarkState(ctx, podID, "draining", true)
}

func (s *RuntimeScheduler) FailoverPod(ctx context.Context, podID int64, reason string) error {
	if s == nil || s.podRepo == nil || s.bindingRepo == nil || s.instanceRepo == nil {
		return fmt.Errorf("runtime scheduler dependencies are not configured")
	}
	var errs []error
	if err := s.podRepo.MarkState(ctx, podID, "unhealthy", false); err != nil {
		errs = append(errs, err)
	}

	bindings, err := s.bindingRepo.ListByRuntimePodID(ctx, podID)
	if err != nil {
		errs = append(errs, err)
		return errors.Join(errs...)
	}
	for _, binding := range bindings {
		nextGeneration := binding.Generation + 1
		instance, err := s.instanceRepo.GetByID(binding.InstanceID)
		if err != nil {
			errs = append(errs, err)
			continue
		}
		if instance != nil {
			nextGeneration = instance.RuntimeGeneration + 1
		}
		message := reason
		if err := s.instanceRepo.UpdateRuntimeState(ctx, binding.InstanceID, "creating", nextGeneration, &message); err != nil {
			errs = append(errs, err)
			continue
		}
		if err := s.bindingRepo.DeleteByInstanceID(ctx, binding.InstanceID); err != nil {
			errs = append(errs, err)
			continue
		}
		if err := s.podRepo.ReleaseSlot(ctx, podID); err != nil {
			errs = append(errs, err)
		}
	}
	return errors.Join(errs...)
}

func (s *RuntimeScheduler) StartRollout(ctx context.Context, rolloutID int64) error {
	if s == nil || s.rolloutRepo == nil || s.podRepo == nil {
		return fmt.Errorf("runtime scheduler rollout dependencies are not configured")
	}
	rollout, err := s.rolloutRepo.GetByID(ctx, rolloutID)
	if err != nil {
		return err
	}
	if rollout == nil {
		return fmt.Errorf("runtime rollout %d not found", rolloutID)
	}

	startedAt := time.Now().UTC()
	if err := s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "running", &startedAt, nil, nil); err != nil {
		return err
	}

	allPods, err := s.podRepo.List(ctx, rollout.RuntimeType)
	if err != nil {
		message := err.Error()
		_ = s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "error", &startedAt, nil, &message)
		return err
	}
	currentPods := s.currentRuntimePods(allPods, time.Now().UTC())
	if runtimePodsAlreadyAtImage(currentPods, rollout.RuntimeType, rollout.TargetImageRef) {
		finishedAt := time.Now().UTC()
		return s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "finished", &startedAt, &finishedAt, nil)
	}
	batchSize := rollout.BatchSize
	if batchSize <= 0 {
		batchSize = 1
	}
	maxUnavailable := rollout.MaxUnavailable
	if maxUnavailable <= 0 {
		maxUnavailable = 1
	}
	if err := s.rolloutRuntimeDeployments(ctx, rollout, allPods, maxUnavailable, batchSize); err != nil {
		message := err.Error()
		_ = s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "error", &startedAt, nil, &message)
		return err
	}
	unavailable := 0
	readyCandidates := 0
	for _, pod := range currentPods {
		if pod.Draining || pod.State != "ready" {
			unavailable++
			continue
		}
		readyCandidates++
	}
	remaining := maxUnavailable - unavailable
	if remaining <= 0 {
		return nil
	}
	drainLimit := minInt(batchSize, remaining)
	var errs []error
	drained := 0
	for _, pod := range currentPods {
		if drained >= drainLimit {
			break
		}
		if pod.State != "ready" || pod.Draining {
			continue
		}
		if err := s.DrainPod(ctx, pod.ID); err != nil {
			errs = append(errs, err)
			continue
		}
		drained++
	}
	if len(errs) > 0 {
		joined := errors.Join(errs...)
		message := joined.Error()
		_ = s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "error", &startedAt, nil, &message)
		return joined
	}
	if drained == 0 && readyCandidates == 0 && unavailable == 0 && len(currentPods) > 0 {
		finishedAt := time.Now().UTC()
		return s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "finished", &startedAt, &finishedAt, nil)
	}
	return nil
}

func (s *RuntimeScheduler) reconcileRollouts(ctx context.Context) error {
	if s == nil || s.rolloutRepo == nil {
		return nil
	}
	rollouts, err := s.rolloutRepo.ListActive(ctx, "")
	if err != nil {
		return fmt.Errorf("list active runtime rollouts: %w", err)
	}
	var errs []error
	for _, rollout := range rollouts {
		switch rollout.Status {
		case "pending":
			if err := s.StartRollout(ctx, rollout.ID); err != nil {
				errs = append(errs, fmt.Errorf("start runtime rollout %d: %w", rollout.ID, err))
			}
		case "running":
			if err := s.finishRolloutIfReady(ctx, rollout); err != nil {
				errs = append(errs, fmt.Errorf("finish runtime rollout %d: %w", rollout.ID, err))
			}
		}
	}
	return errors.Join(errs...)
}

func (s *RuntimeScheduler) rolloutRuntimeDeployments(ctx context.Context, rollout *models.RuntimeRollout, pods []models.RuntimePod, maxUnavailable, maxSurge int) error {
	if s == nil || s.deployments == nil || rollout == nil {
		return nil
	}
	targetImage := strings.TrimSpace(rollout.TargetImageRef)
	if targetImage == "" {
		return nil
	}
	type deploymentRef struct {
		namespace string
		name      string
	}
	refs := map[deploymentRef]struct{}{}
	for _, pod := range pods {
		if pod.RuntimeType != rollout.RuntimeType {
			continue
		}
		namespace := strings.TrimSpace(pod.Namespace)
		name := strings.TrimSpace(pod.DeploymentName)
		if namespace == "" || name == "" {
			continue
		}
		refs[deploymentRef{namespace: namespace, name: name}] = struct{}{}
	}
	if len(refs) == 0 {
		name := defaultRuntimeDeploymentName(rollout.RuntimeType)
		if name == "" {
			return fmt.Errorf("no runtime deployment found for %s rollout", rollout.RuntimeType)
		}
		refs[deploymentRef{namespace: s.runtimeNamespace, name: name}] = struct{}{}
	}
	var errs []error
	for ref := range refs {
		if err := s.deployments.RolloutImage(ctx, ref.namespace, ref.name, targetImage, maxUnavailable, maxSurge); err != nil {
			errs = append(errs, err)
		}
	}
	return errors.Join(errs...)
}

func (s *RuntimeScheduler) RuntimeDeploymentPods(ctx context.Context, runtimeType string) ([]models.RuntimePod, error) {
	if s == nil || s.deployments == nil {
		return nil, nil
	}
	runtimeType = strings.TrimSpace(runtimeType)
	pods, err := s.deployments.ListPods(ctx, s.runtimeNamespace, runtimeType)
	if err != nil {
		return nil, err
	}
	result := make([]models.RuntimePod, 0, len(pods))
	for index, pod := range pods {
		result = append(result, models.RuntimePod{
			ID:             -int64(index + 1),
			RuntimeType:    pod.RuntimeType,
			Namespace:      pod.Namespace,
			PodName:        pod.PodName,
			PodIP:          pod.PodIP,
			NodeName:       pod.NodeName,
			DeploymentName: pod.DeploymentName,
			ImageRef:       pod.ImageRef,
			State:          runtimeDeploymentFallbackState(pod.State),
			Capacity:       s.maxGatewaysPerPod,
			UsedSlots:      0,
			Draining:       false,
		})
	}
	return result, nil
}

func defaultRuntimeDeploymentName(runtimeType string) string {
	switch strings.ToLower(strings.TrimSpace(runtimeType)) {
	case RuntimeTypeOpenClaw:
		return "openclaw-runtime"
	case RuntimeTypeHermes:
		return "hermes-runtime"
	default:
		return ""
	}
}

func runtimeDeploymentFallbackState(k8sState string) string {
	switch strings.TrimSpace(k8sState) {
	case "pending", "deleted":
		return strings.TrimSpace(k8sState)
	default:
		return "unhealthy"
	}
}

func (s *RuntimeScheduler) finishRolloutIfReady(ctx context.Context, rollout models.RuntimeRollout) error {
	if s == nil || s.podRepo == nil || s.rolloutRepo == nil {
		return nil
	}
	targetImage := strings.TrimSpace(rollout.TargetImageRef)
	if targetImage == "" {
		return nil
	}
	pods, err := s.podRepo.List(ctx, rollout.RuntimeType)
	if err != nil {
		return err
	}
	pods = s.currentRuntimePods(pods, time.Now().UTC())
	if len(pods) == 0 {
		return nil
	}
	for _, pod := range pods {
		if pod.RuntimeType != rollout.RuntimeType {
			continue
		}
		if pod.State != "ready" || pod.Draining || strings.TrimSpace(pod.ImageRef) != targetImage {
			return nil
		}
	}
	finishedAt := time.Now().UTC()
	return s.rolloutRepo.UpdateStatus(ctx, rollout.ID, "finished", rollout.StartedAt, &finishedAt, nil)
}

func (s *RuntimeScheduler) currentRuntimePods(pods []models.RuntimePod, now time.Time) []models.RuntimePod {
	if s == nil || s.heartbeatTimeout <= 0 {
		return pods
	}
	cutoff := now.UTC().Add(-runtimeRolloutStaleWindowMultiplier * s.heartbeatTimeout)
	current := pods[:0]
	for _, pod := range pods {
		if pod.LastSeenAt != nil && pod.LastSeenAt.UTC().Before(cutoff) {
			continue
		}
		current = append(current, pod)
	}
	return current
}

func runtimePodsAlreadyAtImage(pods []models.RuntimePod, runtimeType, targetImage string) bool {
	targetImage = strings.TrimSpace(targetImage)
	if targetImage == "" {
		return false
	}
	matched := 0
	for _, pod := range pods {
		if pod.RuntimeType != runtimeType {
			continue
		}
		matched++
		if pod.State != "ready" || pod.Draining || strings.TrimSpace(pod.ImageRef) != targetImage {
			return false
		}
	}
	return matched > 0
}

func (s *RuntimeScheduler) reconcileIfLeader(ctx context.Context) {
	if s.leader != nil && !s.leader.IsLeader(ctx) {
		return
	}
	if err := s.reconcile(ctx); err != nil {
		log.Printf("runtime scheduler reconcile failed: %v", err)
	}
}

func (s *RuntimeScheduler) reconcile(ctx context.Context) error {
	if s == nil || s.instanceRepo == nil || s.bindingRepo == nil || s.podRepo == nil {
		return fmt.Errorf("runtime scheduler dependencies are not configured")
	}
	var errs []error
	if err := s.failoverStalePods(ctx); err != nil {
		errs = append(errs, err)
	}
	if err := s.reconcileRollouts(ctx); err != nil {
		errs = append(errs, err)
	}
	creating, err := s.instanceRepo.GetV2Creating(ctx, runtimeSchedulerBatchLimit)
	if err != nil {
		errs = append(errs, err)
	} else {
		errs = append(errs, s.reconcileCreatingInstances(ctx, creating)...)
	}

	desired, err := s.instanceRepo.GetV2DesiredRunning(ctx, runtimeSchedulerBatchLimit)
	if err != nil {
		errs = append(errs, err)
	} else {
		if err := s.scaleOutForPendingBacklog(ctx, desired); err != nil {
			errs = append(errs, err)
		}
		for _, instance := range desired {
			if !isSchedulerManagedV2Instance(instance) {
				continue
			}
			if isRuntimeErrorInstance(instance) && !isRecoverableRuntimeSchedulingError(instance) {
				continue
			}
			binding, err := s.bindingRepo.GetRunningByInstanceID(ctx, instance.ID)
			if err != nil {
				errs = append(errs, fmt.Errorf("get running binding for instance %d: %w", instance.ID, err))
				continue
			}
			if binding != nil {
				stale, staleErr := s.cleanupStaleInstanceBinding(ctx, instance, binding)
				if staleErr != nil {
					errs = append(errs, fmt.Errorf("clean stale running binding for desired instance %d: %w", instance.ID, staleErr))
					continue
				}
				if stale {
					binding = nil
				} else if err := s.syncInstanceStateFromBinding(ctx, instance, binding); err != nil {
					errs = append(errs, fmt.Errorf("sync desired instance %d from binding: %w", instance.ID, err))
					continue
				} else {
					continue
				}
			}
			binding, err = s.bindingRepo.GetByInstanceID(ctx, instance.ID)
			if err != nil {
				errs = append(errs, fmt.Errorf("get binding for desired instance %d: %w", instance.ID, err))
				continue
			}
			if binding != nil {
				stale, staleErr := s.cleanupStaleInstanceBinding(ctx, instance, binding)
				if staleErr != nil {
					errs = append(errs, fmt.Errorf("clean stale binding for desired instance %d: %w", instance.ID, staleErr))
					continue
				}
				if stale {
					binding = nil
				} else if err := s.syncInstanceStateFromBinding(ctx, instance, binding); err != nil {
					errs = append(errs, fmt.Errorf("sync desired instance %d from binding: %w", instance.ID, err))
					continue
				} else {
					continue
				}
			}
			if assignErr := s.assignInstance(ctx, instance); assignErr != nil {
				if errors.Is(assignErr, errRuntimeScaleOutPending) || errors.Is(assignErr, errRuntimeGatewayStartPending) {
					continue
				}
				errs = append(errs, fmt.Errorf("assign desired instance %d: %w", instance.ID, assignErr))
				s.markInstanceError(ctx, instance, assignErr, &errs)
			}
		}
	}
	return errors.Join(errs...)
}

func (s *RuntimeScheduler) reconcileCreatingInstances(ctx context.Context, instances []models.Instance) []error {
	if len(instances) == 0 {
		return nil
	}
	concurrency := s.gatewayStartInFlightLimit
	if concurrency <= 0 {
		concurrency = 1
	}
	concurrency = minInt(concurrency, len(instances))

	jobs := make(chan models.Instance)
	var wg sync.WaitGroup
	var errsMu sync.Mutex
	var errs []error
	for range concurrency {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for instance := range jobs {
				instanceErrs := s.reconcileCreatingInstance(ctx, instance)
				if len(instanceErrs) == 0 {
					continue
				}
				errsMu.Lock()
				errs = append(errs, instanceErrs...)
				errsMu.Unlock()
			}
		}()
	}
	for _, instance := range instances {
		if isSchedulerManagedV2Instance(instance) {
			jobs <- instance
		}
	}
	close(jobs)
	wg.Wait()
	return errs
}

func (s *RuntimeScheduler) reconcileCreatingInstance(ctx context.Context, instance models.Instance) []error {
	binding, err := s.bindingRepo.GetByInstanceID(ctx, instance.ID)
	if err != nil {
		return []error{fmt.Errorf("get binding for creating instance %d: %w", instance.ID, err)}
	}
	if binding != nil {
		stale, staleErr := s.cleanupStaleInstanceBinding(ctx, instance, binding)
		if staleErr != nil {
			return []error{fmt.Errorf("clean stale binding for creating instance %d: %w", instance.ID, staleErr)}
		}
		if !stale {
			if err := s.syncInstanceStateFromBinding(ctx, instance, binding); err != nil {
				return []error{fmt.Errorf("sync creating instance %d from binding: %w", instance.ID, err)}
			}
			return nil
		}
	}
	if err := s.assignInstance(ctx, instance); err != nil {
		if errors.Is(err, errRuntimeScaleOutPending) || errors.Is(err, errRuntimeGatewayStartPending) {
			return nil
		}
		errs := []error{fmt.Errorf("assign creating instance %d: %w", instance.ID, err)}
		s.markInstanceError(ctx, instance, err, &errs)
		return errs
	}
	return nil
}

// cleanupStaleInstanceBinding removes only a binding from an older runtime
// generation. A newer binding may have been created after the instance snapshot
// was read and must never be removed by this reconciliation pass.
func (s *RuntimeScheduler) cleanupStaleInstanceBinding(ctx context.Context, instance models.Instance, binding *models.InstanceRuntimeBinding) (bool, error) {
	if binding == nil || binding.Generation >= instance.RuntimeGeneration {
		return false, nil
	}
	if s.bindingRepo == nil {
		return false, fmt.Errorf("runtime binding repository is not configured")
	}
	if s.podRepo != nil && s.agentClient != nil && strings.TrimSpace(binding.GatewayID) != "" {
		pod, err := s.podRepo.GetByID(ctx, binding.RuntimePodID)
		if err != nil {
			return false, fmt.Errorf("get runtime pod %d: %w", binding.RuntimePodID, err)
		}
		if pod != nil && pod.AgentEndpoint != nil && strings.TrimSpace(*pod.AgentEndpoint) != "" {
			if err := s.agentClient.DeleteGateway(ctx, strings.TrimSpace(*pod.AgentEndpoint), binding.GatewayID); err != nil && !errors.Is(err, ErrRuntimeAgentNotFound) {
				return false, fmt.Errorf("delete stale gateway %q: %w", binding.GatewayID, err)
			}
		}
	}
	if err := s.bindingRepo.DeleteByInstanceIDAndReleaseSlot(ctx, instance.ID, binding.RuntimePodID); err != nil {
		return false, fmt.Errorf("delete stale binding and release slot: %w", err)
	}
	return true, nil
}

func (s *RuntimeScheduler) syncInstanceStateFromBinding(ctx context.Context, instance models.Instance, binding *models.InstanceRuntimeBinding) error {
	if s == nil || s.instanceRepo == nil || binding == nil {
		return nil
	}
	state := strings.ToLower(strings.TrimSpace(binding.State))
	switch state {
	case "running", "ready", "healthy":
		if strings.EqualFold(strings.TrimSpace(instance.Status), "running") && instance.RuntimeErrorMessage == nil {
			return nil
		}
		return s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "running", maxInt(instance.RuntimeGeneration, binding.Generation), nil)
	case "error", "failed":
		message := "runtime gateway failed"
		if binding.ErrorMessage != nil && strings.TrimSpace(*binding.ErrorMessage) != "" {
			message = strings.TrimSpace(*binding.ErrorMessage)
		}
		if strings.EqualFold(strings.TrimSpace(instance.Status), "error") &&
			instance.RuntimeErrorMessage != nil &&
			strings.TrimSpace(*instance.RuntimeErrorMessage) == message {
			return nil
		}
		return s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "error", maxInt(instance.RuntimeGeneration, binding.Generation), &message)
	default:
		return nil
	}
}

func (s *RuntimeScheduler) failoverStalePods(ctx context.Context) error {
	if s.heartbeatTimeout <= 0 {
		return nil
	}
	pods, err := s.podRepo.List(ctx, "")
	if err != nil {
		return fmt.Errorf("list runtime pods for heartbeat failover: %w", err)
	}
	cutoff := time.Now().UTC().Add(-s.heartbeatTimeout)
	var errs []error
	for _, pod := range pods {
		if pod.State == "unhealthy" || pod.State == "pending" {
			continue
		}
		if pod.LastSeenAt == nil || !pod.LastSeenAt.Before(cutoff) {
			continue
		}
		reason := fmt.Sprintf("runtime pod heartbeat lost since %s", pod.LastSeenAt.UTC().Format(time.RFC3339))
		if err := s.FailoverPod(ctx, pod.ID, reason); err != nil {
			errs = append(errs, fmt.Errorf("failover stale runtime pod %d: %w", pod.ID, err))
		}
	}
	return errors.Join(errs...)
}

func (s *RuntimeScheduler) lockGatewayCreateForPod(podID int64) func() {
	if s == nil {
		return func() {}
	}
	s.gatewayCreateLocksMu.Lock()
	if s.gatewayCreateLocks == nil {
		s.gatewayCreateLocks = map[int64]*sync.Mutex{}
	}
	lock := s.gatewayCreateLocks[podID]
	if lock == nil {
		lock = &sync.Mutex{}
		s.gatewayCreateLocks[podID] = lock
	}
	s.gatewayCreateLocksMu.Unlock()

	lock.Lock()
	return lock.Unlock
}

func (s *RuntimeScheduler) podCanStartGateway(ctx context.Context, podID int64) (bool, error) {
	if s == nil || s.bindingRepo == nil || s.gatewayStartInFlightLimit <= 0 {
		return true, nil
	}
	bindings, err := s.bindingRepo.ListByRuntimePodID(ctx, podID)
	if err != nil {
		return false, fmt.Errorf("list runtime pod %d bindings: %w", podID, err)
	}
	starting := 0
	for _, binding := range bindings {
		switch strings.ToLower(strings.TrimSpace(binding.State)) {
		case "creating", "starting":
			starting++
		}
	}
	return starting < s.gatewayStartInFlightLimit, nil
}

func (s *RuntimeScheduler) scaleOutForPendingBacklog(ctx context.Context, instances []models.Instance) error {
	if s == nil || s.bindingRepo == nil || s.podRepo == nil || s.deployments == nil || len(instances) == 0 {
		return nil
	}
	backlogByType := map[string]int{}
	for _, instance := range instances {
		if !isSchedulerManagedV2Instance(instance) {
			continue
		}
		if isRuntimeErrorInstance(instance) && !isRecoverableRuntimeSchedulingError(instance) {
			continue
		}
		runtimeType, ok := schedulerRuntimeType(instance)
		if !ok {
			continue
		}
		binding, err := s.bindingRepo.GetByInstanceID(ctx, instance.ID)
		if err != nil {
			return fmt.Errorf("get binding for backlog instance %d: %w", instance.ID, err)
		}
		if binding == nil {
			backlogByType[runtimeType]++
		}
	}
	for runtimeType, backlog := range backlogByType {
		if backlog <= 0 {
			continue
		}
		if _, err := s.scaleOutIfBacklogExceedsCapacity(ctx, runtimeType, backlog); err != nil {
			return err
		}
	}
	return nil
}

func (s *RuntimeScheduler) scaleOutIfBacklogExceedsCapacity(ctx context.Context, runtimeType string, backlog int) (bool, error) {
	if s == nil || s.deployments == nil || s.podRepo == nil || backlog <= 0 {
		return false, nil
	}
	pods, err := s.podRepo.List(ctx, runtimeType)
	if err != nil {
		return false, fmt.Errorf("list %s runtime pods for backlog scale-out: %w", runtimeType, err)
	}
	groups := map[string]*runtimeDeploymentCapacity{}
	for _, pod := range pods {
		if pod.RuntimeType != runtimeType || pod.State != "ready" || pod.Draining || pod.Capacity <= 0 {
			continue
		}
		namespace := strings.TrimSpace(pod.Namespace)
		name := strings.TrimSpace(pod.DeploymentName)
		if namespace == "" || name == "" {
			continue
		}
		key := namespace + "/" + name
		group := groups[key]
		if group == nil {
			group = &runtimeDeploymentCapacity{namespace: namespace, name: name}
			groups[key] = group
		}
		group.active++
		group.capacityTotal += pod.Capacity
		group.usedSlots += minInt(pod.UsedSlots, pod.Capacity)
	}
	var target *runtimeDeploymentCapacity
	for _, group := range groups {
		if group.active == 0 || group.capacityTotal <= 0 {
			continue
		}
		if target == nil || group.active > target.active {
			target = group
		}
	}
	if target == nil {
		return false, nil
	}
	desiredSlots := target.usedSlots + backlog
	if desiredSlots <= target.capacityTotal {
		return false, nil
	}
	capacityPerPod := target.capacityTotal / target.active
	if capacityPerPod <= 0 {
		capacityPerPod = s.maxGatewaysPerPod
	}
	if capacityPerPod <= 0 {
		capacityPerPod = RuntimePodCapacity
	}
	targetReplicas := (desiredSlots + capacityPerPod - 1) / capacityPerPod
	if targetReplicas <= target.active {
		targetReplicas = target.active + 1
	}
	// Scale one replica at a time so a large batch cannot stampede the node.
	replicas := int32(minInt(targetReplicas, target.active+1))
	if err := s.deployments.Scale(ctx, target.namespace, target.name, replicas); err != nil {
		return false, err
	}
	if s.events != nil {
		if err := s.events.Publish(ctx, "runtime.pool.scaleout", map[string]any{
			"runtime_type": runtimeType,
			"namespace":    target.namespace,
			"deployment":   target.name,
			"replicas":     replicas,
			"reason":       "pending_backlog",
			"backlog":      backlog,
		}); err != nil {
			log.Printf("runtime scheduler publish backlog scale-out event failed: %v", err)
		}
	}
	return true, nil
}
func (s *RuntimeScheduler) assignInstance(ctx context.Context, instance models.Instance) error {
	if s == nil || s.podRepo == nil {
		return fmt.Errorf("runtime scheduler pod repository is not configured")
	}
	if !isSchedulerManagedV2Instance(instance) {
		return fmt.Errorf("instance %d is not scheduler-managed v2", instance.ID)
	}
	if s.bindingRepo != nil {
		binding, err := s.bindingRepo.GetByInstanceID(ctx, instance.ID)
		if err != nil {
			return fmt.Errorf("failed to check existing binding: %w", err)
		}
		if binding != nil {
			return nil
		}
	}
	runtimeType, ok := schedulerRuntimeType(instance)
	if !ok {
		return fmt.Errorf("unsupported runtime type %q", instance.Type)
	}
	pods, err := s.podRepo.ListSchedulable(ctx, runtimeType)
	if err != nil {
		return err
	}
	if len(pods) == 0 {
		scaled, err := s.scaleOutIfAtCapacity(ctx, runtimeType)
		if err != nil {
			return fmt.Errorf("scale out %s runtime deployment: %w", runtimeType, err)
		}
		if scaled {
			return errRuntimeScaleOutPending
		}
	}
	var lastErr error
	for _, pod := range pods {
		if pod.AgentEndpoint == nil || strings.TrimSpace(*pod.AgentEndpoint) == "" {
			continue
		}
		unlock := s.lockGatewayCreateForPod(pod.ID)
		canStart, err := s.podCanStartGateway(ctx, pod.ID)
		if err != nil {
			unlock()
			lastErr = err
			continue
		}
		if !canStart {
			unlock()
			lastErr = fmt.Errorf("runtime pod %d has gateway start in flight: %w", pod.ID, errRuntimeGatewayStartPending)
			continue
		}
		claimed, err := s.podRepo.TryClaimSlot(ctx, pod.ID)
		if err != nil {
			unlock()
			lastErr = err
			continue
		}
		if !claimed {
			unlock()
			continue
		}
		start, err := s.prepareGatewayStart(ctx, instance, runtimeType, pod)
		unlock()
		if err != nil {
			if releaseErr := s.podRepo.ReleaseSlot(ctx, pod.ID); releaseErr != nil {
				return errors.Join(err, releaseErr)
			}
			lastErr = err
			continue
		}
		if err := s.createGatewayOnPodWithPortFallback(ctx, instance, runtimeType, pod, start); err != nil {
			if releaseErr := s.podRepo.ReleaseSlot(ctx, pod.ID); releaseErr != nil {
				return errors.Join(err, releaseErr)
			}
			lastErr = err
			continue
		}
		return nil
	}
	if lastErr != nil {
		return fmt.Errorf("no schedulable %s runtime pod: %w", runtimeType, lastErr)
	}
	return fmt.Errorf("no schedulable %s runtime pod", runtimeType)
}

type runtimeDeploymentCapacity struct {
	namespace     string
	name          string
	active        int
	full          int
	capacityTotal int
	usedSlots     int
}

func (s *RuntimeScheduler) scaleOutIfAtCapacity(ctx context.Context, runtimeType string) (bool, error) {
	if s == nil || s.deployments == nil || s.podRepo == nil {
		return false, nil
	}
	pods, err := s.podRepo.List(ctx, runtimeType)
	if err != nil {
		return false, fmt.Errorf("list %s runtime pods for scale-out: %w", runtimeType, err)
	}
	groups := map[string]*runtimeDeploymentCapacity{}
	for _, pod := range pods {
		if pod.RuntimeType != runtimeType || pod.State != "ready" || pod.Draining || pod.Capacity <= 0 {
			continue
		}
		namespace := strings.TrimSpace(pod.Namespace)
		name := strings.TrimSpace(pod.DeploymentName)
		if namespace == "" || name == "" {
			continue
		}
		key := namespace + "/" + name
		group := groups[key]
		if group == nil {
			group = &runtimeDeploymentCapacity{namespace: namespace, name: name}
			groups[key] = group
		}
		group.active++
		if pod.UsedSlots >= pod.Capacity {
			group.full++
		}
	}
	var target *runtimeDeploymentCapacity
	for _, group := range groups {
		if group.active == 0 || group.active != group.full {
			continue
		}
		if target == nil || group.active > target.active {
			target = group
		}
	}
	if target == nil {
		return false, nil
	}
	replicas := int32(target.active + 1)
	if err := s.deployments.Scale(ctx, target.namespace, target.name, replicas); err != nil {
		return false, err
	}
	if s.events != nil {
		if err := s.events.Publish(ctx, "runtime.pool.scaleout", map[string]any{
			"runtime_type": runtimeType,
			"namespace":    target.namespace,
			"deployment":   target.name,
			"replicas":     replicas,
		}); err != nil {
			log.Printf("runtime scheduler publish scale-out event failed: %v", err)
		}
	}
	return true, nil
}

type runtimeGatewayStart struct {
	endpoint        string
	workspacePath   string
	environment     map[string]string
	uid             int
	gid             int
	reservedBinding *models.InstanceRuntimeBinding
}

func (s *RuntimeScheduler) prepareGatewayStart(ctx context.Context, instance models.Instance, runtimeType string, pod models.RuntimePod) (*runtimeGatewayStart, error) {
	return s.prepareGatewayStartExcludingPorts(ctx, instance, runtimeType, pod, nil)
}

func (s *RuntimeScheduler) prepareGatewayStartExcludingPorts(
	ctx context.Context,
	instance models.Instance,
	runtimeType string,
	pod models.RuntimePod,
	excludedPorts map[int]struct{},
) (*runtimeGatewayStart, error) {
	if s.agentClient == nil || s.bindingRepo == nil || s.instanceRepo == nil {
		return nil, fmt.Errorf("runtime scheduler gateway dependencies are not configured")
	}
	endpoint := strings.TrimSpace(*pod.AgentEndpoint)
	workspacePath := RuntimeWorkspacePathWithRoot(s.workspaceRoot, runtimeType, instance.UserID, instance.ID)
	environment, err := s.gatewayEnvironment(&instance)
	if err != nil {
		return nil, fmt.Errorf("build runtime gateway environment: %w", err)
	}
	uid, gid := runtimeGatewayLinuxIDs(instance.ID, environment)
	if runtimeType == RuntimeTypeOpenClaw {
		if err := ensureOpenClawPluginLayoutCompatibility(workspacePath, uid, gid); err != nil {
			return nil, fmt.Errorf("prepare OpenClaw workspace compatibility: %w", err)
		}
	}
	reservedBinding, err := s.reserveGatewayPortExcludingPorts(ctx, instance, runtimeType, pod, workspacePath, excludedPorts)
	if err != nil {
		return nil, err
	}
	return &runtimeGatewayStart{
		endpoint:        endpoint,
		workspacePath:   workspacePath,
		environment:     environment,
		uid:             uid,
		gid:             gid,
		reservedBinding: reservedBinding,
	}, nil
}

func (s *RuntimeScheduler) createGatewayOnPod(ctx context.Context, instance models.Instance, runtimeType string, pod models.RuntimePod, start *runtimeGatewayStart) error {
	if start == nil || start.reservedBinding == nil {
		return fmt.Errorf("runtime gateway start is not prepared")
	}
	resp, err := s.agentClient.CreateGateway(ctx, start.endpoint, RuntimeAgentCreateGatewayRequest{
		InstanceID:    instance.ID,
		UserID:        instance.UserID,
		AgentType:     runtimeType,
		WorkspacePath: start.workspacePath,
		GatewayPort:   start.reservedBinding.GatewayPort,
		PortRange: RuntimeAgentPortRange{
			Start: s.gatewayPortStart,
			End:   s.gatewayPortEnd,
		},
		UID:         start.uid,
		GID:         start.gid,
		CPUCores:    instance.CPUCores,
		MemoryMB:    instance.MemoryGB * 1024,
		DiskQuotaMB: instance.DiskGB * 1024,
		Generation:  instance.RuntimeGeneration,
		Environment: start.environment,
	})
	if err != nil {
		return s.cleanupGatewayAfterAssignFailure(ctx, start.endpoint, instance.ID, "", true, err)
	}
	if resp == nil {
		return s.cleanupGatewayAfterAssignFailure(ctx, start.endpoint, instance.ID, "", true, fmt.Errorf("runtime agent returned empty gateway response"))
	}
	if resp.Port != start.reservedBinding.GatewayPort {
		cause := fmt.Errorf("runtime agent returned gateway port %d, want control-plane allocation %d", resp.Port, start.reservedBinding.GatewayPort)
		return s.cleanupGatewayAfterAssignFailure(ctx, start.endpoint, instance.ID, resp.GatewayID, true, cause)
	}

	now := time.Now().UTC()
	lifecycle := NormalizeRuntimeGatewayLifecycle(resp.Status, nil)
	var lastHealthAt *time.Time
	if lifecycle.Running {
		lastHealthAt = &now
	}
	if err := s.bindingRepo.UpdateGatewayAssignment(ctx, instance.ID, instance.RuntimeGeneration, resp.GatewayID, resp.PID, lifecycle.BindingState, lastHealthAt); err != nil {
		return s.cleanupGatewayAfterAssignFailure(ctx, start.endpoint, instance.ID, resp.GatewayID, true, err)
	}
	if !lifecycle.CreateAccepted() {
		cause := fmt.Errorf("runtime gateway %s returned status %q", resp.GatewayID, strings.TrimSpace(resp.Status))
		message := cause.Error()
		var errs []error
		if err := s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "error", instance.RuntimeGeneration, &message); err != nil {
			errs = append(errs, fmt.Errorf("mark instance %d error: %w", instance.ID, err))
		}
		errs = append(errs, s.cleanupGatewayAfterAssignFailure(ctx, start.endpoint, instance.ID, resp.GatewayID, true, cause))
		return errors.Join(errs...)
	}
	if err := s.instanceRepo.SetWorkspacePath(ctx, instance.ID, start.workspacePath); err != nil {
		return s.cleanupGatewayAfterAssignFailure(ctx, start.endpoint, instance.ID, resp.GatewayID, true, err)
	}
	if err := s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, lifecycle.InstanceState, instance.RuntimeGeneration, lifecycle.Message); err != nil {
		return s.cleanupGatewayAfterAssignFailure(ctx, start.endpoint, instance.ID, resp.GatewayID, true, err)
	}
	if s.events != nil {
		if err := s.events.Publish(ctx, lifecycle.EventType, map[string]any{
			"instance_id":    instance.ID,
			"runtime_type":   runtimeType,
			"runtime_pod_id": pod.ID,
			"gateway_id":     resp.GatewayID,
			"gateway_port":   resp.Port,
			"gateway_state":  lifecycle.BindingState,
			"workspace_path": start.workspacePath,
			"generation":     instance.RuntimeGeneration,
		}); err != nil {
			log.Printf("runtime scheduler publish event failed: %v", err)
		}
	}
	return nil
}

func (s *RuntimeScheduler) createGatewayOnPodWithPortFallback(
	ctx context.Context,
	instance models.Instance,
	runtimeType string,
	pod models.RuntimePod,
	start *runtimeGatewayStart,
) error {
	if start == nil || start.reservedBinding == nil {
		return fmt.Errorf("runtime gateway start is not prepared")
	}
	excludedPorts := map[int]struct{}{}
	blockSize := RuntimeGatewayPortBlockSize(runtimeType)
	maxAttempts := (s.gatewayPortEnd - s.gatewayPortStart + 1) / maxInt(blockSize, 1)
	if maxAttempts <= 0 {
		maxAttempts = 1
	}

	var lastErr error
	for attempt := 0; attempt < maxAttempts; attempt++ {
		err := s.createGatewayOnPod(ctx, instance, runtimeType, pod, start)
		if err == nil {
			return nil
		}
		lastErr = err
		if !isRuntimeAgentGatewayPortConflict(err) {
			return err
		}
		excludedPorts[start.reservedBinding.GatewayPort] = struct{}{}
		if attempt+1 >= maxAttempts {
			break
		}

		unlock := s.lockGatewayCreateForPod(pod.ID)
		nextStart, prepareErr := s.prepareGatewayStartExcludingPorts(ctx, instance, runtimeType, pod, excludedPorts)
		unlock()
		if prepareErr != nil {
			return errors.Join(lastErr, prepareErr)
		}
		start = nextStart
	}
	return lastErr
}

func isRuntimeAgentGatewayPortConflict(err error) bool {
	if !errors.Is(err, ErrRuntimeAgentConflict) {
		return false
	}
	message := strings.ToLower(err.Error())
	return strings.Contains(message, "requested gateway port") &&
		(strings.Contains(message, "unavailable") || strings.Contains(message, "no free port"))
}

func (s *RuntimeScheduler) reserveGatewayPort(ctx context.Context, instance models.Instance, runtimeType string, pod models.RuntimePod, workspacePath string) (*models.InstanceRuntimeBinding, error) {
	return s.reserveGatewayPortExcludingPorts(ctx, instance, runtimeType, pod, workspacePath, nil)
}

func (s *RuntimeScheduler) reserveGatewayPortExcludingPorts(
	ctx context.Context,
	instance models.Instance,
	runtimeType string,
	pod models.RuntimePod,
	workspacePath string,
	excludedPorts map[int]struct{},
) (*models.InstanceRuntimeBinding, error) {
	if s == nil || s.bindingRepo == nil {
		return nil, fmt.Errorf("runtime scheduler binding repository is not configured")
	}
	blockSize := RuntimeGatewayPortBlockSize(runtimeType)
	if blockSize <= 0 {
		blockSize = 1
	}
	existingBindings, err := s.bindingRepo.ListByRuntimePodID(ctx, pod.ID)
	if err != nil {
		return nil, fmt.Errorf("list runtime pod %d bindings for gateway port allocation: %w", pod.ID, err)
	}
	for port := s.gatewayPortStart; port+blockSize-1 <= s.gatewayPortEnd; port += blockSize {
		if _, excluded := excludedPorts[port]; excluded {
			continue
		}
		if runtimeGatewayPortBlockOccupied(existingBindings, port, blockSize) {
			deleted, err := s.bindingRepo.DeleteErrorByRuntimePodIDAndGatewayPort(ctx, pod.ID, port)
			if err != nil {
				return nil, fmt.Errorf("clear stale gateway port %d on runtime pod %d: %w", port, pod.ID, err)
			}
			if deleted == 0 {
				continue
			}
		}
		binding := &models.InstanceRuntimeBinding{
			InstanceID:    instance.ID,
			RuntimePodID:  pod.ID,
			RuntimeType:   runtimeType,
			GatewayID:     fmt.Sprintf("pending-%d-%d", instance.ID, instance.RuntimeGeneration),
			GatewayPort:   port,
			WorkspacePath: workspacePath,
			State:         "creating",
			Generation:    instance.RuntimeGeneration,
		}
		if err := s.createRuntimeBindingWithStalePortRecovery(ctx, binding); err != nil {
			return nil, fmt.Errorf("reserve gateway port %d on runtime pod %d: %w", port, pod.ID, err)
		}
		return binding, nil
	}
	return nil, fmt.Errorf("no free gateway port block on runtime pod %d in range %d-%d", pod.ID, s.gatewayPortStart, s.gatewayPortEnd)
}

func runtimeGatewayPortBlockOccupied(bindings []models.InstanceRuntimeBinding, requestedPort, blockSize int) bool {
	requestedEnd := requestedPort + blockSize - 1
	for _, binding := range bindings {
		existingEnd := binding.GatewayPort + blockSize - 1
		if requestedPort <= existingEnd && binding.GatewayPort <= requestedEnd {
			return true
		}
	}
	return false
}

func (s *RuntimeScheduler) createRuntimeBindingWithStalePortRecovery(ctx context.Context, binding *models.InstanceRuntimeBinding) error {
	if err := s.bindingRepo.Create(ctx, binding); err != nil {
		deleted, deleteErr := s.bindingRepo.DeleteErrorByRuntimePodIDAndGatewayPort(ctx, binding.RuntimePodID, binding.GatewayPort)
		if deleteErr != nil {
			return errors.Join(err, deleteErr)
		}
		if deleted == 0 {
			return err
		}
		if retryErr := s.bindingRepo.Create(ctx, binding); retryErr != nil {
			return errors.Join(err, retryErr)
		}
	}
	return nil
}
func runtimeGatewayLinuxIDs(instanceID int, environment map[string]string) (int, int) {
	linuxID := RuntimeLinuxID(instanceID)
	if !strings.EqualFold(strings.TrimSpace(environment["CLAWMANAGER_TEAM_ENABLED"]), "true") {
		return linuxID, linuxID
	}
	sharedGID, err := strconv.Atoi(strings.TrimSpace(environment["CLAWMANAGER_TEAM_SHARED_GID"]))
	if err != nil || sharedGID <= 0 {
		return linuxID, linuxID
	}
	return linuxID, sharedGID
}

func (s *RuntimeScheduler) gatewayEnvironment(instance *models.Instance) (map[string]string, error) {
	if s == nil || s.envBuilder == nil {
		return nil, nil
	}
	env, err := s.envBuilder(instance)
	if err != nil {
		return nil, err
	}
	if len(env) == 0 {
		return nil, nil
	}
	copied := make(map[string]string, len(env))
	for key, value := range env {
		if strings.TrimSpace(key) != "" {
			copied[key] = value
		}
	}
	if len(copied) == 0 {
		return nil, nil
	}
	return copied, nil
}

func (s *RuntimeScheduler) cleanupGatewayAfterAssignFailure(ctx context.Context, endpoint string, instanceID int, gatewayID string, bindingCreated bool, cause error) error {
	errs := []error{cause}
	if gatewayID != "" && s.agentClient != nil {
		if err := s.agentClient.DeleteGateway(ctx, endpoint, gatewayID); err != nil {
			errs = append(errs, fmt.Errorf("delete gateway %s: %w", gatewayID, err))
		}
	}
	if bindingCreated && s.bindingRepo != nil {
		if err := s.bindingRepo.DeleteByInstanceID(ctx, instanceID); err != nil {
			errs = append(errs, fmt.Errorf("delete binding for instance %d: %w", instanceID, err))
		}
	}
	return errors.Join(errs...)
}

func (s *RuntimeScheduler) markInstanceError(ctx context.Context, instance models.Instance, cause error, errs *[]error) {
	message := cause.Error()
	if err := s.instanceRepo.UpdateRuntimeState(ctx, instance.ID, "error", instance.RuntimeGeneration, &message); err != nil {
		*errs = append(*errs, fmt.Errorf("mark instance %d error: %w", instance.ID, err))
	}
}

func schedulerRuntimeType(instance models.Instance) (string, bool) {
	return NormalizeV2RuntimeType(instance.Type)
}

func isSchedulerManagedV2Instance(instance models.Instance) bool {
	if _, ok := schedulerRuntimeType(instance); !ok {
		return false
	}
	if normalizeInstanceRuntimeType(instance.RuntimeType) != RuntimeBackendGateway {
		return false
	}
	if mode, ok := NormalizeInstanceMode(instance.InstanceMode); ok && mode != InstanceModeLite {
		return false
	}
	return instance.WorkspacePath != nil && strings.TrimSpace(*instance.WorkspacePath) != ""
}

func isRuntimeErrorInstance(instance models.Instance) bool {
	return strings.EqualFold(strings.TrimSpace(instance.Status), "error")
}

func isRecoverableRuntimeSchedulingError(instance models.Instance) bool {
	if !isRuntimeErrorInstance(instance) || instance.RuntimeErrorMessage == nil {
		return false
	}
	runtimeType, ok := schedulerRuntimeType(instance)
	if !ok {
		return false
	}
	message := strings.TrimSpace(*instance.RuntimeErrorMessage)
	return message == fmt.Sprintf("no schedulable %s runtime pod", runtimeType) ||
		strings.Contains(message, fmt.Sprintf("no schedulable %s runtime pod:", runtimeType)) ||
		message == "gateway start failed: exit status 1"
}

func minInt(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func maxInt(a, b int) int {
	if a > b {
		return a
	}
	return b
}
