[DEV-815] Added compose for transformers + added Helm feature to support function name (#10388)

- [x] I submit my changes into the `develop` branch
- [x] I have created a changelog fragment
- [x] I have updated the documentation accordingly (separate task)
- [x] I submit _my code changes_ under the same [MIT License](
  https://github.com/cvat-ai/cvat/blob/develop/LICENSE)

---------

Co-authored-by: Petr Iosipov <petr.iosipov@cvat.ai>
Co-authored-by: Andrey Zhavoronkov <andrey@cvat.ai>

_typos.toml

@@ -1,4 +1,5 @@
 files.extend-exclude = [
+    "ai-models/agents_deployment/transformers/supported_models",
     "cvat-ui/src/assets/opencv_4.8.0.js",
     "cvat-data/src/ts/3rdparty/*.js",
     "site/themes/docsy/",

ai-models/agents_deployment/helm/templates/helpers.tpl

@@ -94,3 +94,15 @@ Configure command arguments for cvat-cli
 - name: MODEL_CONFIG_PARAMS
   value: {{ $val | trim | quote }}
 {{- end -}}
+
+{{/*
+Configure function name for agent
+*/}}
+{{- define "agent.functionNameEnv" -}}
+- name: FUNCTION_NAME
+{{- if .Values.agent.function_name }}
+  value: {{ .Values.agent.function_name | quote }}
+{{- else }}
+  value: "MyAgentFunction"
+{{- end }}
+{{- end }}

ai-models/agents_deployment/helm/templates/job.yaml

@@ -28,6 +28,7 @@ spec:
           command: ["./function_registration.sh"]
           env:
             {{- include "agent.commonEnv" . | nindent 12 }}
+            {{- include "agent.functionNameEnv" . | nindent 12 }}
             {{- include "agent.modelParamsOverride" . | nindent 12 }}
             {{- range .Values.job.envVars }}
             - name: {{ .name }}

ai-models/agents_deployment/helm/values.yaml

@@ -8,6 +8,8 @@ image:
 
 agent:
   ### Common configuration
+  # Please specify a unique name for your agent's function. Uniqueness is username + function name.
+  function_name: "myfunction"
   # Please specify the number of agent replicas you want to deploy.
   replicaCount: 1
   # Please specify the organization slug in CVAT if you want to deploy agent under organization. Else leave empty.
@@ -35,12 +37,19 @@ agent:
       model_id:
         type: str
         value: "facebook/sam2.1-hiera-tiny"
+    transformers-detr:
+      model:
+        type: str
+        value: "facebook/detr-resnet-50"
+      task:
+        type: str
+        value: "object-detection"
     custom:
       # use modelParamsOverride to specify custom parameters for your model.
 
   # Please pick a preset for your agent's model configuration. The preset will automatically fill in the necessary parameters for the chosen model type. If you choose a preset, you can still override specific parameters in the `modelParamsOverride` section below.
   # Keys with empty 'value' will be ignored.
-  preset: sam2
+  preset: transformers-detr
   modelParamsOverride: { }
 #  modelParamsOverride:
 #      model:

ai-models/agents_deployment/transformers/.env

@@ -0,0 +1,44 @@
+# Docker compose
+CVAT_BASE_URL=your_cvat_url # Your CVAT instance URL. Default is https://app.cvat.ai
+CVAT_ACCESS_TOKEN=your_cvat_access_token_here # Required to authenticate to CVAT API. You can get it in CVAT UI: https://app.cvat.ai/profile#security
+FUNCTION_NAME=your_function_name # Name of the function to create in CVAT.
+IMAGE_URL= your_cvat_image_here # Image to use for the agent.
+AGENTS_COUNT=1  # Number of agents to create. Default is 1.
+ORG_SLUG=your_organization_slug_here # If you want to create agent for your organization.
+USE_CUDA=false # Set to true if you want to use GPU. Please ensure that you use right docker image.
+
+# Model config params for transformers
+#
+# All -p parameters are passed as **kwargs to the create() function in func.py,
+# which forwards them to transformers.pipeline(**kwargs).
+# Any argument accepted by transformers.pipeline() can be passed here.
+#
+# Required params:
+#   task    - pipeline task type. Supported values are:
+#             "object-detection", "image-classification", "image-segmentation"
+#   model   - Hugging Face model ID (the "foo/bar" string from <ModelClass>.from_pretrained("foo/bar")
+#             in the model's documentation page on https://huggingface.co/docs/transformers)
+#
+# Optional params:
+#   device  - device to run inference on, e.g. "cpu", "cuda", "cuda:0". Default is "cpu".
+#
+# How to find the right model ID:
+#   1. Pick a model class from the supported list (e.g. DetrForObjectDetection)
+#   2. Open its page in Hugging Face Transformers docs
+#   3. Scroll to the from_pretrained() example — the string argument is your model ID
+#      Example: model = DetrForObjectDetection.from_pretrained("facebook/detr-resnet-50")
+#               → model ID is "facebook/detr-resnet-50"
+#   4. Alternatively, search for the model class on https://huggingface.co/models
+#      and use the model's repo name as the model ID
+#
+# Examples:
+#   Object detection:
+#     MODEL_CONFIG_PARAMS="-p task=str:object-detection -p model=str:hustvl/yolos-tiny"
+#     MODEL_CONFIG_PARAMS="-p task=str:object-detection -p model=str:facebook/detr-resnet-50"
+#   Image classification:
+#     MODEL_CONFIG_PARAMS="-p task=str:image-classification -p model=str:microsoft/resnet-50"
+#     MODEL_CONFIG_PARAMS="-p task=str:image-classification -p model=str:google/vit-base-patch16-224"
+#   Image segmentation:
+#     MODEL_CONFIG_PARAMS="-p task=str:image-segmentation -p model=str:facebook/detr-resnet-50-panoptic"
+
+MODEL_CONFIG_PARAMS="-p task=str:object-detection -p model=str:facebook/detr-resnet-50"

ai-models/agents_deployment/transformers/Dockerfile

@@ -0,0 +1,36 @@
+FROM python:3.14-slim
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    curl \
+    git \
+    jq \
+    && rm -rf /var/lib/apt/lists/*
+
+ARG USE_GPU=false
+ARG TARGETARCH
+ADD --chmod=+x https://dl.k8s.io/release/v1.34.0/bin/linux/${TARGETARCH}/kubectl /usr/local/bin/
+
+RUN groupadd --gid 1000 agent && \
+    useradd --uid 1000 --gid 1000 --create-home agent
+
+WORKDIR /app
+
+RUN mkdir /shared && chown agent:agent /shared && chown agent:agent /app
+
+COPY ai-models/detector/transformers/requirements.txt .
+
+RUN if [ "$USE_GPU" = "true" ] && [ "$TARGETARCH" = "amd64" ]; then \
+        echo "Building GPU image" && pip install --no-cache-dir cvat-cli -r requirements.txt; \
+    else \
+        echo "Building CPU only image" && pip install --no-cache-dir cvat-cli -r requirements.txt --extra-index-url=https://download.pytorch.org/whl/cpu; \
+    fi
+
+COPY ai-models/detector/transformers/ .
+COPY --chmod=755 ai-models/agents_deployment/transformers/function_registration.sh .
+COPY --chmod=755 ai-models/agents_deployment/transformers/function_deregistration.sh .
+COPY --chmod=755 ai-models/agents_deployment/transformers/check_env.sh .
+COPY --chmod=755 ai-models/agents_deployment/transformers/entrypoint.sh .
+
+USER agent
+
+ENTRYPOINT ["./entrypoint.sh"]

ai-models/agents_deployment/transformers/check_env.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+validate_access_token() {
+    if [ -z "$CVAT_ACCESS_TOKEN" ]; then
+        echo "Error: CVAT_ACCESS_TOKEN environment variable must be set."
+        exit 1
+    fi
+}
+
+resolve_base_url() {
+    if [ -z "$CVAT_BASE_URL" ]; then
+        echo "Warning: CVAT_BASE_URL environment variable is missing, using https://app.cvat.ai as default."
+        CVAT_BASE_URL="https://app.cvat.ai"
+    fi
+}
+
+resolve_org_slug() {
+    ORG_SLUG_ARGS=()
+    if [ -z "$ORG_SLUG" ]; then
+        echo "Warning: ORG_SLUG environment variable not found. Function will be registered only for your user."
+        echo "ORG_SLUG must be the short name of the organization; it is the name displayed under your username when you switch to the organization in the CVAT UI."
+        ORG_CURL_ARGS=(--data-urlencode "org=")
+    else
+        echo "Using organization: $ORG_SLUG"
+        ORG_SLUG_ARGS=(--organization "$ORG_SLUG")
+        ORG_CURL_ARGS=(--data-urlencode "org=$ORG_SLUG")
+    fi
+}
+
+resolve_cuda() {
+    if [ "$USE_CUDA" = "true" ]; then
+        echo "Using CUDA! Please ensure that you are using proper image with CUDA support"
+        USE_CUDA_ARGS=(-p device=str:cuda)
+    else
+        echo "Info: USE_CUDA environment variable not found. Model will run on CPU."
+    fi
+}
+
+resolve_model_params() {
+    if [ -z "$MODEL_CONFIG_PARAMS" ]; then
+        echo "Warning: MODEL_CONFIG_PARAMS environment variable not found. Default model will be used: facebook/detr-resnet-50"
+        MODEL_CONFIG_PARAMS="-p model=facebook/detr-resnet-50"
+        MODEL="-p task=str:object-detection -p model=str:facebook/detr-resnet-50"
+    elif result=$(echo "$MODEL_CONFIG_PARAMS" | grep -oP 'model=str:\K[^ ]+'); then
+        echo "Extracted MODEL from MODEL_CONFIG_PARAMS: $result"
+        MODEL="$result"
+    else
+        echo "Warning: MODEL_CONFIG_PARAMS environment variable is set but model param is malformed. Your config will be discarded. Default values will be used."
+        echo "MODEL_CONFIG_PARAMS should contain model in format -p model=str:your_model_name"
+        echo "Following params will be used for cvat-cli: -p task=str:object-detection -p model=str:facebook/detr-resnet-50"
+        MODEL_CONFIG_PARAMS="-p task=str:object-detection -p model=str:facebook/detr-resnet-50"
+        MODEL="facebook/detr-resnet-50"
+    fi
+}
+
+resolve_function_name() {
+    if [ -z "$FUNCTION_NAME" ]; then
+        echo "Warning: FUNCTION_NAME environment variable not found. Default is TRANSFORMERS"
+        FUNCTION_NAME="TRANSFORMERS"
+    else
+        echo "Using FUNCTION_NAME: $FUNCTION_NAME"
+    fi
+}
+
+common_env() {
+    validate_access_token
+    resolve_base_url
+    resolve_org_slug
+}

ai-models/agents_deployment/transformers/docker-compose.yaml

@@ -0,0 +1,46 @@
+services:
+  cvat-function-register:
+    image: ${IMAGE_URL}
+    entrypoint: ["/app/function_registration.sh"]
+    volumes:
+      - shared-data:/shared
+    restart: "on-failure"
+    environment:
+      CVAT_BASE_URL:
+      CVAT_ACCESS_TOKEN:
+      MODEL_CONFIG_PARAMS:
+      ORG_SLUG:
+      FUNCTION_NAME:
+
+  cvat-agent:
+    image: ${IMAGE_URL}
+    volumes:
+      - shared-data:/shared:ro
+    depends_on:
+      cvat-function-register:
+        condition: service_completed_successfully
+    deploy:
+      replicas: ${AGENTS_COUNT:-1}
+    environment:
+      CVAT_BASE_URL:
+      CVAT_ACCESS_TOKEN:
+      MODEL_CONFIG_PARAMS:
+      ORG_SLUG:
+
+  #NB! MANUAL RUN!
+  # docker compose run --rm cvat-function-deregister
+  # This command will run this service once and remove container after execution.
+  cvat-function-deregister:
+    image: ${IMAGE_URL}
+    entrypoint: ["/app/function_deregistration.sh"]
+    volumes:
+      - shared-data:/shared:ro
+    profiles:
+      - manual
+    environment:
+      CVAT_BASE_URL:
+      CVAT_ACCESS_TOKEN:
+      ORG_SLUG:
+
+volumes:
+  shared-data:

ai-models/agents_deployment/transformers/entrypoint.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# This script runs transformers model agent in the CVAT.
+
+source "$(dirname "$0")/check_env.sh"
+
+common_env
+resolve_model_params
+resolve_cuda
+
+if [ -f /shared/FUNCTION_ID ]; then
+    echo "FUNCTION_ID file found. Reading FUNCTION_ID from /shared/FUNCTION_ID"
+    FUNCTION_ID="$(cat /shared/FUNCTION_ID)"
+fi
+
+
+if [ -z "$FUNCTION_ID" ]; then
+    echo "FUNCTION_ID environment variable not found. In compose it should be available in /shared/FUNCTION_ID file."
+    echo "If this is Helm - something is wrong with function registration"
+    echo "If this is local run - please ensure FUNCTION_ID environment variable is set or /shared/FUNCTION_ID file is created with the function id of the function you want to run."
+    echo "Exiting..."
+    exit 1
+fi
+
+FUNCTION_FILE_PATH="func.py"
+
+echo "Running transformers function agent for FUNCTION_ID: $FUNCTION_ID with MODEL: $MODEL..."
+# $MODEL_CONFIG_PARAMS should be unquoted to be passed as separate arguments to cvat-cli.
+exec cvat-cli --server-host "$CVAT_BASE_URL" "${ORG_SLUG_ARGS[@]}" function run-agent "$FUNCTION_ID" --function-file="$FUNCTION_FILE_PATH" $MODEL_CONFIG_PARAMS "${USE_CUDA_ARGS[@]}"

ai-models/agents_deployment/transformers/function_deregistration.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# This script removes the transformers model function from the CVAT.
+
+source "$(dirname "$0")/check_env.sh"
+
+common_env
+
+
+if [ -z "$KUBERNETES_SERVICE_HOST" ]; then
+    if [ -f /shared/FUNCTION_ID ]; then
+        FUNCTION_ID="$(cat /shared/FUNCTION_ID)"
+    else
+        echo -e "Warning: FUNCTION_ID file not found at /shared/FUNCTION_ID."
+    fi
+fi
+
+
+if [ -z "$FUNCTION_ID" ]; then
+    echo -e "Error: FUNCTION_ID environment variable must be set to remove function from CVAT.\nPlease consider manual removal using cvat-cli --server-host $CVAT_BASE_URL function delete FUNCTION_ID"
+    echo "Or it might be that the function was not registered at all, in that case you can safely ignore this message."
+    exit 0
+fi
+
+if [ -n "$KUBERNETES_SERVICE_HOST" ]; then
+    kubectl delete configmap "$CONFIGMAP_NAME" --namespace "$NAMESPACE"
+fi
+
+exec cvat-cli --server-host "$CVAT_BASE_URL" "${ORG_SLUG_ARGS[@]}" function delete "$FUNCTION_ID"

ai-models/agents_deployment/transformers/function_registration.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# This script registers the transformers model function in the CVAT.
+
+source "$(dirname "$0")/check_env.sh"
+
+common_env
+resolve_model_params
+resolve_function_name
+
+FUNCTION_FILE_PATH="func.py"
+
+
+# Get the username associated with the access token
+USERNAME=$(curl "$CVAT_BASE_URL"/api/users/self -s --fail --oauth2-bearer "$CVAT_ACCESS_TOKEN" | jq -r '.username')
+if [ -z "$USERNAME" ]; then
+    echo "Error: Unable to retrieve username from CVAT API. Check your access token."
+    exit 1
+else
+    echo "Authenticated as user: $USERNAME"
+fi
+
+# Try to find existing function with the same name and owner
+FILTER=$(jq -nc --arg name "${FUNCTION_NAME}" --arg owner "$USERNAME" \
+  '{"and":[{"==":[{"var":"name"},$name]},{"==":[{"var":"owner"},$owner]}]}')
+
+if FUNCTION_ID=$(curl --get --fail --data-urlencode "filter=$FILTER" "${ORG_CURL_ARGS[@]}" "$CVAT_BASE_URL"/api/functions -s --oauth2-bearer "$CVAT_ACCESS_TOKEN" | jq -r '.results[0].id') && [ "$FUNCTION_ID" != "null" ]; then
+    echo "Found existing function with ID: $FUNCTION_ID, new function won't be created."
+    echo "$FUNCTION_ID" > /shared/FUNCTION_ID
+else
+    echo -e "Function with name $FUNCTION_NAME not found. Proceeding to create a new one.\nPlease have some patience, function creation might take some time..."
+    # Register the transformers function in CVAT
+    RAW_OUTPUT="$(cvat-cli --server-host "$CVAT_BASE_URL" "${ORG_SLUG_ARGS[@]}" function create-native "$FUNCTION_NAME" --function-file="$FUNCTION_FILE_PATH" $MODEL_CONFIG_PARAMS)"
+    FUNCTION_ID=$(echo "$RAW_OUTPUT" | tail -1 | tr -d '[:space:]')
+    if [[ $FUNCTION_ID =~ ^[0-9]+$ ]]; then
+      echo "Successfully created $FUNCTION_NAME function"
+      echo "$FUNCTION_ID" > /shared/FUNCTION_ID
+    else
+      echo "cvat-cli function create-native failed. Output:"
+      echo "$RAW_OUTPUT"
+      exit 1
+    fi
+fi
+
+# Create configmap with FUNCTION_ID
+if [ -z "$KUBERNETES_SERVICE_HOST" ]; then
+    echo "This is not a Kubernetes deployment. Skipping creation of ConfigMap."
+    echo "Function registration complete!"
+    exit 0
+fi
+
+
+if kubectl -n "$NAMESPACE" get cm "$CONFIGMAP_NAME" > /dev/null 2>&1; then
+    echo "ConfigMap $CONFIGMAP_NAME already exists. Updating it with the new FUNCTION_ID."
+    kubectl -n "$NAMESPACE" patch configmap "$CONFIGMAP_NAME" \
+    -p "$(jq -n --arg func_id "$FUNCTION_ID" '{"data":{"FUNCTION_ID": $func_id}}')"
+    echo "Function registration complete!"
+    exit 0
+fi
+
+echo "Creating ConfigMap $CONFIGMAP_NAME with FUNCTION_ID."
+exec kubectl create configmap "$CONFIGMAP_NAME" \
+  --namespace "$NAMESPACE" \
+  --from-literal=FUNCTION_ID="$FUNCTION_ID"

ai-models/agents_deployment/transformers/supported_models

@@ -0,0 +1,74 @@
+We expect these models to be working as agents. Other models are not guaranteed.
+
+
+BeitForImageClassification
+BeitForSemanticSegmentation
+BitForImageClassification
+CLIPForImageClassification
+ConditionalDetrForObjectDetection	
+ConvNextForImageClassification
+ConvNextV2ForImageClassification
+CvtForImageClassification
+DFineForObjectDetection
+DabDetrForObjectDetection	
+Data2VecVisionForImageClassification
+Data2VecVisionForSemanticSegmentation
+DeformableDetrForObjectDetection	
+DeiTForImageClassification
+DeiTForImageClassificationWithTeacher
+DetrForObjectDetection	
+DetrForSegmentation	
+DinatForImageClassification	natten
+Dinov2ForImageClassification
+Dinov2WithRegistersForImageClassification
+DonutSwinForImageClassification
+DPTForSemanticSegmentation
+EfficientNetForImageClassification
+EomtForUniversalSegmentation	
+EomtDinov3ForUniversalSegmentation	
+FocalNetForImageClassification
+HGNetV2ForImageClassification
+HieraForImageClassification
+IJepaForImageClassification
+ImageGPTForImageClassification
+LevitForImageClassification
+LevitForImageClassificationWithTeacher
+LwDetrForObjectDetection
+Mask2FormerForUniversalSegmentation	
+MaskFormerForInstanceSegmentation	
+MetaClip2ForImageClassification
+MobileNetV1ForImageClassification
+MobileNetV2ForImageClassification
+MobileNetV2ForSemanticSegmentation
+MobileViTForImageClassification
+MobileViTForSemanticSegmentation
+MobileViTV2ForImageClassification
+MobileViTV2ForSemanticSegmentation
+OneFormerForUniversalSegmentation	
+PerceiverForImageClassificationLearned
+PerceiverForImageClassificationFourier
+PerceiverForImageClassificationConvProcessing
+PoolFormerForImageClassification
+PPDocLayoutV2ForObjectDetection
+PPDocLayoutV3ForObjectDetection
+PvtForImageClassification
+PvtV2ForImageClassification
+RegNetForImageClassification
+ResNetForImageClassification
+RTDetrForObjectDetection
+RTDetrV2ForObjectDetection
+SegformerForImageClassification
+SegformerForSemanticSegmentation
+ShieldGemma2ForImageClassification
+SiglipForImageClassification
+Siglip2ForImageClassification
+SwiftFormerForImageClassification
+SwinForImageClassification
+Swinv2ForImageClassification
+TableTransformerForObjectDetection	
+TextNetForImageClassification
+WrapperForImageClassification	
+UperNetForSemanticSegmentation
+ViTForImageClassification
+ViTMSNForImageClassification
+YolosForObjectDetection

changelog.d/20260318_160408_petr.iosipov_DEV_815.md

@@ -0,0 +1,4 @@
+### Added
+
+- Compose for transformers + Helm support for function name env var
+  (<https://github.com/cvat-ai/cvat/pull/10388>)