Add support for verl 0.6.0 (#246)

.github/workflows/examples-compat.yml

@@ -54,6 +54,10 @@ jobs:
         run: |
           uv sync --frozen --no-default-groups --extra apo --extra verl \
             --group dev --group experiment --group agents --group torch-gpu-${{ matrix.setup-script }}
+      - name: Override VERL (stable)
+        run: |
+          uv pip install verl==0.5.0
+        if: matrix.setup-script == 'stable'
       - name: Freeze dependencies
         run: |
           set -ex

agentlightning/algorithm/verl/interface.py

@@ -99,6 +99,8 @@ class VERL(Algorithm):
 
         # Merge your dict overrides
         override_conf = OmegaConf.create(config)
+        # Allow adding new fields
+        OmegaConf.set_struct(base_cfg, False)
         self.config = OmegaConf.merge(base_cfg, override_conf)
 
     def run(

agentlightning/instrumentation/agentops.py

@@ -28,6 +28,12 @@ _original_handle_chat_attributes: Callable[..., Any] | None = None
 _original_handle_response: Callable[..., Any] | None = None
 
 
+def _unwrap_legacy_response(response: Any) -> Any:
+    if hasattr(response, "parse") and callable(response.parse):
+        return response.parse()
+    return response
+
+
 def _patch_new_agentops():
     import agentops.instrumentation.providers.openai.stream_wrapper
     import agentops.instrumentation.providers.openai.wrappers.chat
@@ -44,6 +50,11 @@ def _patch_new_agentops():
     @no_type_check
     def _handle_chat_attributes_with_tokens(args=None, kwargs=None, return_value=None, **kws):  # type: ignore
         attributes = _original_handle_chat_attributes(args=args, kwargs=kwargs, return_value=return_value, **kws)
+
+        # In some cases, response is a openai._legacy_response.LegacyAPIResponse (e.g., LiteLLM, or LangChain),
+        # This is created by client.with_raw_response.create()
+        return_value = _unwrap_legacy_response(return_value)
+
         if (
             return_value is not None
             and hasattr(return_value, "prompt_token_ids")
@@ -89,20 +100,6 @@ def _patch_new_agentops():
                         [logprob.model_dump() for logprob in first_choice.logprobs.refusal]
                     )
 
-        # For LiteLLM, response is a openai._legacy_response.LegacyAPIResponse
-        if (
-            return_value is not None
-            and hasattr(return_value, "http_response")
-            and return_value.http_response is not None
-            and hasattr(return_value.http_response, "json")
-        ):
-            json_data = return_value.http_response.json()
-            if isinstance(json_data, dict):
-                if json_data.get("prompt_token_ids") is not None:
-                    attributes["prompt_token_ids"] = list(json_data["prompt_token_ids"])
-                if json_data.get("response_token_ids") is not None:
-                    attributes["response_token_ids"] = list(json_data["response_token_ids"][0])
-
         return attributes
 
     agentops.instrumentation.providers.openai.wrappers.chat.handle_chat_attributes = _handle_chat_attributes_with_tokens

agentlightning/verl/entrypoint.py

@@ -2,10 +2,12 @@
 
 # type: ignore
 
+from importlib.metadata import version
 from typing import Any
 
 import hydra
 import ray
+from packaging import version as packaging_version
 from verl.trainer.main_ppo import create_rl_sampler
 from verl.trainer.ppo.reward import load_reward_manager
 
@@ -39,11 +41,17 @@ def run_ppo(
 ) -> None:
     if not ray.is_initialized():
         # this is for local ray cluster
+        try:
+            # verl >= 0.6.0
+            num_cpus = config.ray_kwargs.ray_init.num_cpus
+        except AttributeError:
+            # verl < 0.6.0
+            num_cpus = config.ray_init.num_cpus
         ray.init(
             runtime_env={
                 "env_vars": {"TOKENIZERS_PARALLELISM": "true", "NCCL_DEBUG": "WARN", "VLLM_LOGGING_LEVEL": "WARN"}
             },
-            num_cpus=config.ray_init.num_cpus,
+            num_cpus=num_cpus,
         )
 
     runner = TaskRunner.remote()

docs/algorithm-zoo/verl.md

@@ -32,7 +32,7 @@ pip install agentlightning[verl]
 
 !!! note
 
-    The resource type created by [VERL][agentlightning.algorithm.verl.VERL] is actually a [ProxyLLM][agentlightning.ProxyLLM], a subclass of the [LLM][agentlightning.LLM] type. This object contains a **URL template** provided by [VERL][agentlightning.algorithm.verl.VERL], with placeholders for rollout and attempt IDs. When a rollout begins on the agent side, the framework uses the current `rollout_id` and `attempt_id` to format this template, generating a final, unique endpoint URL. This URL points to [VERL][agentlightning.algorithm.verl.VERL]'s internal proxy, allowing it to intercept and log all traffic for that specific attempt, for tracing and load balancing purposes. For agents created with the `@rollout` decorator, this resolution of the template is handled automatically ("auto-stripped"). Class-based agents will need to manually resolve the `ProxyLLM` using the rollout context.
+    The resource type created by [VERL][agentlightning.algorithm.verl.VERL] is actually a [ProxyLLM][agentlightning.ProxyLLM], a subclass of the [LLM][agentlightning.LLM] type. This object contains a **URL template** provided by [VERL][agentlightning.algorithm.verl.VERL], with placeholders for rollout and attempt IDs. When a rollout begins on the agent side, the framework uses the current `rollout_id` and `attempt_id` to format this template, generating a final, unique endpoint URL. This URL points to [VERL][agentlightning.algorithm.verl.VERL]'s internal proxy, allowing it to intercept and log all traffic for that specific attempt, for tracing and load balancing purposes. For agents created with the `@rollout` decorator, this resolution of the template is handled automatically ("auto-stripped"). Class-based agents will need to manually resolve the [`ProxyLLM`][agentlightning.ProxyLLM] using the rollout context.
 
     ```python
     proxy_llm = resources["main_llm"]

docs/how-to/train-sql-agent.md

@@ -264,7 +264,7 @@ dev_data = pd.read_parquet("data/test_dev_500.parquet").to_dict("records")[:10]
 trainer.dev(agent, dev_dataset=dev_data)
 ```
 
-Run this in a Python session or adapt your script to include a `--dev` flag. Once the spans appear healthy and the rewards are non-zero, switch back to [`trainer.fit(...)`][agentlightning.Trainer.fit] for full RL training.
+Run this in a Python session or adapt your script to include a `--dev` flag. Once the spans appear healthy and the rewards are non-zero, switch back to [`trainer.fit(...)`][agentlightning.Trainer.fit] for full RL training. See the [debugging tutorial](../tutorials/debug.md) for more tips on how to debug the agent.
 
 ## Running the Sample Code
 

docs/tutorials/debug.md

@@ -66,6 +66,48 @@ Swap in an [`AgentOpsTracer`][agentlightning.AgentOpsTracer] instead of [`OtelTr
 
     You can also call [`Runner.step`][agentlightning.Runner.step] to inject ad-hoc rollouts into a running store being used by another algorithm, so that the rollouts can be consumed by the algorithms. This is very recently known as the paradigm of ["online RL"](https://cursor.com/blog/tab-rl). At the moment, no algorithm in the [algorithm zoo](../algorithm-zoo/index.md) consumes externally generated rollouts, but the data flow is available there if you need it.
 
+## Debug with LLM Proxy
+
+If you are dealing with LLM optimization like Reinforcement Learning, we generally recommend using an online stable LLM service for your debugging purposes, like `openai/gpt-4.1-nano`. After the debugging is done, you can switch to a local training endpoint.
+
+However, if you want to use a local LLM features like [getting the token IDs](../deep-dive/serving-llm.md), you can also manually start a local vLLM server by:
+
+```bash
+vllm serve Qwen/Qwen2.5-0.5B-Instruct --port 8080
+```
+
+Then start the LLM proxy via the following script:
+
+```python
+import agentlightning as agl
+
+llm_proxy = agl.LLMProxy(
+    port=8081,
+    model_list=[
+        {
+            "model_name": "Qwen/Qwen2.5-0.5B-Instruct",
+            "litellm_params": {
+                "model": "hosted_vllm/Qwen/Qwen2.5-0.5B-Instruct",
+                "api_base": "http://localhost:8080/v1",
+            },
+        }
+    ],
+    store=agl.InMemoryLightningStore(),
+)
+
+llm_proxy.start()
+time.sleep(1000000)
+```
+
+You can now use the LLM proxy by specifying environment variables:
+
+```bash
+export OPENAI_API_BASE=http://localhost:8081/v1
+export OPENAI_API_KEY=dummy
+```
+
+You might see warnings about `Missing or invalid rollout_id, attempt_id, or sequence_id` in the LLM proxy logs. This is fine because you don't have a rollout and attempt yet when you are debugging. When you started the training, the algorithm will create the rollouts for you and the warnings will go away.
+
 ## Hook into Runner's Lifecycle
 
 [`Runner.run_context`][agentlightning.Runner.run_context] accepts a `hooks` argument so you can observe or augment lifecycle events without editing your agent. Hooks subclass [`Hook`][agentlightning.Hook] and can respond to four asynchronous callbacks: [`on_trace_start`][agentlightning.Hook.on_trace_start], [`on_rollout_start`][agentlightning.Hook.on_rollout_start], [`on_rollout_end`][agentlightning.Hook.on_rollout_end], and [`on_trace_end`][agentlightning.Hook.on_trace_end]. This is useful for:

examples/calc_x/train_calc_agent.py

@@ -40,7 +40,7 @@ import agentlightning as agl
 
 
 def verl_default_config() -> Dict[str, Any]:
-    return {
+    config = {
         "algorithm": {
             "adv_estimator": "grpo",
             "use_kl_in_reward": False,
@@ -58,6 +58,12 @@ def verl_default_config() -> Dict[str, Any]:
                 "multi_turn": {"format": "hermes"},
                 "name": "vllm",
                 "gpu_memory_utilization": 0.6,
+                "engine_kwargs": {
+                    "vllm": {
+                        "enable_auto_tool_choice": True,
+                        "tool_call_parser": "hermes",
+                    }
+                },
             },
             "actor": {
                 "ppo_mini_batch_size": 32,
@@ -96,6 +102,7 @@ def verl_default_config() -> Dict[str, Any]:
             "total_epochs": 2,
         },
     }
+    return config
 
 
 def train(

examples/spider/train_sql_agent.py

@@ -52,6 +52,12 @@ RL_TRAINING_CONFIG: Dict[str, Any] = {
             "multi_turn": {"format": "hermes"},
             "name": "vllm",
             "gpu_memory_utilization": 0.8,
+            "engine_kwargs": {
+                "vllm": {
+                    "enable_auto_tool_choice": True,
+                    "tool_call_parser": "hermes",
+                }
+            },
         },
         "actor": {
             "ppo_mini_batch_size": 32,
@@ -139,6 +145,7 @@ def config_train_llama() -> Dict[str, Any]:
 
     config = deepcopy(RL_TRAINING_CONFIG)
     config["actor_rollout_ref"]["rollout"]["multi_turn"]["format"] = "llama3_json"
+    config["actor_rollout_ref"]["rollout"]["engine_kwargs"]["vllm"]["tool_call_parser"] = "llama3_json"
     config["actor_rollout_ref"]["model"]["path"] = "meta-llama/Llama-3.2-1B-Instruct"
     return config
 

pyproject.toml

@@ -30,8 +30,8 @@ apo = [
 # It's not recommended to use agentlightning[verl] to install VERL and its dependencies.
 # though it's listed here for completeness.
 verl = [
-  "verl>=0.5.0,<0.6.0",
-  "vllm>=0.8.4,<0.11.0",
+  "verl>=0.5.0",
+  "vllm>=0.8.4,<0.11.0", # Due to interface change of ExternalZeroMQDistributedExecutor
 ]
 
 [project.scripts]
@@ -112,12 +112,14 @@ torch-gpu-stable = [
   {include-group = "torch-cu128"},
   "flash-attn>=2.8.3",
   "tensordict>=0.9.1",
+  "verl>=0.6.0",
 ]
 # Use this instead of --group torch-legacy --group torch-gpu
 torch-gpu-legacy = [
   {include-group = "torch-legacy"},
   {include-group = "torch-cu128"},
   "flash-attn==2.8.1",
+  "verl==0.5.0",
 ]
 
 # For the TRL/Unsloth example.