[FIX] add guidellm preflight check and pin vllm to <=0.19

auto_tune_vllm/benchmarks/providers.py

@@ -199,27 +199,20 @@ def start_benchmark(
         # Build GuideLLM command
         cmd = self._build_guidellm_command(model_url, config, self._results_file)
 
-        # Validate binary and basic inputs
-        import shutil
-
-        if shutil.which("guidellm") is None:
-            raise RuntimeError(
-                "GuideLLM CLI not found on PATH. "
-                "Please install or provide the full path."
-            )
+        # Validate benchmark target inputs
         if not (model_url.startswith("http://") or model_url.startswith("https://")):
             raise ValueError(f"Invalid model_url: {model_url!r} (expected http/https)")
 
+        env = os.environ.copy()
+        env["GUIDELLM__LOGGING__CONSOLE_LOG_LEVEL"] = config.logging_level
+
         # Run GuideLLM
         self._logger.info(f"Running: {' '.join(cmd)}")
         self._logger.info(f"Results will be saved to: {self._results_file}")
 
         # Use Popen so we can terminate if vLLM dies
         # start_new_session=True puts it in its own process group for clean
         # termination
-        env = os.environ.copy()
-        env["GUIDELLM__LOGGING__CONSOLE_LOG_LEVEL"] = config.logging_level
-
         self._process = subprocess.Popen(
             cmd,
             stdout=subprocess.PIPE,

auto_tune_vllm/cli/main.py

@@ -1,7 +1,9 @@
 """Command-line interface for auto-tune-vllm."""
 
 import logging
+import os
 import shutil
+import subprocess
 import sys
 from pathlib import Path
 from typing import Optional
@@ -109,6 +111,41 @@ def _display_log_viewing_instructions(config: StudyConfig):
         )
 
 
+def _run_guidellm_benchmark_help_preflight_for_optimize(config: StudyConfig) -> None:
+    """Once per `optimize` run: verify `guidellm benchmark --help` succeeds (GuideLLM only)."""
+    if config.benchmark.benchmark_type != "guidellm":
+        return
+
+    console.print("[blue]Checking GuideLLM CLI (guidellm benchmark --help)...[/blue]")
+    if shutil.which("guidellm") is None:
+        raise RuntimeError(
+            "GuideLLM CLI not found on PATH. "
+            "Install GuideLLM or ensure `guidellm` is available."
+        )
+    env = os.environ.copy()
+    env["GUIDELLM__LOGGING__CONSOLE_LOG_LEVEL"] = config.benchmark.logging_level
+    try:
+        result = subprocess.run(
+            ["guidellm", "benchmark", "--help"],
+            capture_output=True,
+            text=True,
+            timeout=120,
+            env=env,
+        )
+    except subprocess.TimeoutExpired as exc:
+        raise RuntimeError(
+            "GuideLLM CLI check timed out while running 'guidellm benchmark --help'."
+        ) from exc
+
+    if result.returncode != 0:
+        err = (result.stderr or result.stdout or "").strip()
+        raise RuntimeError(
+            f"GuideLLM CLI check failed (exit code {result.returncode}) "
+            f"for 'guidellm benchmark --help'. Output: {err}"
+        )
+    console.print("[green]✓ GuideLLM CLI OK[/green]")
+
+
 @app.command("optimize")
 def optimize_command(
     config: str = typer.Option(..., "--config", "-c", help="Study configuration file"),
@@ -405,6 +442,8 @@ def run_optimization_sync(
             "so startup sampling would consume the full trial budget. "
             f"n_startup_trials is now {config.optimization.n_startup_trials}.[/yellow]"
         )
+    _run_guidellm_benchmark_help_preflight_for_optimize(config)
+
     # Create study controller (uses config with possibly updated sampler/n_trials)
     controller = StudyController.create_from_config(
         backend, config, create_db=create_db

pyproject.toml

@@ -26,7 +26,7 @@ requires-python = ">=3.10"
 dependencies = [
     "optuna>=3.0.0",
     "optuna-integration[botorch]>=4.0.0",
-    "vllm>=0.11.0",
+    "vllm>=0.11.0,<=0.19",
     "guidellm>=0.1.0",
     "pyyaml>=6.0",
     "pydantic>=2.0.0",