Modernize benchmark from-file

docs/guides/v0.7.0_migration_guide.md

@@ -57,11 +57,11 @@ Load a saved benchmark report and optionally re-export data.
 > [!WARNING]\
 > This command may be changed to be more consistent with the `run` command in the future.
 
-| Option                                                                                                                                                                                         | v0.7.1 equivalent |
-| :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :---------------- |
-| PATH Path to the saved benchmark report file (default: ./benchmarks.                                                                                                                           | Unchanged         |
-| --output-path Directory or file path to save re-exported benchmark results. If a directory, all output formats will be saved there. If a file, the matching format will be saved to that file. | Unchanged         |
-| --output-formats Output formats for benchmark results (e.g., console, json, html, csv).                                                                                                        | Unchanged         |
+| Option                                                                                                                                                                                         | Replacement                                                                                                                          |
+| :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------- |
+| PATH Path to the saved benchmark report file (default: ./benchmarks.                                                                                                                           | Unchanged                                                                                                                            |
+| --output-path Directory or file path to save re-exported benchmark results. If a directory, all output formats will be saved there. If a file, the matching format will be saved to that file. | Removed: use `--output` instead                                                                                                      |
+| --output-formats Output formats for benchmark results (e.g., console, json, html, csv).                                                                                                        | Use `--output` instead to specify the kind and path, for example `--output kind=console` or `--output kind=json,path=benchmark.json` |
 
 ## `guidellm config`
 

pyproject.toml

@@ -141,6 +141,7 @@ test = [
     "pytest-timeout~=2.4.0",
     "pytest-httpx~=0.36.2",
     "respx~=0.23.1",
+    "trio~=0.33.0",
 ]
 env = [
     "pre-commit",

src/guidellm/benchmark/entrypoints.py

@@ -562,21 +562,13 @@ async def benchmark_generative_text(
 
 async def reimport_benchmarks_report(
     file: Path,
-    output_path: Path | None,
-    output_formats: tuple[str, ...] | list[str] = (
-        "console",
-        "json",
-        "html",
-        "csv",
-        "plot",
-    ),
+    outputs: tuple[BenchmarkOutputArgs, ...] | list[dict[str, Any]],
 ) -> tuple[GenerativeBenchmarksReport, dict[str, Any]]:
     """
     Load and re-export an existing benchmarks report in specified output formats.
 
     :param file: Path to the existing benchmark report file to load
-    :param output_path: Base path for output file generation, or None for default
-    :param output_formats: Output format kind strings to resolve and finalize
+    :param outputs: Output format kind strings to resolve and finalize
     :return: Tuple of loaded GenerativeBenchmarksReport and dictionary of output
         results
     """
@@ -591,30 +583,16 @@ async def reimport_benchmarks_report(
             f" loaded {len(report.benchmarks)} benchmark(s)"
         )
 
-    base_path = Path(output_path) if output_path else Path.cwd()
     output_args: list[BenchmarkOutputArgs] = []
-    for fmt in output_formats:
-        data: dict[str, Any] = {"kind": fmt}
-
-        # Temporary workaround: map format name to file extension.
-        # For the plot format, default to .png since .plot is not a valid extension.
-        # This will be removed once from-file config supports typed outputs:
-        # https://github.com/vllm-project/guidellm/pull/923#discussion_r3582378419
-        ext = "png" if fmt == "plot" else fmt
-        if len(output_formats) == 1 and base_path.suffix:
-            data["path"] = base_path
-        elif base_path.suffix:
-            data["path"] = base_path.parent / f"{base_path.stem}.{ext}"
-        else:
-            data["path"] = base_path / f"benchmarks.{ext}"
-        output_args.append(BenchmarkOutputArgs.model_validate(data))
+    for fmt in outputs:
+        output_args.append(BenchmarkOutputArgs.model_validate(fmt))
 
-    output_format_results: dict[str, Any] = {}
+    output_results: dict[str, Any] = {}
     for args in output_args:
         output = GenerativeBenchmarkerOutput.resolve(args)
-        output_format_results[args.kind] = await output.finalize(report)
+        output_results[args.kind] = await output.finalize(report)
 
-    for key, value in output_format_results.items():
+    for key, value in output_results.items():
         console.print_update(title=f"  {key:<8}: {value}", status="debug")
 
-    return report, output_format_results
+    return report, output_results

src/guidellm/cli/benchmark/from_file.py

@@ -8,6 +8,8 @@
 import click
 
 from guidellm.benchmark import reimport_benchmarks_report
+from guidellm.benchmark.schemas import BenchmarkOutputArgs
+from guidellm.utils.click_pydantic import registry_option
 
 __all__ = ["from_file"]
 
@@ -24,24 +26,12 @@
     type=click.Path(file_okay=True, dir_okay=False, exists=True),
     default=Path.cwd() / "benchmarks.json",
 )
-@click.option(
-    "--output-path",
-    type=click.Path(),
-    default=Path.cwd(),
-    help=(
-        "Directory or file path where the re-exported benchmark results will be saved. "
-        "If a directory, default filenames are used. "
-        "If a file path, the suffix is used directly when generating a "
-        "single format, or replaced by each format's extension when generating "
-        "multiple formats."
-    ),
-)
-@click.option(
-    "--output-formats",
+@registry_option(
+    "--output",
+    "outputs",
+    registry=BenchmarkOutputArgs,
     multiple=True,
-    type=str,
-    default=("console", "json"),  # ("console", "json", "html", "csv")
-    help="Output formats for benchmark results (e.g., console, json, html, csv).",
+    default=[{"kind": "console"}, {"kind": "json"}, {"kind": "html"}, {"kind": "csv"}],
 )
-def from_file(path, output_path, output_formats):
-    asyncio.run(reimport_benchmarks_report(path, output_path, output_formats))
+def from_file(path, outputs):
+    asyncio.run(reimport_benchmarks_report(path, outputs))

tests/unit/benchmark/test_plot_output.py

@@ -303,16 +303,13 @@ async def test_reimport_benchmarks_report_custom_extension_plot(
     report_file.write_text(minimal_report.model_dump_json(), encoding="utf-8")
 
     # Re-import and save specifically to a named .pdf file
-    target_path = tmp_path / "my_report.pdf"
+    expected_pdf = tmp_path / "my_report.pdf"
 
     await reimport_benchmarks_report(
-        file=report_file,
-        output_path=target_path,
-        output_formats=["plot"],
+        file=report_file, outputs=[{"kind": "plot", "path": expected_pdf}]
     )
 
     # Assert that it resolved and created the correct file: my_report.pdf
-    expected_pdf = tmp_path / "my_report.pdf"
     assert expected_pdf.exists()
     assert expected_pdf.is_file()
     assert expected_pdf.stat().st_size > 0
@@ -332,17 +329,17 @@ async def test_reimport_benchmarks_report_multiple_sibling_file_path(
     report_file = tmp_path / "report.json"
     report_file.write_text(minimal_report.model_dump_json(), encoding="utf-8")
 
-    target_path = tmp_path / "my_report.aaf"
+    expected_json = tmp_path / "my_report.json"
+    expected_png = tmp_path / "my_report.png"
 
     await reimport_benchmarks_report(
         file=report_file,
-        output_path=target_path,
-        output_formats=["json", "plot"],
+        outputs=[
+            {"kind": "json", "path": expected_json},
+            {"kind": "plot", "path": expected_png},
+        ],
     )
 
-    expected_json = tmp_path / "my_report.json"
-    expected_png = tmp_path / "my_report.png"
-
     assert expected_json.exists()
     assert expected_json.is_file()
     assert expected_png.exists()