diff --git a/docs/guides/v0.7.0_migration_guide.md b/docs/guides/v0.7.0_migration_guide.md index 2d2469ffe..c46519644 100644 --- a/docs/guides/v0.7.0_migration_guide.md +++ b/docs/guides/v0.7.0_migration_guide.md @@ -57,11 +57,11 @@ Load a saved benchmark report and optionally re-export data. > [!WARNING]\ > This command may be changed to be more consistent with the `run` command in the future. -| Option | v0.7.1 equivalent | -| :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :---------------- | -| PATH Path to the saved benchmark report file (default: ./benchmarks. | Unchanged | -| --output-path Directory or file path to save re-exported benchmark results. If a directory, all output formats will be saved there. If a file, the matching format will be saved to that file. | Unchanged | -| --output-formats Output formats for benchmark results (e.g., console, json, html, csv). | Unchanged | +| Option | Replacement | +| :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------- | +| PATH Path to the saved benchmark report file (default: ./benchmarks. | Unchanged | +| --output-path Directory or file path to save re-exported benchmark results. If a directory, all output formats will be saved there. If a file, the matching format will be saved to that file. | Removed: use `--output` instead | +| --output-formats Output formats for benchmark results (e.g., console, json, html, csv). | Use `--output` instead to specify the kind and path, for example `--output kind=console` or `--output kind=json,path=benchmark.json` | ## `guidellm config` diff --git a/pyproject.toml b/pyproject.toml index 160961e12..84ea25d24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -141,6 +141,7 @@ test = [ "pytest-timeout~=2.4.0", "pytest-httpx~=0.36.2", "respx~=0.23.1", + "trio~=0.33.0", ] env = [ "pre-commit", diff --git a/src/guidellm/benchmark/entrypoints.py b/src/guidellm/benchmark/entrypoints.py index 7afd04587..460d752cb 100644 --- a/src/guidellm/benchmark/entrypoints.py +++ b/src/guidellm/benchmark/entrypoints.py @@ -562,21 +562,13 @@ async def benchmark_generative_text( async def reimport_benchmarks_report( file: Path, - output_path: Path | None, - output_formats: tuple[str, ...] | list[str] = ( - "console", - "json", - "html", - "csv", - "plot", - ), + outputs: tuple[BenchmarkOutputArgs, ...] | list[dict[str, Any]], ) -> tuple[GenerativeBenchmarksReport, dict[str, Any]]: """ Load and re-export an existing benchmarks report in specified output formats. :param file: Path to the existing benchmark report file to load - :param output_path: Base path for output file generation, or None for default - :param output_formats: Output format kind strings to resolve and finalize + :param outputs: Output format kind strings to resolve and finalize :return: Tuple of loaded GenerativeBenchmarksReport and dictionary of output results """ @@ -591,30 +583,16 @@ async def reimport_benchmarks_report( f" loaded {len(report.benchmarks)} benchmark(s)" ) - base_path = Path(output_path) if output_path else Path.cwd() output_args: list[BenchmarkOutputArgs] = [] - for fmt in output_formats: - data: dict[str, Any] = {"kind": fmt} - - # Temporary workaround: map format name to file extension. - # For the plot format, default to .png since .plot is not a valid extension. - # This will be removed once from-file config supports typed outputs: - # https://github.com/vllm-project/guidellm/pull/923#discussion_r3582378419 - ext = "png" if fmt == "plot" else fmt - if len(output_formats) == 1 and base_path.suffix: - data["path"] = base_path - elif base_path.suffix: - data["path"] = base_path.parent / f"{base_path.stem}.{ext}" - else: - data["path"] = base_path / f"benchmarks.{ext}" - output_args.append(BenchmarkOutputArgs.model_validate(data)) + for fmt in outputs: + output_args.append(BenchmarkOutputArgs.model_validate(fmt)) - output_format_results: dict[str, Any] = {} + output_results: dict[str, Any] = {} for args in output_args: output = GenerativeBenchmarkerOutput.resolve(args) - output_format_results[args.kind] = await output.finalize(report) + output_results[args.kind] = await output.finalize(report) - for key, value in output_format_results.items(): + for key, value in output_results.items(): console.print_update(title=f" {key:<8}: {value}", status="debug") - return report, output_format_results + return report, output_results diff --git a/src/guidellm/cli/benchmark/from_file.py b/src/guidellm/cli/benchmark/from_file.py index 1bfbb3845..569d6534a 100644 --- a/src/guidellm/cli/benchmark/from_file.py +++ b/src/guidellm/cli/benchmark/from_file.py @@ -8,6 +8,8 @@ import click from guidellm.benchmark import reimport_benchmarks_report +from guidellm.benchmark.schemas import BenchmarkOutputArgs +from guidellm.utils.click_pydantic import registry_option __all__ = ["from_file"] @@ -24,24 +26,12 @@ type=click.Path(file_okay=True, dir_okay=False, exists=True), default=Path.cwd() / "benchmarks.json", ) -@click.option( - "--output-path", - type=click.Path(), - default=Path.cwd(), - help=( - "Directory or file path where the re-exported benchmark results will be saved. " - "If a directory, default filenames are used. " - "If a file path, the suffix is used directly when generating a " - "single format, or replaced by each format's extension when generating " - "multiple formats." - ), -) -@click.option( - "--output-formats", +@registry_option( + "--output", + "outputs", + registry=BenchmarkOutputArgs, multiple=True, - type=str, - default=("console", "json"), # ("console", "json", "html", "csv") - help="Output formats for benchmark results (e.g., console, json, html, csv).", + default=[{"kind": "console"}, {"kind": "json"}, {"kind": "html"}, {"kind": "csv"}], ) -def from_file(path, output_path, output_formats): - asyncio.run(reimport_benchmarks_report(path, output_path, output_formats)) +def from_file(path, outputs): + asyncio.run(reimport_benchmarks_report(path, outputs)) diff --git a/tests/unit/benchmark/test_plot_output.py b/tests/unit/benchmark/test_plot_output.py index 7fcaaad77..1f05e9bad 100644 --- a/tests/unit/benchmark/test_plot_output.py +++ b/tests/unit/benchmark/test_plot_output.py @@ -303,16 +303,13 @@ async def test_reimport_benchmarks_report_custom_extension_plot( report_file.write_text(minimal_report.model_dump_json(), encoding="utf-8") # Re-import and save specifically to a named .pdf file - target_path = tmp_path / "my_report.pdf" + expected_pdf = tmp_path / "my_report.pdf" await reimport_benchmarks_report( - file=report_file, - output_path=target_path, - output_formats=["plot"], + file=report_file, outputs=[{"kind": "plot", "path": expected_pdf}] ) # Assert that it resolved and created the correct file: my_report.pdf - expected_pdf = tmp_path / "my_report.pdf" assert expected_pdf.exists() assert expected_pdf.is_file() assert expected_pdf.stat().st_size > 0 @@ -332,17 +329,17 @@ async def test_reimport_benchmarks_report_multiple_sibling_file_path( report_file = tmp_path / "report.json" report_file.write_text(minimal_report.model_dump_json(), encoding="utf-8") - target_path = tmp_path / "my_report.aaf" + expected_json = tmp_path / "my_report.json" + expected_png = tmp_path / "my_report.png" await reimport_benchmarks_report( file=report_file, - output_path=target_path, - output_formats=["json", "plot"], + outputs=[ + {"kind": "json", "path": expected_json}, + {"kind": "plot", "path": expected_png}, + ], ) - expected_json = tmp_path / "my_report.json" - expected_png = tmp_path / "my_report.png" - assert expected_json.exists() assert expected_json.is_file() assert expected_png.exists() diff --git a/tests/unit/entrypoints/assets/benchmarks_stripped.json b/tests/unit/entrypoints/assets/benchmarks_stripped.json deleted file mode 100644 index a95d2880a..000000000 --- a/tests/unit/entrypoints/assets/benchmarks_stripped.json +++ /dev/null @@ -1 +0,0 @@ -{"benchmarks": [{"type_": "generative_benchmark", "id_": "97ece514-8717-412f-9dba-2b42bcd9866f", "run_id": "93e36b31-b454-471d-ba62-6b2671585485", "args": {"profile": {"type_": "sweep", "completed_strategies": 10, "measured_rates": [1.5481806532737452], "measured_concurrencies": [0.9977627456483604], "max_concurrency": null, "strategy_type": "constant", "rate": -1.0, "initial_burst": true, "random_seed": 42, "sweep_size": 10, "rate_type": "constant", "strategy_types": ["synchronous", "throughput", "constant", "constant", "constant", "constant", "constant", "constant", "constant", "constant"]}, "strategy_index": 0, "strategy": {"type_": "synchronous"}, "max_number": null, "max_duration": 30.0, "warmup_number": null, "warmup_duration": null, "cooldown_number": null, "cooldown_duration": null}, "run_stats": {"start_time": 1749157168.054225, "end_time": 1749157198.213826, "requests_made": {"successful": 1, "errored": 0, "incomplete": 0, "total": 1}, "queued_time_avg": 0.631589580089488, "scheduled_time_delay_avg": 3.784260851271609e-06, "scheduled_time_sleep_avg": 0.0, "worker_start_delay_avg": 2.8021792148021943e-05, "worker_time_avg": 0.6373953819274902, "worker_start_time_targeted_delay_avg": 0.6319031715393066, "request_start_time_delay_avg": 0.316034068452551, "request_start_time_targeted_delay_avg": 0.6319856542222043, "request_time_delay_avg": 0.00029866238857837433, "request_time_avg": 0.6370967195389119}, "worker": {"type_": "generative_requests_worker", "backend_type": "openai_http", "backend_target": "example_target", "backend_model": "example_model", "backend_info": {"max_output_tokens": 16384, "timeout": 300, "http2": true, "authorization": false, "organization": null, "project": null, "text_completions_path": "/v1/completions", "chat_completions_path": "/v1/chat/completions"}}, "request_loader": {"type_": "generative_request_loader", "data": "prompt_tokens=256,output_tokens=128", "data_args": null, "processor": "example_processor", "processor_args": null}, "extras": {}, "metrics": {"requests_per_second": {"successful": {"mean": 1.5481806532737452, "median": 1.5530116578512305, "mode": 1.555484186315253, "variance": 0.0003352629331303757, "std_dev": 0.01831018659463567, "min": 1.4509899157628907, "max": 1.5597664461806156, "count": 45, "total_sum": 69.6707872953874, "percentiles": {"p001": 1.4509899157628907, "p01": 1.4509899157628907, "p05": 1.5190957942495127, "p10": 1.5377883923356668, "p25": 1.5483918601985445, "p75": 1.5567531615313124, "p90": 1.5583715343236735, "p95": 1.5590938878953722, "p99": 1.5597664461806156, "p999": 1.5597664461806156}, "cumulative_distribution_function": null}, "errored": {"mean": 0.0, "median": 0.0, "mode": 0.0, "variance": 0.0, "std_dev": 0.0, "min": 0.0, "max": 0.0, "count": 0, "total_sum": 0.0, "percentiles": {"p001": 0.0, "p01": 0.0, "p05": 0.0, "p10": 0.0, "p25": 0.0, "p75": 0.0, "p90": 0.0, "p95": 0.0, "p99": 0.0, "p999": 0.0}, "cumulative_distribution_function": null}, "incomplete": {"mean": 0.0, "median": 0.0, "mode": 0.0, "variance": 0.0, "std_dev": 0.0, "min": 0.0, "max": 0.0, "count": 0, "total_sum": 0.0, "percentiles": {"p001": 0.0, "p01": 0.0, "p05": 0.0, "p10": 0.0, "p25": 0.0, "p75": 0.0, "p90": 0.0, "p95": 0.0, "p99": 0.0, "p999": 0.0}, "cumulative_distribution_function": null}, "total": {"mean": 1.5668128271815418, "median": 1.5530312090734288, "mode": 1.555484186315253, "variance": 0.036536424510388923, "std_dev": 0.19114503527528232, "min": 1.4509899157628907, "max": 3.509921881864626, "count": 46, "total_sum": 73.18070917725203, "percentiles": {"p001": 1.4509899157628907, "p01": 1.4509899157628907, "p05": 1.5190957942495127, "p10": 1.5377883923356668, "p25": 1.5483918601985445, "p75": 1.5567531615313124, "p90": 1.5583715343236735, "p95": 1.5591048992639953, "p99": 1.5597664461806156, "p999": 3.509921881864626}, "cumulative_distribution_function": null}}, "request_concurrency": {"successful": {"mean": 0.9977627456483604, "median": 1.0, "mode": 1.0, "variance": 0.002232249044605607, "std_dev": 0.047246682895263736, "min": 0.0, "max": 1.0, "count": 2, "total_sum": 1.0, "percentiles": {"p001": 0.0, "p01": 1.0, "p05": 1.0, "p10": 1.0, "p25": 1.0, "p75": 1.0, "p90": 1.0, "p95": 1.0, "p99": 1.0, "p999": 1.0}, "cumulative_distribution_function": null}, "errored": {"mean": 0.0, "median": 0.0, "mode": 0.0, "variance": 0.0, "std_dev": 0.0, "min": 0.0, "max": 0.0, "count": 0, "total_sum": 0.0, "percentiles": {"p001": 0.0, "p01": 0.0, "p05": 0.0, "p10": 0.0, "p25": 0.0, "p75": 0.0, "p90": 0.0, "p95": 0.0, "p99": 0.0, "p999": 0.0}, "cumulative_distribution_function": null}, "incomplete": {"mean": 1.0, "median": 1.0, "mode": 1.0, "variance": 0.0, "std_dev": 0.0, "min": 1.0, "max": 1.0, "count": 1, "total_sum": 1.0, "percentiles": {"p001": 1.0, "p01": 1.0, "p05": 1.0, "p10": 1.0, "p25": 1.0, "p75": 1.0, "p90": 1.0, "p95": 1.0, "p99": 1.0, "p999": 1.0}, "cumulative_distribution_function": null}, "total": {"mean": 0.9977433642674269, "median": 1.0, "mode": 1.0, "variance": 0.002251543327743578, "std_dev": 0.047450430216633206, "min": 0.0, "max": 1.0, "count": 2, "total_sum": 1.0, "percentiles": {"p001": 0.0, "p01": 1.0, "p05": 1.0, "p10": 1.0, "p25": 1.0, "p75": 1.0, "p90": 1.0, "p95": 1.0, "p99": 1.0, "p999": 1.0}, "cumulative_distribution_function": null}}, "request_latency": {"successful": {"mean": 0.6444743664368339, "median": 0.6424565315246582, "mode": 0.6395885944366455, "variance": 6.414585873782315e-05, "std_dev": 0.008009110982988258, "min": 0.6395885944366455, "max": 0.6891846656799316, "count": 46, "total_sum": 29.64582085609436, "percentiles": {"p001": 0.6395885944366455, "p01": 0.6395885944366455, "p05": 0.6399857997894287, "p10": 0.6403069496154785, "p25": 0.6409540176391602, "p75": 0.644390344619751, "p90": 0.6488735675811768, "p95": 0.656728982925415, "p99": 0.6891846656799316, "p999": 0.6891846656799316}, "cumulative_distribution_function": null}, "errored": {"mean": 0.0, "median": 0.0, "mode": 0.0, "variance": 0.0, "std_dev": 0.0, "min": 0.0, "max": 0.0, "count": 0, "total_sum": 0.0, "percentiles": {"p001": 0.0, "p01": 0.0, "p05": 0.0, "p10": 0.0, "p25": 0.0, "p75": 0.0, "p90": 0.0, "p95": 0.0, "p99": 0.0, "p999": 0.0}, "cumulative_distribution_function": null}, "incomplete": {"mean": 0.2836878299713135, "median": 0.2836878299713135, "mode": 0.2836878299713135, "variance": 0.0, "std_dev": 0.0, "min": 0.2836878299713135, "max": 0.2836878299713135, "count": 1, "total_sum": 0.2836878299713135, "percentiles": {"p001": 0.2836878299713135, "p01": 0.2836878299713135, "p05": 0.2836878299713135, "p10": 0.2836878299713135, "p25": 0.2836878299713135, "p75": 0.2836878299713135, "p90": 0.2836878299713135, "p95": 0.2836878299713135, "p99": 0.2836878299713135, "p999": 0.2836878299713135}, "cumulative_distribution_function": null}, "total": {"mean": 0.6367980571503334, "median": 0.642310380935669, "mode": 0.2836878299713135, "variance": 0.0027733643692853522, "std_dev": 0.05266274175624881, "min": 0.2836878299713135, "max": 0.6891846656799316, "count": 47, "total_sum": 29.929508686065674, "percentiles": {"p001": 0.2836878299713135, "p01": 0.2836878299713135, "p05": 0.6398613452911377, "p10": 0.6402454376220703, "p25": 0.640899658203125, "p75": 0.644390344619751, "p90": 0.6488735675811768, "p95": 0.656728982925415, "p99": 0.6891846656799316, "p999": 0.6891846656799316}, "cumulative_distribution_function": null}}, "prompt_token_count": {"successful": {"mean": 257.1086956521739, "median": 257.0, "mode": 257.0, "variance": 0.14035916824196598, "std_dev": 0.37464538999161057, "min": 257.0, "max": 259.0, "count": 46, "total_sum": 11827.0, "percentiles": {"p001": 257.0, "p01": 257.0, "p05": 257.0, "p10": 257.0, "p25": 257.0, "p75": 257.0, "p90": 257.0, "p95": 258.0, "p99": 259.0, "p999": 259.0}, "cumulative_distribution_function": null}, "errored": {"mean": 0.0, "median": 0.0, "mode": 0.0, "variance": 0.0, "std_dev": 0.0, "min": 0.0, "max": 0.0, "count": 0, "total_sum": 0.0, "percentiles": {"p001": 0.0, "p01": 0.0, "p05": 0.0, "p10": 0.0, "p25": 0.0, "p75": 0.0, "p90": 0.0, "p95": 0.0, "p99": 0.0, "p999": 0.0}, "cumulative_distribution_function": null}, "incomplete": {"mean": 256.0, "median": 256.0, "mode": 256.0, "variance": 0.0, "std_dev": 0.0, "min": 256.0, "max": 256.0, "count": 1, "total_sum": 256.0, "percentiles": {"p001": 256.0, "p01": 256.0, "p05": 256.0, "p10": 256.0, "p25": 256.0, "p75": 256.0, "p90": 256.0, "p95": 256.0, "p99": 256.0, "p999": 256.0}, "cumulative_distribution_function": null}, "total": {"mean": 257.0851063829787, "median": 257.0, "mode": 256.0, "variance": 0.16296966953372566, "std_dev": 0.40369502044702715, "min": 256.0, "max": 259.0, "count": 47, "total_sum": 12083.0, "percentiles": {"p001": 256.0, "p01": 256.0, "p05": 257.0, "p10": 257.0, "p25": 257.0, "p75": 257.0, "p90": 257.0, "p95": 258.0, "p99": 259.0, "p999": 259.0}, "cumulative_distribution_function": null}}, "output_token_count": {"successful": {"mean": 127.99999999999999, "median": 128.0, "mode": 128.0, "variance": 2.01948391736579e-28, "std_dev": 1.4210854715202002e-14, "min": 128.0, "max": 128.0, "count": 46, "total_sum": 5888.0, "percentiles": {"p001": 128.0, "p01": 128.0, "p05": 128.0, "p10": 128.0, "p25": 128.0, "p75": 128.0, "p90": 128.0, "p95": 128.0, "p99": 128.0, "p999": 128.0}, "cumulative_distribution_function": null}, "errored": {"mean": 0.0, "median": 0.0, "mode": 0.0, "variance": 0.0, "std_dev": 0.0, "min": 0.0, "max": 0.0, "count": 0, "total_sum": 0.0, "percentiles": {"p001": 0.0, "p01": 0.0, "p05": 0.0, "p10": 0.0, "p25": 0.0, "p75": 0.0, "p90": 0.0, "p95": 0.0, "p99": 0.0, "p999": 0.0}, "cumulative_distribution_function": null}, "incomplete": {"mean": 0.0, "median": 0.0, "mode": 0.0, "variance": 0.0, "std_dev": 0.0, "min": 0.0, "max": 0.0, "count": 0, "total_sum": 0.0, "percentiles": {"p001": 0.0, "p01": 0.0, "p05": 0.0, "p10": 0.0, "p25": 0.0, "p75": 0.0, "p90": 0.0, "p95": 0.0, "p99": 0.0, "p999": 0.0}, "cumulative_distribution_function": null}, "total": {"mean": 126.44680851063832, "median": 128.0, "mode": 55.0, "variance": 110.97057492077867, "std_dev": 10.534257207832866, "min": 55.0, "max": 128.0, "count": 47, "total_sum": 5943.0, "percentiles": {"p001": 55.0, "p01": 55.0, "p05": 128.0, "p10": 128.0, "p25": 128.0, "p75": 128.0, "p90": 128.0, "p95": 128.0, "p99": 128.0, "p999": 128.0}, "cumulative_distribution_function": null}}, "time_to_first_token_ms": {"successful": {"mean": 16.792535781860348, "median": 16.38054847717285, "mode": 15.790939331054688, "variance": 1.2776652847210441, "std_dev": 1.1303385708366516, "min": 15.790939331054688, "max": 21.281957626342773, "count": 46, "total_sum": 772.4566459655762, "percentiles": {"p001": 15.790939331054688, "p01": 15.790939331054688, "p05": 15.971660614013672, "p10": 16.034841537475586, "p25": 16.111373901367188, "p75": 16.840696334838867, "p90": 18.505334854125977, "p95": 19.00935173034668, "p99": 21.281957626342773, "p999": 21.281957626342773}, "cumulative_distribution_function": null}, "errored": {"mean": 0.0, "median": 0.0, "mode": 0.0, "variance": 0.0, "std_dev": 0.0, "min": 0.0, "max": 0.0, "count": 0, "total_sum": 0.0, "percentiles": {"p001": 0.0, "p01": 0.0, "p05": 0.0, "p10": 0.0, "p25": 0.0, "p75": 0.0, "p90": 0.0, "p95": 0.0, "p99": 0.0, "p999": 0.0}, "cumulative_distribution_function": null}, "incomplete": {"mean": 0.0, "median": 0.0, "mode": 0.0, "variance": 0.0, "std_dev": 0.0, "min": 0.0, "max": 0.0, "count": 0, "total_sum": 0.0, "percentiles": {"p001": 0.0, "p01": 0.0, "p05": 0.0, "p10": 0.0, "p25": 0.0, "p75": 0.0, "p90": 0.0, "p95": 0.0, "p99": 0.0, "p999": 0.0}, "cumulative_distribution_function": null}, "total": {"mean": 16.777170465347616, "median": 16.371726989746094, "mode": 15.790939331054688, "variance": 1.2613411927317046, "std_dev": 1.1230944718641014, "min": 15.790939331054688, "max": 21.281957626342773, "count": 47, "total_sum": 788.5270118713379, "percentiles": {"p001": 15.790939331054688, "p01": 15.790939331054688, "p05": 15.971660614013672, "p10": 16.034841537475586, "p25": 16.100645065307617, "p75": 16.840696334838867, "p90": 18.505334854125977, "p95": 19.00935173034668, "p99": 21.281957626342773, "p999": 21.281957626342773}, "cumulative_distribution_function": null}}, "time_per_output_token_ms": {"successful": {"mean": 4.90300272307966, "median": 4.885653033852577, "mode": 4.870360717177391, "variance": 0.003163643010108571, "std_dev": 0.05624627107736628, "min": 4.870360717177391, "max": 5.217265337705612, "count": 46, "total_sum": 225.5381252616644, "percentiles": {"p001": 4.870360717177391, "p01": 4.870360717177391, "p05": 4.8728808760643005, "p10": 4.873953759670258, "p25": 4.876237362623215, "p75": 4.904214292764664, "p90": 4.934689030051231, "p95": 4.993332549929619, "p99": 5.217265337705612, "p999": 5.217265337705612}, "cumulative_distribution_function": null}, "errored": {"mean": 0.0, "median": 0.0, "mode": 0.0, "variance": 0.0, "std_dev": 0.0, "min": 0.0, "max": 0.0, "count": 0, "total_sum": 0.0, "percentiles": {"p001": 0.0, "p01": 0.0, "p05": 0.0, "p10": 0.0, "p25": 0.0, "p75": 0.0, "p90": 0.0, "p95": 0.0, "p99": 0.0, "p999": 0.0}, "cumulative_distribution_function": null}, "incomplete": {"mean": 0.0, "median": 0.0, "mode": 0.0, "variance": 0.0, "std_dev": 0.0, "min": 0.0, "max": 0.0, "count": 0, "total_sum": 0.0, "percentiles": {"p001": 0.0, "p01": 0.0, "p05": 0.0, "p10": 0.0, "p25": 0.0, "p75": 0.0, "p90": 0.0, "p95": 0.0, "p99": 0.0, "p999": 0.0}, "cumulative_distribution_function": null}, "total": {"mean": 4.9022222114856975, "median": 4.882922396063805, "mode": 4.870360717177391, "variance": 0.003199582258516055, "std_dev": 0.05656485002646127, "min": 4.81866489757191, "max": 5.217265337705612, "count": 47, "total_sum": 230.3567901592363, "percentiles": {"p001": 4.81866489757191, "p01": 4.870360717177391, "p05": 4.872731864452362, "p10": 4.873953759670258, "p25": 4.876237362623215, "p75": 4.904214292764664, "p90": 4.934689030051231, "p95": 4.993332549929619, "p99": 5.217265337705612, "p999": 5.217265337705612}, "cumulative_distribution_function": null}}, "inter_token_latency_ms": {"successful": {"mean": 4.941609043733832, "median": 4.9241227427805505, "mode": 4.90871001416304, "variance": 0.003213660306132974, "std_dev": 0.056689155101597465, "min": 4.90871001416304, "max": 5.258346167136365, "count": 46, "total_sum": 227.31401601175622, "percentiles": {"p001": 4.90871001416304, "p01": 4.90871001416304, "p05": 4.911250016820713, "p10": 4.9123313483290785, "p25": 4.91463293240765, "p75": 4.9428301533376136, "p90": 4.973544849185493, "p95": 5.032650129062923, "p99": 5.258346167136365, "p999": 5.258346167136365}, "cumulative_distribution_function": null}, "errored": {"mean": 0.0, "median": 0.0, "mode": 0.0, "variance": 0.0, "std_dev": 0.0, "min": 0.0, "max": 0.0, "count": 0, "total_sum": 0.0, "percentiles": {"p001": 0.0, "p01": 0.0, "p05": 0.0, "p10": 0.0, "p25": 0.0, "p75": 0.0, "p90": 0.0, "p95": 0.0, "p99": 0.0, "p999": 0.0}, "cumulative_distribution_function": null}, "incomplete": {"mean": 0.0, "median": 0.0, "mode": 0.0, "variance": 0.0, "std_dev": 0.0, "min": 0.0, "max": 0.0, "count": 0, "total_sum": 0.0, "percentiles": {"p001": 0.0, "p01": 0.0, "p05": 0.0, "p10": 0.0, "p25": 0.0, "p75": 0.0, "p90": 0.0, "p95": 0.0, "p99": 0.0, "p999": 0.0}, "cumulative_distribution_function": null}, "total": {"mean": 4.9413003057767115, "median": 4.921370603906826, "mode": 4.90871001416304, "variance": 0.003194539306669541, "std_dev": 0.056520255720135776, "min": 4.9078994327121315, "max": 5.258346167136365, "count": 47, "total_sum": 232.22191544446835, "percentiles": {"p001": 4.9078994327121315, "p01": 4.90871001416304, "p05": 4.911099831888995, "p10": 4.9123313483290785, "p25": 4.91463293240765, "p75": 4.9428301533376136, "p90": 4.973544849185493, "p95": 5.032650129062923, "p99": 5.258346167136365, "p999": 5.258346167136365}, "cumulative_distribution_function": null}}, "output_tokens_per_second": {"successful": {"mean": 198.13346751788123, "median": 203.04516628745705, "mode": 203.5378269520066, "variance": 613.9948900522365, "std_dev": 24.778920276158857, "min": 0.0, "max": 203.69598368219124, "count": 122, "total_sum": 17849.590625912137, "percentiles": {"p001": 46.71289356157213, "p01": 55.502236337170835, "p05": 190.14888022486173, "p10": 200.69400449782287, "p25": 202.23259402121505, "p75": 203.42923658938793, "p90": 203.5378269520066, "p95": 203.58722454130668, "p99": 203.6860916860917, "p999": 203.69598368219124}, "cumulative_distribution_function": null}, "errored": {"mean": 0.0, "median": 0.0, "mode": 0.0, "variance": 0.0, "std_dev": 0.0, "min": 0.0, "max": 0.0, "count": 0, "total_sum": 0.0, "percentiles": {"p001": 0.0, "p01": 0.0, "p05": 0.0, "p10": 0.0, "p25": 0.0, "p75": 0.0, "p90": 0.0, "p95": 0.0, "p99": 0.0, "p999": 0.0}, "cumulative_distribution_function": null}, "incomplete": {"mean": 0.0, "median": 0.0, "mode": 0.0, "variance": 0.0, "std_dev": 0.0, "min": 0.0, "max": 0.0, "count": 0, "total_sum": 0.0, "percentiles": {"p001": 0.0, "p01": 0.0, "p05": 0.0, "p10": 0.0, "p25": 0.0, "p75": 0.0, "p90": 0.0, "p95": 0.0, "p99": 0.0, "p999": 0.0}, "cumulative_distribution_function": null}, "total": {"mean": 198.08514508750469, "median": 203.04516628745705, "mode": 203.5378269520066, "variance": 619.6237334717947, "std_dev": 24.89224243558211, "min": 0.0, "max": 203.69598368219124, "count": 125, "total_sum": 18310.99071823841, "percentiles": {"p001": 46.71289356157213, "p01": 55.502236337170835, "p05": 190.14888022486173, "p10": 200.69400449782287, "p25": 202.23259402121505, "p75": 203.4193704835346, "p90": 203.5378269520066, "p95": 203.58722454130668, "p99": 203.6860916860917, "p999": 203.69598368219124}, "cumulative_distribution_function": null}}, "tokens_per_second": {"successful": {"mean": 992.6867036588937, "median": 614.3700014647723, "mode": 615.2712336805046, "variance": 62014350.40386989, "std_dev": 7874.919072845758, "min": 0.0, "max": 159300.81436773148, "count": 139, "total_sum": 5852579.912913391, "percentiles": {"p001": 46.71289356157213, "p01": 55.502236337170835, "p05": 574.9559972583961, "p10": 606.8148148148148, "p25": 611.5928842228055, "p75": 615.0907757735738, "p90": 615.4517975055026, "p95": 615.542119166422, "p99": 617.5359246171967, "p999": 157985.65557672578}, "cumulative_distribution_function": null}, "errored": {"mean": 0.0, "median": 0.0, "mode": 0.0, "variance": 0.0, "std_dev": 0.0, "min": 0.0, "max": 0.0, "count": 0, "total_sum": 0.0, "percentiles": {"p001": 0.0, "p01": 0.0, "p05": 0.0, "p10": 0.0, "p25": 0.0, "p75": 0.0, "p90": 0.0, "p95": 0.0, "p99": 0.0, "p999": 0.0}, "cumulative_distribution_function": null}, "incomplete": {"mean": 0.0, "median": 0.0, "mode": 0.0, "variance": 0.0, "std_dev": 0.0, "min": 0.0, "max": 0.0, "count": 0, "total_sum": 0.0, "percentiles": {"p001": 0.0, "p01": 0.0, "p05": 0.0, "p10": 0.0, "p25": 0.0, "p75": 0.0, "p90": 0.0, "p95": 0.0, "p99": 0.0, "p999": 0.0}, "cumulative_distribution_function": null}, "total": {"mean": 1002.1268169766876, "median": 614.3700014647723, "mode": 615.2712336805046, "variance": 63939736.95341249, "std_dev": 7996.232672541019, "min": 0.0, "max": 296531.848660591, "count": 143, "total_sum": 6151486.576325966, "percentiles": {"p001": 46.71289356157213, "p01": 55.502236337170835, "p05": 574.9559972583961, "p10": 606.8148148148148, "p25": 611.5928842228055, "p75": 615.0907757735738, "p90": 615.4517975055026, "p95": 615.542119166422, "p99": 1158.3275338304336, "p999": 158008.81383758428}, "cumulative_distribution_function": null}}}, "start_time": 1749157168.1827004, "end_time": 1749157198.1799018, "request_totals": {"successful": 46, "errored": 0, "incomplete": 1, "total": 47}, "request_samples": null, "requests": {"successful": [{"type_": "generative_text_response", "request_id": "73054dd1-486f-4894-a861-075750b82453", "request_type": "text_completions", "scheduler_info": {"requested": true, "completed": true, "errored": false, "canceled": false, "targeted_start_time": 1749157168.179883, "queued_time": 1749157168.1811602, "dequeued_time": 1749157168.1818697, "scheduled_time": 1749157168.181895, "worker_start": 1749157168.1820004, "request_start": 1749157168.1827004, "request_end": 1749157168.871885, "worker_end": 1749157168.8723884, "process_id": 0}, "prompt": "such a sacrifice to her advantage as years of gratitude cannot enough acknowledge. By this time she is actually with them! If such goodness does not make her miserable now, she will never deserve to be happy! What a meeting for her, when she first sees my aunt! We must endeavour to forget all that has passed on either side, said Jane I hope and trust they will yet be happy. His consenting to marry her is a proof, I will believe, that he is come to a right way of thinking. Their mutual affection will steady them; and I flatter myself they will settle so quietly, and live in so rational a manner, as may in time make their past imprudence forgotten. Their conduct has been such, replied Elizabeth, as neither you, nor I, nor anybody, can ever forget. It is useless to talk of it. It now occurred to the girls that their mother was in all likelihood perfectly ignorant of what had happened. They went to the library, therefore, and asked their father whether he would not wish them to make it known to her. He was writing, and, without raising his head, coolly replied, Just as you please. May we take my uncle s letter to read to her? Take whatever you like, and get away", "output": ", said Jane. The letter was read, and the girls retired to their own apartments. Elizabeth was the first to return. She found her mother seated in the drawing-room, and looking very pale. She was dressed in a loose white gown, and her hair was disordered. She rose as they entered, and clasped them both in her arms, and then, without saying a word, took her seat on the sofa, and began to weep. Elizabeth and Jane stood by her side, and listened to the sobs which issued from her heart. She had no words to express her gratitude, and, in a few minutes,", "prompt_tokens": 257, "output_tokens": 128, "start_time": 1749157168.1827004, "end_time": 1749157168.871885, "first_token_time": 1749157168.2039824, "last_token_time": 1749157168.8717923, "request_latency": 0.6891846656799316, "time_to_first_token_ms": 21.281957626342773, "time_per_output_token_ms": 5.217265337705612, "inter_token_latency_ms": 5.258346167136365, "tokens_per_second": 558.631117568713, "output_tokens_per_second": 185.72670921765}], "errored": [], "incomplete": [], "total": null}, "duration": 29.997201442718506}]} \ No newline at end of file diff --git a/tests/unit/entrypoints/assets/benchmarks_stripped.yaml b/tests/unit/entrypoints/assets/benchmarks_stripped.yaml deleted file mode 100644 index 1d39e62d3..000000000 --- a/tests/unit/entrypoints/assets/benchmarks_stripped.yaml +++ /dev/null @@ -1,1026 +0,0 @@ ---- -benchmarks: -- type_: generative_benchmark - id_: 97ece514-8717-412f-9dba-2b42bcd9866f - run_id: 93e36b31-b454-471d-ba62-6b2671585485 - args: - profile: - type_: sweep - completed_strategies: 10 - measured_rates: - - 1.5481806532737452 - measured_concurrencies: - - 0.9977627456483604 - max_concurrency: - strategy_type: constant - rate: -1 - initial_burst: true - random_seed: 42 - sweep_size: 10 - rate_type: constant - strategy_types: - - synchronous - strategy_index: 0 - strategy: - type_: synchronous - max_number: - max_duration: 30 - warmup_number: - warmup_duration: - cooldown_number: - cooldown_duration: - run_stats: - start_time: 1749157168.054225 - end_time: 1749157198.213826 - requests_made: - successful: 1 - errored: 0 - incomplete: 0 - total: 1 - queued_time_avg: 0.631589580089488 - scheduled_time_delay_avg: 3.784260851271609e-06 - scheduled_time_sleep_avg: 0 - worker_start_delay_avg: 2.8021792148021943e-05 - worker_time_avg: 0.6373953819274902 - worker_start_time_targeted_delay_avg: 0.6319031715393066 - request_start_time_delay_avg: 0.316034068452551 - request_start_time_targeted_delay_avg: 0.6319856542222043 - request_time_delay_avg: 0.00029866238857837433 - request_time_avg: 0.6370967195389119 - worker: - type_: generative_requests_worker - backend_type: openai_http - backend_target: example_target - backend_model: example_model - backend_info: - max_output_tokens: 16384 - timeout: 300 - http2: true - authorization: false - organization: - project: - text_completions_path: "/v1/completions" - chat_completions_path: "/v1/chat/completions" - request_loader: - type_: generative_request_loader - data: prompt_tokens=256,output_tokens=128 - data_args: - processor: example_processor - processor_args: - extras: {} - metrics: - requests_per_second: - successful: - mean: 1.5481806532737452 - median: 1.5530116578512305 - mode: 1.555484186315253 - variance: 0.0003352629331303757 - std_dev: 0.01831018659463567 - min: 1.4509899157628907 - max: 1.5597664461806156 - count: 45 - total_sum: 69.6707872953874 - percentiles: - p001: 1.4509899157628907 - p01: 1.4509899157628907 - p05: 1.5190957942495127 - p10: 1.5377883923356668 - p25: 1.5483918601985445 - p75: 1.5567531615313124 - p90: 1.5583715343236735 - p95: 1.5590938878953722 - p99: 1.5597664461806156 - p999: 1.5597664461806156 - cumulative_distribution_function: - errored: - mean: 0 - median: 0 - mode: 0 - variance: 0 - std_dev: 0 - min: 0 - max: 0 - count: 0 - total_sum: 0 - percentiles: - p001: 0 - p01: 0 - p05: 0 - p10: 0 - p25: 0 - p75: 0 - p90: 0 - p95: 0 - p99: 0 - p999: 0 - cumulative_distribution_function: - incomplete: - mean: 0 - median: 0 - mode: 0 - variance: 0 - std_dev: 0 - min: 0 - max: 0 - count: 0 - total_sum: 0 - percentiles: - p001: 0 - p01: 0 - p05: 0 - p10: 0 - p25: 0 - p75: 0 - p90: 0 - p95: 0 - p99: 0 - p999: 0 - cumulative_distribution_function: - total: - mean: 1.5668128271815418 - median: 1.5530312090734288 - mode: 1.555484186315253 - variance: 0.036536424510388923 - std_dev: 0.19114503527528232 - min: 1.4509899157628907 - max: 3.509921881864626 - count: 46 - total_sum: 73.18070917725203 - percentiles: - p001: 1.4509899157628907 - p01: 1.4509899157628907 - p05: 1.5190957942495127 - p10: 1.5377883923356668 - p25: 1.5483918601985445 - p75: 1.5567531615313124 - p90: 1.5583715343236735 - p95: 1.5591048992639953 - p99: 1.5597664461806156 - p999: 3.509921881864626 - cumulative_distribution_function: - request_concurrency: - successful: - mean: 0.9977627456483604 - median: 1 - mode: 1 - variance: 0.002232249044605607 - std_dev: 0.047246682895263736 - min: 0 - max: 1 - count: 2 - total_sum: 1 - percentiles: - p001: 0 - p01: 1 - p05: 1 - p10: 1 - p25: 1 - p75: 1 - p90: 1 - p95: 1 - p99: 1 - p999: 1 - cumulative_distribution_function: - errored: - mean: 0 - median: 0 - mode: 0 - variance: 0 - std_dev: 0 - min: 0 - max: 0 - count: 0 - total_sum: 0 - percentiles: - p001: 0 - p01: 0 - p05: 0 - p10: 0 - p25: 0 - p75: 0 - p90: 0 - p95: 0 - p99: 0 - p999: 0 - cumulative_distribution_function: - incomplete: - mean: 1 - median: 1 - mode: 1 - variance: 0 - std_dev: 0 - min: 1 - max: 1 - count: 1 - total_sum: 1 - percentiles: - p001: 1 - p01: 1 - p05: 1 - p10: 1 - p25: 1 - p75: 1 - p90: 1 - p95: 1 - p99: 1 - p999: 1 - cumulative_distribution_function: - total: - mean: 0.9977433642674269 - median: 1 - mode: 1 - variance: 0.002251543327743578 - std_dev: 0.047450430216633206 - min: 0 - max: 1 - count: 2 - total_sum: 1 - percentiles: - p001: 0 - p01: 1 - p05: 1 - p10: 1 - p25: 1 - p75: 1 - p90: 1 - p95: 1 - p99: 1 - p999: 1 - cumulative_distribution_function: - request_latency: - successful: - mean: 0.6444743664368339 - median: 0.6424565315246582 - mode: 0.6395885944366455 - variance: 6.414585873782315e-05 - std_dev: 0.008009110982988258 - min: 0.6395885944366455 - max: 0.6891846656799316 - count: 46 - total_sum: 29.64582085609436 - percentiles: - p001: 0.6395885944366455 - p01: 0.6395885944366455 - p05: 0.6399857997894287 - p10: 0.6403069496154785 - p25: 0.6409540176391602 - p75: 0.644390344619751 - p90: 0.6488735675811768 - p95: 0.656728982925415 - p99: 0.6891846656799316 - p999: 0.6891846656799316 - cumulative_distribution_function: - errored: - mean: 0 - median: 0 - mode: 0 - variance: 0 - std_dev: 0 - min: 0 - max: 0 - count: 0 - total_sum: 0 - percentiles: - p001: 0 - p01: 0 - p05: 0 - p10: 0 - p25: 0 - p75: 0 - p90: 0 - p95: 0 - p99: 0 - p999: 0 - cumulative_distribution_function: - incomplete: - mean: 0.2836878299713135 - median: 0.2836878299713135 - mode: 0.2836878299713135 - variance: 0 - std_dev: 0 - min: 0.2836878299713135 - max: 0.2836878299713135 - count: 1 - total_sum: 0.2836878299713135 - percentiles: - p001: 0.2836878299713135 - p01: 0.2836878299713135 - p05: 0.2836878299713135 - p10: 0.2836878299713135 - p25: 0.2836878299713135 - p75: 0.2836878299713135 - p90: 0.2836878299713135 - p95: 0.2836878299713135 - p99: 0.2836878299713135 - p999: 0.2836878299713135 - cumulative_distribution_function: - total: - mean: 0.6367980571503334 - median: 0.642310380935669 - mode: 0.2836878299713135 - variance: 0.0027733643692853522 - std_dev: 0.05266274175624881 - min: 0.2836878299713135 - max: 0.6891846656799316 - count: 47 - total_sum: 29.929508686065674 - percentiles: - p001: 0.2836878299713135 - p01: 0.2836878299713135 - p05: 0.6398613452911377 - p10: 0.6402454376220703 - p25: 0.640899658203125 - p75: 0.644390344619751 - p90: 0.6488735675811768 - p95: 0.656728982925415 - p99: 0.6891846656799316 - p999: 0.6891846656799316 - cumulative_distribution_function: - prompt_token_count: - successful: - mean: 257.1086956521739 - median: 257 - mode: 257 - variance: 0.14035916824196598 - std_dev: 0.37464538999161057 - min: 257 - max: 259 - count: 46 - total_sum: 11827 - percentiles: - p001: 257 - p01: 257 - p05: 257 - p10: 257 - p25: 257 - p75: 257 - p90: 257 - p95: 258 - p99: 259 - p999: 259 - cumulative_distribution_function: - errored: - mean: 0 - median: 0 - mode: 0 - variance: 0 - std_dev: 0 - min: 0 - max: 0 - count: 0 - total_sum: 0 - percentiles: - p001: 0 - p01: 0 - p05: 0 - p10: 0 - p25: 0 - p75: 0 - p90: 0 - p95: 0 - p99: 0 - p999: 0 - cumulative_distribution_function: - incomplete: - mean: 256 - median: 256 - mode: 256 - variance: 0 - std_dev: 0 - min: 256 - max: 256 - count: 1 - total_sum: 256 - percentiles: - p001: 256 - p01: 256 - p05: 256 - p10: 256 - p25: 256 - p75: 256 - p90: 256 - p95: 256 - p99: 256 - p999: 256 - cumulative_distribution_function: - total: - mean: 257.0851063829787 - median: 257 - mode: 256 - variance: 0.16296966953372566 - std_dev: 0.40369502044702715 - min: 256 - max: 259 - count: 47 - total_sum: 12083 - percentiles: - p001: 256 - p01: 256 - p05: 257 - p10: 257 - p25: 257 - p75: 257 - p90: 257 - p95: 258 - p99: 259 - p999: 259 - cumulative_distribution_function: - output_token_count: - successful: - mean: 127.99999999999999 - median: 128 - mode: 128 - variance: 2.01948391736579e-28 - std_dev: 1.4210854715202002e-14 - min: 128 - max: 128 - count: 46 - total_sum: 5888 - percentiles: - p001: 128 - p01: 128 - p05: 128 - p10: 128 - p25: 128 - p75: 128 - p90: 128 - p95: 128 - p99: 128 - p999: 128 - cumulative_distribution_function: - errored: - mean: 0 - median: 0 - mode: 0 - variance: 0 - std_dev: 0 - min: 0 - max: 0 - count: 0 - total_sum: 0 - percentiles: - p001: 0 - p01: 0 - p05: 0 - p10: 0 - p25: 0 - p75: 0 - p90: 0 - p95: 0 - p99: 0 - p999: 0 - cumulative_distribution_function: - incomplete: - mean: 0 - median: 0 - mode: 0 - variance: 0 - std_dev: 0 - min: 0 - max: 0 - count: 0 - total_sum: 0 - percentiles: - p001: 0 - p01: 0 - p05: 0 - p10: 0 - p25: 0 - p75: 0 - p90: 0 - p95: 0 - p99: 0 - p999: 0 - cumulative_distribution_function: - total: - mean: 126.44680851063832 - median: 128 - mode: 55 - variance: 110.97057492077867 - std_dev: 10.534257207832866 - min: 55 - max: 128 - count: 47 - total_sum: 5943 - percentiles: - p001: 55 - p01: 55 - p05: 128 - p10: 128 - p25: 128 - p75: 128 - p90: 128 - p95: 128 - p99: 128 - p999: 128 - cumulative_distribution_function: - time_to_first_token_ms: - successful: - mean: 16.792535781860348 - median: 16.38054847717285 - mode: 15.790939331054688 - variance: 1.2776652847210441 - std_dev: 1.1303385708366516 - min: 15.790939331054688 - max: 21.281957626342773 - count: 46 - total_sum: 772.4566459655762 - percentiles: - p001: 15.790939331054688 - p01: 15.790939331054688 - p05: 15.971660614013672 - p10: 16.034841537475586 - p25: 16.111373901367188 - p75: 16.840696334838867 - p90: 18.505334854125977 - p95: 19.00935173034668 - p99: 21.281957626342773 - p999: 21.281957626342773 - cumulative_distribution_function: - errored: - mean: 0 - median: 0 - mode: 0 - variance: 0 - std_dev: 0 - min: 0 - max: 0 - count: 0 - total_sum: 0 - percentiles: - p001: 0 - p01: 0 - p05: 0 - p10: 0 - p25: 0 - p75: 0 - p90: 0 - p95: 0 - p99: 0 - p999: 0 - cumulative_distribution_function: - incomplete: - mean: 0 - median: 0 - mode: 0 - variance: 0 - std_dev: 0 - min: 0 - max: 0 - count: 0 - total_sum: 0 - percentiles: - p001: 0 - p01: 0 - p05: 0 - p10: 0 - p25: 0 - p75: 0 - p90: 0 - p95: 0 - p99: 0 - p999: 0 - cumulative_distribution_function: - total: - mean: 16.777170465347616 - median: 16.371726989746094 - mode: 15.790939331054688 - variance: 1.2613411927317046 - std_dev: 1.1230944718641014 - min: 15.790939331054688 - max: 21.281957626342773 - count: 47 - total_sum: 788.5270118713379 - percentiles: - p001: 15.790939331054688 - p01: 15.790939331054688 - p05: 15.971660614013672 - p10: 16.034841537475586 - p25: 16.100645065307617 - p75: 16.840696334838867 - p90: 18.505334854125977 - p95: 19.00935173034668 - p99: 21.281957626342773 - p999: 21.281957626342773 - cumulative_distribution_function: - time_per_output_token_ms: - successful: - mean: 4.90300272307966 - median: 4.885653033852577 - mode: 4.870360717177391 - variance: 0.003163643010108571 - std_dev: 0.05624627107736628 - min: 4.870360717177391 - max: 5.217265337705612 - count: 46 - total_sum: 225.5381252616644 - percentiles: - p001: 4.870360717177391 - p01: 4.870360717177391 - p05: 4.8728808760643005 - p10: 4.873953759670258 - p25: 4.876237362623215 - p75: 4.904214292764664 - p90: 4.934689030051231 - p95: 4.993332549929619 - p99: 5.217265337705612 - p999: 5.217265337705612 - cumulative_distribution_function: - errored: - mean: 0 - median: 0 - mode: 0 - variance: 0 - std_dev: 0 - min: 0 - max: 0 - count: 0 - total_sum: 0 - percentiles: - p001: 0 - p01: 0 - p05: 0 - p10: 0 - p25: 0 - p75: 0 - p90: 0 - p95: 0 - p99: 0 - p999: 0 - cumulative_distribution_function: - incomplete: - mean: 0 - median: 0 - mode: 0 - variance: 0 - std_dev: 0 - min: 0 - max: 0 - count: 0 - total_sum: 0 - percentiles: - p001: 0 - p01: 0 - p05: 0 - p10: 0 - p25: 0 - p75: 0 - p90: 0 - p95: 0 - p99: 0 - p999: 0 - cumulative_distribution_function: - total: - mean: 4.9022222114856975 - median: 4.882922396063805 - mode: 4.870360717177391 - variance: 0.003199582258516055 - std_dev: 0.05656485002646127 - min: 4.81866489757191 - max: 5.217265337705612 - count: 47 - total_sum: 230.3567901592363 - percentiles: - p001: 4.81866489757191 - p01: 4.870360717177391 - p05: 4.872731864452362 - p10: 4.873953759670258 - p25: 4.876237362623215 - p75: 4.904214292764664 - p90: 4.934689030051231 - p95: 4.993332549929619 - p99: 5.217265337705612 - p999: 5.217265337705612 - cumulative_distribution_function: - inter_token_latency_ms: - successful: - mean: 4.941609043733832 - median: 4.9241227427805505 - mode: 4.90871001416304 - variance: 0.003213660306132974 - std_dev: 0.056689155101597465 - min: 4.90871001416304 - max: 5.258346167136365 - count: 46 - total_sum: 227.31401601175622 - percentiles: - p001: 4.90871001416304 - p01: 4.90871001416304 - p05: 4.911250016820713 - p10: 4.9123313483290785 - p25: 4.91463293240765 - p75: 4.9428301533376136 - p90: 4.973544849185493 - p95: 5.032650129062923 - p99: 5.258346167136365 - p999: 5.258346167136365 - cumulative_distribution_function: - errored: - mean: 0 - median: 0 - mode: 0 - variance: 0 - std_dev: 0 - min: 0 - max: 0 - count: 0 - total_sum: 0 - percentiles: - p001: 0 - p01: 0 - p05: 0 - p10: 0 - p25: 0 - p75: 0 - p90: 0 - p95: 0 - p99: 0 - p999: 0 - cumulative_distribution_function: - incomplete: - mean: 0 - median: 0 - mode: 0 - variance: 0 - std_dev: 0 - min: 0 - max: 0 - count: 0 - total_sum: 0 - percentiles: - p001: 0 - p01: 0 - p05: 0 - p10: 0 - p25: 0 - p75: 0 - p90: 0 - p95: 0 - p99: 0 - p999: 0 - cumulative_distribution_function: - total: - mean: 4.9413003057767115 - median: 4.921370603906826 - mode: 4.90871001416304 - variance: 0.003194539306669541 - std_dev: 0.056520255720135776 - min: 4.9078994327121315 - max: 5.258346167136365 - count: 47 - total_sum: 232.22191544446835 - percentiles: - p001: 4.9078994327121315 - p01: 4.90871001416304 - p05: 4.911099831888995 - p10: 4.9123313483290785 - p25: 4.91463293240765 - p75: 4.9428301533376136 - p90: 4.973544849185493 - p95: 5.032650129062923 - p99: 5.258346167136365 - p999: 5.258346167136365 - cumulative_distribution_function: - output_tokens_per_second: - successful: - mean: 198.13346751788123 - median: 203.04516628745705 - mode: 203.5378269520066 - variance: 613.9948900522365 - std_dev: 24.778920276158857 - min: 0 - max: 203.69598368219124 - count: 122 - total_sum: 17849.590625912137 - percentiles: - p001: 46.71289356157213 - p01: 55.502236337170835 - p05: 190.14888022486173 - p10: 200.69400449782287 - p25: 202.23259402121505 - p75: 203.42923658938793 - p90: 203.5378269520066 - p95: 203.58722454130668 - p99: 203.6860916860917 - p999: 203.69598368219124 - cumulative_distribution_function: - errored: - mean: 0 - median: 0 - mode: 0 - variance: 0 - std_dev: 0 - min: 0 - max: 0 - count: 0 - total_sum: 0 - percentiles: - p001: 0 - p01: 0 - p05: 0 - p10: 0 - p25: 0 - p75: 0 - p90: 0 - p95: 0 - p99: 0 - p999: 0 - cumulative_distribution_function: - incomplete: - mean: 0 - median: 0 - mode: 0 - variance: 0 - std_dev: 0 - min: 0 - max: 0 - count: 0 - total_sum: 0 - percentiles: - p001: 0 - p01: 0 - p05: 0 - p10: 0 - p25: 0 - p75: 0 - p90: 0 - p95: 0 - p99: 0 - p999: 0 - cumulative_distribution_function: - total: - mean: 198.08514508750469 - median: 203.04516628745705 - mode: 203.5378269520066 - variance: 619.6237334717947 - std_dev: 24.89224243558211 - min: 0 - max: 203.69598368219124 - count: 125 - total_sum: 18310.99071823841 - percentiles: - p001: 46.71289356157213 - p01: 55.502236337170835 - p05: 190.14888022486173 - p10: 200.69400449782287 - p25: 202.23259402121505 - p75: 203.4193704835346 - p90: 203.5378269520066 - p95: 203.58722454130668 - p99: 203.6860916860917 - p999: 203.69598368219124 - cumulative_distribution_function: - tokens_per_second: - successful: - mean: 992.6867036588937 - median: 614.3700014647723 - mode: 615.2712336805046 - variance: 62014350.40386989 - std_dev: 7874.919072845758 - min: 0 - max: 159300.81436773148 - count: 139 - total_sum: 5852579.912913391 - percentiles: - p001: 46.71289356157213 - p01: 55.502236337170835 - p05: 574.9559972583961 - p10: 606.8148148148148 - p25: 611.5928842228055 - p75: 615.0907757735738 - p90: 615.4517975055026 - p95: 615.542119166422 - p99: 617.5359246171967 - p999: 157985.65557672578 - cumulative_distribution_function: - errored: - mean: 0 - median: 0 - mode: 0 - variance: 0 - std_dev: 0 - min: 0 - max: 0 - count: 0 - total_sum: 0 - percentiles: - p001: 0 - p01: 0 - p05: 0 - p10: 0 - p25: 0 - p75: 0 - p90: 0 - p95: 0 - p99: 0 - p999: 0 - cumulative_distribution_function: - incomplete: - mean: 0 - median: 0 - mode: 0 - variance: 0 - std_dev: 0 - min: 0 - max: 0 - count: 0 - total_sum: 0 - percentiles: - p001: 0 - p01: 0 - p05: 0 - p10: 0 - p25: 0 - p75: 0 - p90: 0 - p95: 0 - p99: 0 - p999: 0 - cumulative_distribution_function: - total: - mean: 1002.1268169766876 - median: 614.3700014647723 - mode: 615.2712336805046 - variance: 63939736.95341249 - std_dev: 7996.232672541019 - min: 0 - max: 296531.848660591 - count: 143 - total_sum: 6151486.576325966 - percentiles: - p001: 46.71289356157213 - p01: 55.502236337170835 - p05: 574.9559972583961 - p10: 606.8148148148148 - p25: 611.5928842228055 - p75: 615.0907757735738 - p90: 615.4517975055026 - p95: 615.542119166422 - p99: 1158.3275338304336 - p999: 158008.81383758428 - cumulative_distribution_function: - start_time: 1749157168.1827004 - end_time: 1749157198.1799018 - request_totals: - successful: 46 - errored: 0 - incomplete: 1 - total: 47 - request_samples: - requests: - successful: - - type_: generative_text_response - request_id: 73054dd1-486f-4894-a861-075750b82453 - request_type: text_completions - scheduler_info: - requested: true - completed: true - errored: false - canceled: false - targeted_start_time: 1749157168.179883 - queued_time: 1749157168.1811602 - dequeued_time: 1749157168.1818697 - scheduled_time: 1749157168.181895 - worker_start: 1749157168.1820004 - request_start: 1749157168.1827004 - request_end: 1749157168.871885 - worker_end: 1749157168.8723884 - process_id: 0 - prompt: such a sacrifice to her advantage as years of gratitude cannot enough - acknowledge. By this time she is actually with them! If such goodness does - not make her miserable now, she will never deserve to be happy! What a meeting - for her, when she first sees my aunt! We must endeavour to forget all that - has passed on either side, said Jane I hope and trust they will yet be happy. - His consenting to marry her is a proof, I will believe, that he is come to - a right way of thinking. Their mutual affection will steady them; and I flatter - myself they will settle so quietly, and live in so rational a manner, as may - in time make their past imprudence forgotten. Their conduct has been such, - replied Elizabeth, as neither you, nor I, nor anybody, can ever forget. It - is useless to talk of it. It now occurred to the girls that their mother was - in all likelihood perfectly ignorant of what had happened. They went to the - library, therefore, and asked their father whether he would not wish them - to make it known to her. He was writing, and, without raising his head, coolly - replied, Just as you please. May we take my uncle s letter to read to her? - Take whatever you like, and get away - output: ", said Jane. The letter was read, and the girls retired to their own - apartments. Elizabeth was the first to return. She found her mother seated - in the drawing-room, and looking very pale. She was dressed in a loose white - gown, and her hair was disordered. She rose as they entered, and clasped them - both in her arms, and then, without saying a word, took her seat on the sofa, - and began to weep. Elizabeth and Jane stood by her side, and listened to the - sobs which issued from her heart. She had no words to express her gratitude, - and, in a few minutes," - prompt_tokens: 257 - output_tokens: 128 - start_time: 1749157168.1827004 - end_time: 1749157168.871885 - first_token_time: 1749157168.2039824 - last_token_time: 1749157168.8717923 - request_latency: 0.6891846656799316 - time_to_first_token_ms: 21.281957626342773 - time_per_output_token_ms: 5.217265337705612 - inter_token_latency_ms: 5.258346167136365 - tokens_per_second: 558.631117568713 - output_tokens_per_second: 185.72670921765 - total: - duration: 29.997201442718506 diff --git a/tests/unit/entrypoints/assets/benchmarks_stripped_output.txt b/tests/unit/entrypoints/assets/benchmarks_stripped_output.txt deleted file mode 100644 index 170d1e6a4..000000000 --- a/tests/unit/entrypoints/assets/benchmarks_stripped_output.txt +++ /dev/null @@ -1,31 +0,0 @@ - - -Benchmarks Metadata: - Run id:93e36b31-b454-471d-ba62-6b2671585485 - Duration:30.2 seconds - Profile:type=sweep, strategies=['synchronous', 'throughput', 'constant', 'constant', 'constant', 'constant', 'constant', 'constant', 'constant', 'constant'], - max_concurrency=None - Args:max_number=None, max_duration=30.0, warmup_number=None, warmup_duration=None, cooldown_number=None, cooldown_duration=None - Worker:type_='generative_requests_worker' backend_type='openai_http' backend_target='example_target' backend_model='example_model' backend_info={'max_output_tokens': 16384, - 'timeout': 300, 'http2': True, 'authorization': False, 'organization': None, 'project': None, 'text_completions_path': '/v1/completions', 'chat_completions_path': - '/v1/chat/completions'} - Request Loader:type_='generative_request_loader' data='prompt_tokens=256,output_tokens=128' data_args=None processor='example_processor' processor_args=None - Extras:None - - -Benchmarks Info: -=================================================================================================================================================== -Metadata |||| Requests Made ||| Prompt Tok/Req ||| Output Tok/Req ||| Prompt Tok Total||| Output Tok Total || - Benchmark| Start Time| End Time| Duration (s)| Comp| Inc| Err| Comp| Inc| Err| Comp| Inc| Err| Comp| Inc| Err| Comp| Inc| Err ------------|-----------|---------|-------------|------|-----|-----|------|------|----|-------|-----|-----|-------|-----|-----|-------|------|------ -synchronous| 16:59:28| 16:59:58| 30.0| 46| 1| 0| 257.1| 256.0| 0.0| 128.0| 0.0| 0.0| 11827| 256| 0| 5888| 0| 0 -=================================================================================================================================================== - - -Benchmarks Stats: -=============================================================================================================================================== -Metadata | Request Stats || Out Tok/sec| Tot Tok/sec| Req Latency (sec) ||| TTFT (ms) ||| ITL (ms) ||| TPOT (ms) || - Benchmark| Per Second| Concurrency| mean| mean| mean| median| p99| mean| median| p99| mean| median| p99| mean| median| p99 ------------|-----------|------------|------------|------------|------|--------|------|-----|-------|-----|-----|-------|----|-----|-------|---- -synchronous| 1.55| 1.00| 198.1| 992.7| 0.64| 0.64| 0.69| 16.8| 16.4| 21.3| 4.9| 4.9| 5.3| 4.9| 4.9| 5.2 -=============================================================================================================================================== diff --git a/tests/unit/entrypoints/test_benchmark_from_file_entrypoint.py b/tests/unit/entrypoints/test_benchmark_from_file_entrypoint.py index b3a552b13..bbbe0072b 100644 --- a/tests/unit/entrypoints/test_benchmark_from_file_entrypoint.py +++ b/tests/unit/entrypoints/test_benchmark_from_file_entrypoint.py @@ -1,82 +1,254 @@ import filecmp +import json import os -import unittest -from pathlib import Path import pytest +import yaml +from trio import Path from guidellm.benchmark import reimport_benchmarks_report +from guidellm.benchmark.outputs.serialized import ( + JSONBenchmarkOutputArgs, + YAMLBenchmarkOutputArgs, +) +from guidellm.benchmark.schemas import ( + BenchmarkScenario, + GenerativeBenchmarksReport, +) -# Set to true to re-write the expected output. -REGENERATE_ARTIFACTS = False + +@pytest.fixture +def sample_report() -> GenerativeBenchmarksReport: + """ + Create a valid report that can be loaded by reimport_benchmarks_report. + + ## WRITTEN BY AI ## + """ + config = BenchmarkScenario.model_validate( + { + "spec": { + "backend": { + "kind": "openai_http", + "target": "http://localhost:8000/v1", + "model": "test-model", + }, + "data": [{"kind": "huggingface", "source": "test_data.jsonl"}], + "profile": {"kind": "constant", "rate": 10.0}, + }, + } + ) + return GenerativeBenchmarksReport(config=config) + + +@pytest.fixture +def saved_json_report( + tmp_path: Path, sample_report: GenerativeBenchmarksReport +) -> Path: + """ + Persist a sample report as JSON for from-file entrypoint tests. + + ## WRITTEN BY AI ## + """ + return sample_report.save_file(tmp_path / "benchmarks.json") @pytest.fixture -def get_test_asset_dir(): - def _() -> Path: - return Path(__file__).parent / "assets" +def saved_yaml_report( + tmp_path: Path, sample_report: GenerativeBenchmarksReport +) -> Path: + """ + Persist a sample report as YAML for from-file entrypoint tests. - return _ + ## WRITTEN BY AI ## + """ + return sample_report.save_file(tmp_path / "benchmarks.yaml") @pytest.fixture -def cleanup(): - to_delete: list[Path] = [] - yield to_delete - for item in to_delete: - if item.exists(): - item.unlink() # Deletes the file - - -@pytest.mark.skip(reason="currently broken") -def test_display_entrypoint_json(capfd, get_test_asset_dir): - generic_test_display_entrypoint( - "benchmarks_stripped.json", - capfd, - get_test_asset_dir, +def report_directory(tmp_path: Path, sample_report: GenerativeBenchmarksReport) -> Path: + """ + Directory containing the default benchmarks.json report file. + + ## WRITTEN BY AI ## + """ + sample_report.save_file(tmp_path / "benchmarks.json") + return tmp_path + + +@pytest.mark.asyncio +@pytest.mark.sanity +async def test_loads_json_report( + saved_json_report: Path, sample_report: GenerativeBenchmarksReport +): + """ + Loading a JSON report returns the saved configuration and benchmark list. + + ## WRITTEN BY AI ## + """ + report, output_results = await reimport_benchmarks_report(saved_json_report, []) + + assert output_results == {} + assert report.config == sample_report.config + assert report.benchmarks == [] + assert report.metadata.version == 2 + + +@pytest.mark.asyncio +@pytest.mark.sanity +async def test_loads_yaml_report( + saved_yaml_report: Path, sample_report: GenerativeBenchmarksReport +): + """ + Loading a YAML report returns the saved configuration and benchmark list. + + ## WRITTEN BY AI ## + """ + report, output_results = await reimport_benchmarks_report(saved_yaml_report, []) + + assert output_results == {} + assert report.config == sample_report.config + assert report.benchmarks == [] + + +@pytest.mark.asyncio +@pytest.mark.sanity +async def test_loads_report_from_directory( + report_directory: Path, sample_report: GenerativeBenchmarksReport +): + """ + A directory path resolves to benchmarks.json before loading the report. + + ## WRITTEN BY AI ## + """ + report, output_results = await reimport_benchmarks_report(report_directory, []) + + assert output_results == {} + assert report.config == sample_report.config + + +@pytest.mark.asyncio +@pytest.mark.sanity +async def test_console_output(capfd, saved_json_report: Path): + """ + Console output is produced when the console output format is requested. + + ## WRITTEN BY AI ## + """ + os.environ["COLUMNS"] = "180" + + _, output_results = await reimport_benchmarks_report( + saved_json_report, + [{"kind": "console"}], ) + out, err = capfd.readouterr() + combined = out + err + assert "Import of old benchmarks complete" in combined + assert "Run Summary Info" in combined + assert output_results == {"console": "printed to console"} + -@pytest.mark.skip(reason="currently broken") -def test_display_entrypoint_yaml(capfd, get_test_asset_dir): - generic_test_display_entrypoint( - "benchmarks_stripped.yaml", - capfd, - get_test_asset_dir, +@pytest.mark.asyncio +@pytest.mark.regression +async def test_reexports_json(saved_json_report: Path, tmp_path: Path): + """ + Re-exporting to JSON writes a file with the same report content. + + ## WRITTEN BY AI ## + """ + exported_file = tmp_path / "benchmarks_reexported.json" + + report, output_results = await reimport_benchmarks_report( + saved_json_report, + [{"kind": "json", "path": exported_file}], ) + assert exported_file.exists() + assert output_results["json"] == exported_file + assert filecmp.cmp(saved_json_report, exported_file, shallow=False) + + reloaded = GenerativeBenchmarksReport.load_file(exported_file) + assert reloaded.config == report.config + assert reloaded.benchmarks == report.benchmarks + + +@pytest.mark.asyncio +@pytest.mark.regression +async def test_reexports_yaml(saved_yaml_report: Path, tmp_path: Path): + """ + Re-exporting to YAML writes a file with the same report content. + + ## WRITTEN BY AI ## + """ + exported_file = tmp_path / "benchmarks_reexported.yaml" + + report, output_results = await reimport_benchmarks_report( + saved_yaml_report, + [{"kind": "yaml", "path": exported_file}], + ) -def generic_test_display_entrypoint(filename, capfd, get_test_asset_dir): - os.environ["COLUMNS"] = "180" # CLI output depends on terminal width. - asset_dir = get_test_asset_dir() - reimport_benchmarks_report(asset_dir / filename, None) - out, err = capfd.readouterr() - expected_output_path = asset_dir / "benchmarks_stripped_output.txt" - if REGENERATE_ARTIFACTS: - expected_output_path.write_text(out) - # Fail to prevent accidentally leaving regeneration mode on - pytest.fail("Test bypassed to regenerate output") - else: - with expected_output_path.open(encoding="utf_8") as file: - expected_output = file.read() - assert out == expected_output - - -@pytest.mark.skip(reason="currently broken") -def test_reexporting_benchmark(get_test_asset_dir, cleanup): - asset_dir = get_test_asset_dir() - source_file = asset_dir / "benchmarks_stripped.json" - exported_file = asset_dir / "benchmarks_reexported.json" - # If you need to inspect the output to see why it failed, comment out - # the cleanup statement. - cleanup.append(exported_file) - if exported_file.exists(): - exported_file.unlink() - reimport_benchmarks_report(source_file, exported_file) - # The reexported file should exist and be identical to the source. assert exported_file.exists() - assert filecmp.cmp(source_file, exported_file, shallow=False) + assert output_results["yaml"] == exported_file + assert filecmp.cmp(saved_yaml_report, exported_file, shallow=False) + + reloaded = GenerativeBenchmarksReport.load_file(exported_file) + assert reloaded.config == report.config + assert reloaded.benchmarks == report.benchmarks + + +@pytest.mark.asyncio +@pytest.mark.sanity +async def test_multiple_output_formats(saved_json_report: Path, tmp_path: Path): + """ + Multiple output formats are finalized and returned in a single call. + + ## WRITTEN BY AI ## + """ + exported_json = tmp_path / "benchmarks.json" + exported_yaml = tmp_path / "benchmarks.yaml" + + report, output_results = await reimport_benchmarks_report( + saved_json_report, + [ + {"kind": "console"}, + {"kind": "json", "path": exported_json}, + {"kind": "yaml", "path": exported_yaml}, + ], + ) + assert set(output_results) == {"console", "json", "yaml"} + assert output_results["console"] == "printed to console" + assert output_results["json"] == exported_json + assert output_results["yaml"] == exported_yaml + assert exported_json.exists() + assert exported_yaml.exists() + assert json.loads(exported_json.read_text()) == json.loads( + saved_json_report.read_text() + ) + parsed_yaml = yaml.safe_load(exported_yaml.read_text()) + assert isinstance(parsed_yaml, dict) + assert parsed_yaml["config"] == json.loads(saved_json_report.read_text())["config"] + assert report.benchmarks == [] + + +@pytest.mark.asyncio +@pytest.mark.sanity +async def test_outputs_accepts_tuple_of_args(saved_json_report: Path, tmp_path: Path): + """ + Output arguments may be provided as validated BenchmarkOutputArgs instances. + + ## WRITTEN BY AI ## + """ + exported_file = tmp_path / "benchmarks.json" + + _, output_results = await reimport_benchmarks_report( + saved_json_report, + ( + JSONBenchmarkOutputArgs(path=exported_file), + YAMLBenchmarkOutputArgs(path=tmp_path / "benchmarks.yaml"), + ), + ) -if __name__ == "__main__": - unittest.main() + assert set(output_results) == {"json", "yaml"} + assert exported_file.exists() + assert (tmp_path / "benchmarks.yaml").exists() diff --git a/uv.lock b/uv.lock index e7d5ab21e..dd16bb870 100644 --- a/uv.lock +++ b/uv.lock @@ -348,6 +348,46 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/7d/9bc192684cea499815ff478dfcdc13835ddf401365057044fb721ec6bddb/certifi-2025.11.12-py3-none-any.whl", hash = "sha256:97de8790030bbd5c2d96b7ec782fc2f7820ef8dba6db909ccf95449f2d062d4b", size = 159438, upload-time = "2025-11-12T02:54:49.735Z" }, ] +[[package]] +name = "cffi" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "(implementation_name != 'PyPy' and platform_machine != 'arm64') or (implementation_name != 'PyPy' and sys_platform != 'darwin')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/57/5f/ff100cae70ebe9d8df1c01a00e510e45d9adb5c1fdda84791b199141de97/cffi-2.1.0.tar.gz", hash = "sha256:efc1cdd798b1aaf39b4610bba7aad28c9bea9b910f25c784ccf9ec1fa719d1f9", size = 531036, upload-time = "2026-07-06T21:34:30.382Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/70/b6/9003c33a3e7d2c1306f5962e646457dcfe5a8cd8fce6bbe02d7af25db783/cffi-2.1.0-cp310-cp310-win32.whl", hash = "sha256:9d72af0cf10a76a600a9690078fe31c63b9588c8e86bf9fd353f713c84b5db0f", size = 174578, upload-time = "2026-07-06T21:32:23.073Z" }, + { url = "https://files.pythonhosted.org/packages/8a/26/710688310447531c7a22f857c7f79d9855ec18b03e04494ced723fb37e2f/cffi-2.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:fb62edb5bb52cca65fab91a63afa7561607120d26090a7e8fda6fb9f064726da", size = 185071, upload-time = "2026-07-06T21:32:24.671Z" }, + { url = "https://files.pythonhosted.org/packages/eb/da/5c4918a2d61d86fa927d716cb3d8e4626ef8dc8f605a599d32f33897f59a/cffi-2.1.0-cp311-cp311-win32.whl", hash = "sha256:64c753a0f87a256020004f37a1c8c02c480e725f910f0b2a0f3f07debd1b2479", size = 174496, upload-time = "2026-07-06T21:32:40.467Z" }, + { url = "https://files.pythonhosted.org/packages/f9/c8/6c2de1d55cf35ef8b92885d5ef280790f0fb9634d87ea1cc315176aecd61/cffi-2.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:4f26194e3d95e06501b942642855aed4f953d55e95d7d01b7c4483db3ecff458", size = 185113, upload-time = "2026-07-06T21:32:41.761Z" }, + { url = "https://files.pythonhosted.org/packages/9e/4e/e8d7cb5783f1841a3c8fb3a7735838d7484d08ec08c9f984b14cac1ac0e9/cffi-2.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:35aaea0c7ee0e58a5cd8c2fd1a48fdf7ece0d2699b7ecdda08194e9ce5dd9b3d", size = 179927, upload-time = "2026-07-06T21:32:42.961Z" }, + { url = "https://files.pythonhosted.org/packages/b0/80/c138990aa2a70b1a269f6e06348729836d733d6f970867943f61d367f8cc/cffi-2.1.0-cp312-cp312-win32.whl", hash = "sha256:9b8f0f26ca4e7513c534d351eca551947d053fac438f2a04ac96d882909b0d3a", size = 175269, upload-time = "2026-07-06T21:32:57.777Z" }, + { url = "https://files.pythonhosted.org/packages/a8/eb/f636456ff21a83fc13c032b58cc5dde061691546ac79efa284b2989b7982/cffi-2.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:c97f080ea627e2863524c5af3836e2270b5f5dfff1f104392b959f8df0c5d384", size = 185881, upload-time = "2026-07-06T21:32:59.253Z" }, + { url = "https://files.pythonhosted.org/packages/dd/2c/400ea43e721727dca8a65c4521390e9196757caba4a45643acb2b63271b8/cffi-2.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:6d194185eabd279f1c05ebe3504265ddfc5ad2b58d0714f7db9f01da592e9eb6", size = 180088, upload-time = "2026-07-06T21:33:02.278Z" }, + { url = "https://files.pythonhosted.org/packages/96/88/a996879e2eeccb815f6e3a5967b12a308257412acec882039d386bd2aa7b/cffi-2.1.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:10537b1df4967ca26d21e5072d7d54188354483b91dc75058968d3f0cf13fbda", size = 194331, upload-time = "2026-07-06T21:33:03.697Z" }, + { url = "https://files.pythonhosted.org/packages/58/85/7ae00d5c8dd6266f4e944c3db630f3c5c9a98b61d469c714d848b1d8138a/cffi-2.1.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:a95b05f9baf29b91171b3a8bd2020b028835243e7b0ff6bb23e2a3c228518b1b", size = 196966, upload-time = "2026-07-06T21:33:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/c6/4b/e706f67279140f92939da3475ad610df18bfd52d50f14953a8e5fede71d5/cffi-2.1.0-cp313-cp313-win32.whl", hash = "sha256:db3eb7d46527159a878ec3460e9d40615bc25ba337d477db681aea6e4f05c5d2", size = 175248, upload-time = "2026-07-06T21:33:19.799Z" }, + { url = "https://files.pythonhosted.org/packages/5a/47/59eb7975cb0e4ef0afa764ea945b29a5bb4537a9f771cb7d6c8a5dd74c95/cffi-2.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:8e74a6135550c4748af665b1b1118b6aab33b1fc6a16f9aff630af107c3b4512", size = 185717, upload-time = "2026-07-06T21:33:21.47Z" }, + { url = "https://files.pythonhosted.org/packages/5a/af/34fee85c48f8d94efc8597bc09470c9dd274c145f1c12e0fbc6ab6d38d74/cffi-2.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:2282cd5e38aa8accd03e99d1256af8411c84cdbee6a89d841b563fdbd1f3e50f", size = 180114, upload-time = "2026-07-06T21:33:22.515Z" }, + { url = "https://files.pythonhosted.org/packages/d8/f0/81478e482afa03f6d18dc8f2afb5edc45b3080853b634b5ed91961be0998/cffi-2.1.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d2117334c3af3bdcb9a88522b844a2bdb5efdc4f71c6c822df55486ae1c3347a", size = 194142, upload-time = "2026-07-06T21:33:23.657Z" }, + { url = "https://files.pythonhosted.org/packages/7d/95/8de304305cd9204974b0ca051b86d307cafca13aa575a0ef1b44d92c0d8c/cffi-2.1.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:702c436735fbe99d59ada02a1f65cfc0d31c0ee8b7290912f8fbc5cd1e4b16c3", size = 196819, upload-time = "2026-07-06T21:33:25.007Z" }, + { url = "https://files.pythonhosted.org/packages/0f/6f/ade5ce9863a57992a6ea3d0d10d7e29b8749fc127204b3d493d667b2815f/cffi-2.1.0-cp314-cp314-win32.whl", hash = "sha256:1854b724d00f6654c742097d5387569021be12d3a0f770eae1df8f8acfcc6acd", size = 177723, upload-time = "2026-07-06T21:33:51.626Z" }, + { url = "https://files.pythonhosted.org/packages/41/de/92b9eeed4ae4a21d6fd9b2a2c8505cbed573299902ea73981cc13f7ff62c/cffi-2.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:1b96bfe2c4bd825681b7d311ad6d9b7280a091f43e8f63da5729638083cd3bfb", size = 187937, upload-time = "2026-07-06T21:33:53.403Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1a/cc6ae6c2913a03aab8898eee57963cf1035b8df5872ed8b9115fcc7e2be8/cffi-2.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:7d28dff1db6764108bc30788d85d61c876beff416d9a49cb9dd7c5a9f34f5804", size = 183001, upload-time = "2026-07-06T21:33:54.74Z" }, + { url = "https://files.pythonhosted.org/packages/dc/78/aa01ac599a8a4322533d45a1f9bc93b338276d2d59dabbe7c6d92a775c81/cffi-2.1.0-cp314-cp314t-win32.whl", hash = "sha256:7d034dcffa09e9a46c93fa3a3be402096cb5354ac6e41ab8e5cc9cd8b642ad76", size = 182857, upload-time = "2026-07-06T21:33:47.696Z" }, + { url = "https://files.pythonhosted.org/packages/b9/26/d00496b22de4d4228f32dde94ad996f350c8aad676d63bcca0743c8dea4d/cffi-2.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0582a58f3051372229ca8e7f5f589f9e5632678208d8636fea3676711fdf7fe5", size = 194065, upload-time = "2026-07-06T21:33:48.953Z" }, + { url = "https://files.pythonhosted.org/packages/d5/dd/0c7dbf815a579ff005008a2d815a55d6bb047c349eef536d9dc53d3f0a8d/cffi-2.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:510aeeeac94811b138077451da1fb18b308a5feab47dd2b603af55804155e1c8", size = 186404, upload-time = "2026-07-06T21:33:50.309Z" }, + { url = "https://files.pythonhosted.org/packages/55/c7/8c8c50cb11c6750051daf12164098a9a6f027ac4356967fd4d800a07f242/cffi-2.1.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:2e9dabb9abcb7ad15938c7196ad5c1718a4e6d33cc79b4c0209bdb64c4a54a5c", size = 194121, upload-time = "2026-07-06T21:33:56.109Z" }, + { url = "https://files.pythonhosted.org/packages/99/e2/67680bf19a6b60d2bb7ff83baefa2a4c3d2d7dc0f3277034b802e1fc504c/cffi-2.1.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:37f525a7e7e50c017fdebe58b787be310ad59357ae43a053943a6e1a6c526001", size = 196820, upload-time = "2026-07-06T21:33:57.288Z" }, + { url = "https://files.pythonhosted.org/packages/2c/d8/772b8259bf75749adffb1c546828978381fb516f60cf701f6c83daf60c85/cffi-2.1.0-cp315-cp315-win32.whl", hash = "sha256:0a42c688d19fca6e095a53c6a6e2295a5b050a8b289f109adab02a9e61a25de6", size = 177696, upload-time = "2026-07-06T21:34:26.355Z" }, + { url = "https://files.pythonhosted.org/packages/2f/dd/afa2191fc6d57fedd26e5844a2fe2fcc0bbfa00961bbaa5a41e4921e7cca/cffi-2.1.0-cp315-cp315-win_amd64.whl", hash = "sha256:bccbbb5ee76a61f9d99b5bf3846a51d7fca4b6a732fe46f89295610edaf41853", size = 187914, upload-time = "2026-07-06T21:34:27.58Z" }, + { url = "https://files.pythonhosted.org/packages/05/ef/6cd4f8c671517162379dc79cfae5aea9106bc38abb89628d5c16adf6a838/cffi-2.1.0-cp315-cp315-win_arm64.whl", hash = "sha256:8d35c139744adb3e727cd51b1a18324bbe44b8bd41bf8322bca4d41289f48eda", size = 183004, upload-time = "2026-07-06T21:34:28.905Z" }, + { url = "https://files.pythonhosted.org/packages/68/5a/e536c528bc8057496c360c0978559a2dc45653f89dd6151078aa7d8fca1a/cffi-2.1.0-cp315-cp315t-win32.whl", hash = "sha256:cb96698e3c7413d906ce83f8ffd245ec1bd94707541f299d0ce4d6b0193e982b", size = 182760, upload-time = "2026-07-06T21:34:22.059Z" }, + { url = "https://files.pythonhosted.org/packages/d3/0b/0ffe8b82d3875bced5fa1e7986a7a46b748262a40ab7f60b475eb9fb1bb3/cffi-2.1.0-cp315-cp315t-win_amd64.whl", hash = "sha256:f146d154428a2523f9cc7936c02353c2459b8f6cf07d3cd1ee1c0a611109c5d5", size = 193769, upload-time = "2026-07-06T21:34:23.589Z" }, + { url = "https://files.pythonhosted.org/packages/a0/17/1073b53b68c9b5ca6914adf5f8bf55aacc2d3be102418c90700160ea8605/cffi-2.1.0-cp315-cp315t-win_arm64.whl", hash = "sha256:cbb7640ce37159548d2147b5b8c241f962143d4c71231431820783f4dc78f210", size = 186405, upload-time = "2026-07-06T21:34:24.857Z" }, +] + [[package]] name = "cfgv" version = "3.5.0" @@ -1173,6 +1213,7 @@ dev = [ { name = "setuptools-git-versioning" }, { name = "tox" }, { name = "tox-uv" }, + { name = "trio" }, { name = "types-pyyaml" }, ] env = [ @@ -1201,6 +1242,7 @@ test = [ { name = "pytest-rerunfailures" }, { name = "pytest-timeout" }, { name = "respx" }, + { name = "trio" }, ] [package.metadata] @@ -1277,6 +1319,7 @@ dev = [ { name = "setuptools-git-versioning", specifier = ">=2.0,<3" }, { name = "tox" }, { name = "tox-uv" }, + { name = "trio", specifier = "~=0.33.0" }, { name = "types-pyyaml", specifier = "~=6.0.1" }, ] env = [ @@ -1305,6 +1348,7 @@ test = [ { name = "pytest-rerunfailures", specifier = "~=16.4" }, { name = "pytest-timeout", specifier = "~=2.4.0" }, { name = "respx", specifier = "~=0.23.1" }, + { name = "trio", specifier = "~=0.33.0" }, ] [[package]] @@ -2943,6 +2987,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/55/74/f473a3ec7a0a7ebc825ca8e3c86763f7d039f379860c81ba12dcdd456547/orjson-3.11.6-cp314-cp314-win_arm64.whl", hash = "sha256:fe71f6b283f4f1832204ab8235ce07adad145052614f77c876fcf0dac97bc06f", size = 135168, upload-time = "2026-01-29T15:13:05.932Z" }, ] +[[package]] +name = "outcome" +version = "1.3.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/df/77698abfac98571e65ffeb0c1fba8ffd692ab8458d617a0eed7d9a8d38f2/outcome-1.3.0.post0.tar.gz", hash = "sha256:9dcf02e65f2971b80047b377468e72a268e15c0af3cf1238e6ff14f7f91143b8", size = 21060, upload-time = "2023-10-26T04:26:04.361Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/55/8b/5ab7257531a5d830fc8000c476e63c935488d74609b50f9384a643ec0a62/outcome-1.3.0.post0-py2.py3-none-any.whl", hash = "sha256:e771c5ce06d1415e356078d3bdd68523f284b4ce5419828922b6871e65eda82b", size = 10692, upload-time = "2023-10-26T04:26:02.532Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -3392,6 +3448,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b1/ec/1fb891d8a2660716aadb2143235481d15ed1cbfe3ad669194690b0604492/pycountry-24.6.1-py3-none-any.whl", hash = "sha256:f1a4fb391cd7214f8eefd39556d740adcc233c778a27f8942c8dca351d6ce06f", size = 6335189, upload-time = "2024-06-01T04:11:49.711Z" }, ] +[[package]] +name = "pycparser" +version = "3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, +] + [[package]] name = "pycryptodomex" version = "3.23.0" @@ -4320,6 +4385,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + [[package]] name = "sympy" version = "1.14.0" @@ -4772,6 +4846,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/28/35f7411ff80a3640c1f4fc907dcbb6a65061ebb82f66950e38bfc9f7f740/transformers-5.5.0-py3-none-any.whl", hash = "sha256:821a9ff0961abbb29eb1eb686d78df1c85929fdf213a3fe49dc6bd94f9efa944", size = 10245591, upload-time = "2026-04-02T16:13:03.462Z" }, ] +[[package]] +name = "trio" +version = "0.33.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "cffi", marker = "(implementation_name != 'pypy' and os_name == 'nt' and platform_machine != 'arm64') or (implementation_name != 'pypy' and os_name == 'nt' and sys_platform != 'darwin')" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "idna" }, + { name = "outcome" }, + { name = "sniffio" }, + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/b6/c744031c6f89b18b3f5f4f7338603ab381d740a7f45938c4607b2302481f/trio-0.33.0.tar.gz", hash = "sha256:a29b92b73f09d4b48ed249acd91073281a7f1063f09caba5dc70465b5c7aa970", size = 605109, upload-time = "2026-02-14T18:40:55.386Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/93/dab25dc87ac48da0fe0f6419e07d0bfd98799bed4e05e7b9e0f85a1a4b4b/trio-0.33.0-py3-none-any.whl", hash = "sha256:3bd5d87f781d9b0192d592aef28691f8951d6c2e41b7e1da4c25cde6c180ae9b", size = 510294, upload-time = "2026-02-14T18:40:53.313Z" }, +] + [[package]] name = "typer" version = "0.24.1"