diff --git a/README.md b/README.md index 33e7fe2..2753f6c 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ The fork baseline is commit [`3c4264d`](https://github.com/InseeFrLab/auto-tunin ### Benchmarking (GuideLLM) - **`benchmark.warmup` / `benchmark.cooldown`** — exclude cold-start and shutdown phases from reported metrics to reduce variance ([#24](https://github.com/InseeFrLab/auto-tuning-vllm/pull/24)) +- **`benchmark.rampup`** — linear load ramp-up duration in seconds before reaching target concurrency - **`benchmark.sample_requests`** — control per-request samples in benchmark JSON output (default `0` keeps files small; requires GuideLLM `>= 0.5.4`) ([#27](https://github.com/InseeFrLab/auto-tuning-vllm/pull/27)) ### Bug fixes diff --git a/auto_tune_vllm/benchmarks/config.py b/auto_tune_vllm/benchmarks/config.py index 78b2b37..7a50e5f 100644 --- a/auto_tune_vllm/benchmarks/config.py +++ b/auto_tune_vllm/benchmarks/config.py @@ -34,6 +34,9 @@ class BenchmarkConfig: warmup: Optional[float] = None cooldown: Optional[float] = None + # GuideLLM ramp-up duration in seconds (linear increase to target rate). Omit to disable. + rampup: Optional[float] = None + # Max detailed request samples stored in benchmark JSON output (GuideLLM --sample-requests). sample_requests: int = 0 @@ -42,7 +45,11 @@ class BenchmarkConfig: logging_level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "INFO" def __post_init__(self) -> None: - for name, value in (("warmup", self.warmup), ("cooldown", self.cooldown)): + for name, value in ( + ("warmup", self.warmup), + ("cooldown", self.cooldown), + ("rampup", self.rampup), + ): if value is None: continue if value <= 0: diff --git a/auto_tune_vllm/benchmarks/providers.py b/auto_tune_vllm/benchmarks/providers.py index b3e6f3e..dcdd675 100644 --- a/auto_tune_vllm/benchmarks/providers.py +++ b/auto_tune_vllm/benchmarks/providers.py @@ -338,6 +338,8 @@ def _build_guidellm_command( cmd.extend(["--warmup", str(config.warmup)]) if config.cooldown is not None: cmd.extend(["--cooldown", str(config.cooldown)]) + if config.rampup is not None: + cmd.extend(["--rampup", str(config.rampup)]) # Add dataset or synthetic data configuration if config.use_synthetic_data: diff --git a/docs/configuration.md b/docs/configuration.md index 002b65a..f956d8e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -306,6 +306,14 @@ When both `warmup` and `cooldown` are fractional, their sum must stay below `1` #### `cooldown` (number, optional) GuideLLM cooldown period at the end of the run, also excluded from metrics. Same format as `warmup` (e.g. `0.1` for the last 10%). +#### `rampup` (number, optional) +GuideLLM ramp-up duration in seconds. Requests are spread linearly from zero up to the target concurrency (`rate`) over this period at the start of the benchmark. + +- **Seconds**: e.g. `rampup: 10` = 10 seconds to reach full concurrency. +- Omit or set to `null` to disable (GuideLLM default). + +Unlike `warmup`, ramp-up requests are included in reported metrics — it controls how load increases, not which phase is measured. See [GuideLLM benchmark docs](https://github.com/vllm-project/guidellm/blob/main/docs/getting-started/benchmark.md). + #### `sample_requests` (integer, optional) Maximum number of detailed request samples stored in GuideLLM benchmark JSON output (`--sample-requests`). Default: `0` (no per-request samples; keeps result files small). Set to a positive value when you need request-level timings for debugging or deeper analysis. Requires [GuideLLM](https://github.com/vllm-project/guidellm) `>= 0.5.4` (fix for `--sample-requests` in [v0.5.4](https://github.com/vllm-project/guidellm/releases/tag/v0.5.4)). @@ -318,6 +326,7 @@ benchmark: rate: 16 warmup: 0.1 cooldown: 0.1 + rampup: 10 # sample_requests: 20 # optional; default 0 ``` @@ -691,6 +700,7 @@ benchmark: rate: 100 # warmup: 0.1 # cooldown: 0.1 + # rampup: 10 # sample_requests: 0 # default; increase to keep detailed request samples in benchmark JSON logging: diff --git a/examples/study_config.yaml b/examples/study_config.yaml index 39e3cb9..9d0bba5 100644 --- a/examples/study_config.yaml +++ b/examples/study_config.yaml @@ -31,6 +31,7 @@ benchmark: # Optional: reduce metric variance (requires recent GuideLLM CLI) # warmup: 0.1 # cooldown: 0.1 + # rampup: 10 # sample_requests: 0 # default; increase to keep detailed request samples in benchmark JSON logging: diff --git a/examples/study_config_minimal.yaml b/examples/study_config_minimal.yaml index 676b9e4..36e3ac2 100644 --- a/examples/study_config_minimal.yaml +++ b/examples/study_config_minimal.yaml @@ -31,6 +31,7 @@ benchmark: output_tokens: 100 # warmup: 0.1 # cooldown: 0.1 + # rampup: 10 # sample_requests: 0 # default; increase to keep detailed request samples in benchmark JSON # No logging section specified - will use console logging only diff --git a/tests/benchmarks/test_guidellm_command.py b/tests/benchmarks/test_guidellm_command.py index ee83ba4..763c7b7 100644 --- a/tests/benchmarks/test_guidellm_command.py +++ b/tests/benchmarks/test_guidellm_command.py @@ -50,3 +50,23 @@ def test_fraction_warmup_plus_cooldown_must_leave_measured_window(): def test_mixed_fraction_and_absolute_skips_sum_check(): BenchmarkConfig(model="m", warmup=0.1, cooldown=10) + + +def test_rampup_included_when_set(): + cmd = _build_cmd(rampup=10) + assert cmd[cmd.index("--rampup") + 1] == "10" + + +def test_rampup_omitted_when_unset(): + cmd = _build_cmd() + assert "--rampup" not in cmd + + +def test_rampup_zero_rejected(): + with pytest.raises(ValueError, match="rampup"): + BenchmarkConfig(model="m", rampup=0) + + +def test_negative_rampup_rejected(): + with pytest.raises(ValueError, match="rampup"): + BenchmarkConfig(model="m", rampup=-1)