diff --git a/src/hyperloom/inference_optimizer/tests/golden/coordinator_behavior_golden.json b/src/hyperloom/inference_optimizer/tests/golden/coordinator_behavior_golden.json index 07fbe2e412..86f92d89bb 100644 --- a/src/hyperloom/inference_optimizer/tests/golden/coordinator_behavior_golden.json +++ b/src/hyperloom/inference_optimizer/tests/golden/coordinator_behavior_golden.json @@ -117,12 +117,10 @@ "workspace": null } ], - "baseline_cold_tput": 0.0, "baseline_config_path": "", "baseline_double_run": true, "baseline_eager_fallback": false, "baseline_failure_streak": 2, - "baseline_hot_tput": 0.0, "baseline_roofline_ceiling": {}, "baseline_runtime_sec": 0.0, "baseline_total_failures": 2, diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_methods_coverage_unit.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_methods_coverage_unit.py index 56c2a7b412..585b482274 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_methods_coverage_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_methods_coverage_unit.py @@ -63,8 +63,6 @@ async def test_promote_baseline_sets_anchor_and_current_best(coord: Coordinator) }, ) assert coord.shared_state.baseline_tput == 1000.0 - assert coord.shared_state.baseline_cold_tput == 900.0 - assert coord.shared_state.baseline_hot_tput == 1000.0 assert coord.shared_state.baseline_failure_streak == 0 assert coord.shared_state.baseline_arg_error_streak == 0 assert coord.shared_state.current_best["action"] == "baseline" diff --git a/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py b/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py index d1e38496f4..611adc9a66 100644 --- a/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py +++ b/src/hyperloom/inference_optimizer/tests/test_critic_verdict_map.py @@ -688,6 +688,102 @@ async def create_or_return_existing(self, **kwargs: Any): # noqa: ANN401 assert create_calls[0]["params"]["grid"] == grid +@pytest.mark.asyncio +async def test_delegate_explore_seeds_the_stack_with_the_anchor(tmp_path: Path): + """A delegate explore carries current_best's args/envs, not just its throughput. + + Seeding the anchor alone launched every variant on a bare config and graded it + against the established recipe, so a whole round read as a ~40% regression. + """ + coord = _delegate_coord(tmp_path) + coord.shared_state.baseline_tput = 4663.79 + coord.shared_state.current_best = { + "tput": 7725.6, + "extra_server_args": "--kv-cache-dtype fp8_e4m3 --max-num-seqs 64", + "extra_envs": {"VLLM_ROCM_USE_AITER": "1"}, + } + created: list[dict[str, Any]] = [] + + class _TaskRegistry: + async def create_or_return_existing(self, **kwargs: Any): # noqa: ANN401 + created.append(dict(kwargs["params"])) + from hyperloom.orchestrator.state.task_registry import Task + + return ( + Task( + task_id="t-explore-stack", + kind=kwargs["kind"], + state="queued", + params=kwargs["params"], + idempotency_key=kwargs["idempotency_key"], + ), + False, + ) + + coord.tasks = _TaskRegistry() + intent = Intent( + type=IntentType.DELEGATE, + payload={ + "action_name": "explore", + "params": {"grid": [{"name": "v_a", "extra_args": "--flag-a"}]}, + "idempotency_key": "explore-round-4", + }, + ) + await coord._handle_delegate("orchestration", intent) + assert len(created) == 1 + params = created[0] + assert params["base_tput"] == 7725.6 + assert params["base_extra_args"] == "--kv-cache-dtype fp8_e4m3 --max-num-seqs 64" + assert params["base_extra_envs"] == {"VLLM_ROCM_USE_AITER": "1"} + + +@pytest.mark.asyncio +async def test_delegate_sweep_seeds_the_stack_too(tmp_path: Path): + """A delegated sweep scans on top of current_best, as its action contract states. + + ``executors/sweep.py`` assembles each (CONC, ISL, OSL) point from the + ``base_*`` params, so seeding only ``explore`` left a delegated sweep + measuring the bare baseline config. + """ + coord = _delegate_coord(tmp_path) + coord.shared_state.baseline_tput = 4663.79 + coord.shared_state.current_best = { + "tput": 7725.6, + "extra_server_args": "--max-num-seqs 64", + "extra_envs": {"VLLM_ROCM_USE_AITER": "1"}, + } + created: list[dict[str, Any]] = [] + + class _TaskRegistry: + async def create_or_return_existing(self, **kwargs: Any): # noqa: ANN401 + created.append(dict(kwargs["params"])) + from hyperloom.orchestrator.state.task_registry import Task + + return ( + Task( + task_id="t-sweep-stack", + kind=kwargs["kind"], + state="queued", + params=kwargs["params"], + idempotency_key=kwargs["idempotency_key"], + ), + False, + ) + + coord.tasks = _TaskRegistry() + intent = Intent( + type=IntentType.DELEGATE, + payload={"action_name": "sweep", "params": {}, "idempotency_key": "sweep-1"}, + ) + await coord._handle_delegate("orchestration", intent) + assert len(created) == 1 + params = created[0] + assert params["base_extra_args"] == "--max-num-seqs 64" + assert params["base_extra_envs"] == {"VLLM_ROCM_USE_AITER": "1"} + # explore-only runtime knobs must not leak onto a sweep task. + assert "explore_overtime_kill_ratio" not in params + + # 6. Specialist prompt — proposal self-curation contract (Section 1 + 8) def _build_specialist_prompt_text() -> str: from hyperloom.orchestrator.specialists.domains import get_domain diff --git a/src/hyperloom/inference_optimizer/tests/test_explore_executor.py b/src/hyperloom/inference_optimizer/tests/test_explore_executor.py index 2653a71ac1..3a4b745785 100644 --- a/src/hyperloom/inference_optimizer/tests/test_explore_executor.py +++ b/src/hyperloom/inference_optimizer/tests/test_explore_executor.py @@ -53,6 +53,19 @@ def _isolate_leak_root(tmp_path_factory, monkeypatch): monkeypatch.setenv("INFERENCE_OPTIMIZER_LEAK_ROOTS", str(sandbox)) +def _force_cold_decision(monkeypatch) -> None: + """Make server_lifecycle reuse ineligible, one of the two warm-decision preconditions.""" + monkeypatch.setattr( + "hyperloom.orchestrator.actions.executors.explore.resolve_lifecycle_params", + lambda _config_path: { + "eligible": False, + "framework": "sglang", + "port": 30000, + "reason": "test: server_lifecycle reuse disabled", + }, + ) + + def _write_baseline_yaml(path: Path) -> None: cfg = { "benchmark": { @@ -814,10 +827,129 @@ def _fake_run(cmd, *args, **kwargs): assert tested["outcome"] == expected_outcome +@pytest.mark.asyncio +async def test_explore_executor_takes_live_base_args_with_the_live_anchor( + sub_agent_runner, + tmp_path, +): + """Superseding a stale ``base_tput`` also re-reads the args it was measured on.""" + sub, tr, _ = sub_agent_runner + state = SharedState() + state.baseline_tput = 800.0 + state.current_best = { + "action": "explore", + "tput": 1000.0, + "extra_server_args": "--live-layer 1", + "extra_envs": {"LIVE_ENV": "1"}, + } + sub.shared_state = state + + base = tmp_path / "base.yaml" + _write_baseline_yaml(base) + + def _fake_run(cmd, *args, **kwargs): + out_idx = cmd.index("--output-dir") + _fake_workspace(Path(cmd[out_idx + 1]), tput=1100.0) # +10% vs the live 1000 + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + + task = await tr.create( + kind="explore", + params={ + "config_path": str(base), + "output_dir": str(tmp_path / "explore-live-base"), + # Snapshotted together at dispatch, before the newer layer landed. + "base_tput": 800.0, + "base_extra_args": "--stale-layer 1", + "enable_stack_rebench": False, + "grid": [ + { + "name": "on_live_stack", + "extra_args": "--variant 2", + "extra_envs": {}, + "provenance": "llm_direct", + } + ], + "variant_timeout_sec": 10, + }, + idempotency_key="ex-live-base-args", + ) + sub.register_executor("explore", ExploreExecutor(session_dir=tmp_path, enable_stack_rebench=False)) + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_fake_run, + ): + res = await sub.run_task(task) + + out = res.result + fp = canonical_fingerprint("--variant 2", {}) + assert out["explore_search_update"]["tested"][fp]["base_tput"] == 1000.0 + winner = out["winners"][0] + assert "--live-layer 1" in winner["extra_server_args"] + assert "--variant 2" in winner["extra_server_args"] + assert "--stale-layer" not in winner["extra_server_args"] + assert winner["extra_envs"]["LIVE_ENV"] == "1" + + +@pytest.mark.asyncio +async def test_explore_stack_rebench_floor_follows_the_in_batch_anchor( + sub_agent_runner, + tmp_path, +): + """A 2nd KEEP that regresses against the 1st is evicted, not KEPT with a negative gain.""" + sub, tr, _ = sub_agent_runner + base = tmp_path / "base.yaml" + _write_baseline_yaml(base) + + def _fake_run(cmd, *args, **kwargs): + out_idx = cmd.index("--output-dir") + slot = Path(cmd[out_idx + 1]) + # Match on path segments: ``tmp_path`` is named after the test, so a + # substring check would fire on every round. + parts = set(slot.parts) + if "v01_second" in parts: + # Round 1 clears the advanced bar (1260 vs 1200); the confirmation + # round drops below it while still beating the round-start 1000. + tput = 1100.0 if "stack_rebench" in parts else 1260.0 + else: + tput = 1200.0 + _fake_workspace(slot, tput=tput) + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + + task = await tr.create( + kind="explore", + params={ + "config_path": str(base), + "output_dir": str(tmp_path / "explore-floor"), + "base_tput": 1000.0, + "grid": [ + {"name": "first", "extra_args": "--first", "extra_envs": {}, "provenance": "llm_direct"}, + {"name": "second", "extra_args": "--second", "extra_envs": {}, "provenance": "llm_direct"}, + ], + "variant_timeout_sec": 10, + }, + idempotency_key="ex-rebench-floor", + ) + sub.register_executor("explore", ExploreExecutor(session_dir=tmp_path)) + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_fake_run, + ): + res = await sub.run_task(task) + + out = res.result + tested = out["explore_search_update"]["tested"] + assert tested[canonical_fingerprint("--first", {})]["outcome"] == "KEEP" + assert tested[canonical_fingerprint("--second", {})]["outcome"] == "KEEP_UNSTABLE" + assert {w["name"] for w in out["winners"]} == {"first"} + assert all(w["gain_pct"] > 0 for w in out["winners"]) + # The evicted variant must not drag the stack anchor down with it. + assert out["running_base_tput"] == 1200.0 + + @pytest.mark.asyncio async def test_explore_executor_dedups_against_ledger(sub_agent_runner, tmp_path, monkeypatch): """A variant whose fingerprint already lives in explore_search.tested lands in ``skipped_dup``, not re-benched.""" - monkeypatch.setenv("INFERENCE_OPTIMIZER_EXPLORE_WARM_DECISION", "0") + _force_cold_decision(monkeypatch) sub, tr, _ = sub_agent_runner base = tmp_path / "base.yaml" _write_baseline_yaml(base) @@ -891,7 +1023,6 @@ async def test_explore_executor_defaults_to_warm_decision_matching_hot_baseline( monkeypatch, ): """Default EXPLORE measures hot decisions, matching default hot baseline.""" - monkeypatch.delenv("INFERENCE_OPTIMIZER_EXPLORE_WARM_DECISION", raising=False) sub, tr, _ = sub_agent_runner base = tmp_path / "base.yaml" _write_baseline_yaml(base) @@ -959,7 +1090,6 @@ async def test_explore_decision_round_skips_eval_warmup_keeps_it( """The overtime deadline is anchored on a throughput-only baseline, so the rounds it gates must measure throughput only. The warmup round is ungated and remains the accuracy source.""" - monkeypatch.delenv("INFERENCE_OPTIMIZER_EXPLORE_WARM_DECISION", raising=False) sub, tr, _ = sub_agent_runner base = tmp_path / "base.yaml" _write_baseline_yaml(base) @@ -1016,9 +1146,9 @@ async def test_explore_cold_decision_keeps_eval( tmp_path, monkeypatch, ): - """With warm-decision disabled there is no warmup eval to fall back on, so - the decision round must still run its own accuracy gate.""" - monkeypatch.setenv("INFERENCE_OPTIMIZER_EXPLORE_WARM_DECISION", "0") + """Without server_lifecycle reuse there is no warmup round whose eval the + decision round could fall back on, so it must run its own accuracy gate.""" + _force_cold_decision(monkeypatch) sub, tr, _ = sub_agent_runner base = tmp_path / "base.yaml" _write_baseline_yaml(base) @@ -1065,6 +1195,55 @@ def _fake_run(cmd, *args, **kwargs): assert decision and all(ev not in _RUN_EVAL_FALSE for ev in decision) +@pytest.mark.asyncio +async def test_explore_decision_stays_cold_when_the_session_skips_the_double_run( + sub_agent_runner, + tmp_path, +): + """A cold ``baseline_tput`` must be graded cold even when lifecycle reuse is available. + + The baseline gates its cold+hot double run on ``baseline_double_run`` as well + as lifecycle eligibility, so warm-decision has to honour both or a hot + candidate is scored against a cold anchor. + """ + sub, tr, _ = sub_agent_runner + state = SharedState() + state.baseline_tput = 800.0 + state.baseline_double_run = False + sub.shared_state = state + + base = tmp_path / "base.yaml" + _write_baseline_yaml(base) + seen: list[str] = [] + + def _fake_run(cmd, *args, **kwargs): + out_idx = cmd.index("--output-dir") + slot = Path(cmd[out_idx + 1]) + seen.append(str(slot)) + _fake_workspace(slot, tput=920.0) + return subprocess.CompletedProcess(args=cmd, returncode=0, stdout="ok", stderr="") + + task = await tr.create( + kind="explore", + params={ + "config_path": str(base), + "output_dir": str(tmp_path / "explore-singleround"), + "base_tput": 800.0, + "grid": [{"name": "v", "extra_args": "--flag", "extra_envs": {}, "provenance": "llm_direct"}], + "variant_timeout_sec": 30, + }, + idempotency_key="ex-no-double-run", + ) + sub.register_executor("explore", ExploreExecutor(session_dir=tmp_path)) + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_fake_run, + ): + await sub.run_task(task) + + assert not [slot for slot in seen if "warmup_round" in slot] + + @pytest.mark.asyncio async def test_explore_executor_warm_decision_warmup_failure_marks_failed( sub_agent_runner, @@ -1072,7 +1251,6 @@ async def test_explore_executor_warm_decision_warmup_failure_marks_failed( monkeypatch, ): """A failed warmup round records the variant FAILED(reason=warmup_failed), no decision run.""" - monkeypatch.setenv("INFERENCE_OPTIMIZER_EXPLORE_WARM_DECISION", "1") sub, tr, _ = sub_agent_runner base = tmp_path / "base.yaml" _write_baseline_yaml(base) @@ -1221,7 +1399,7 @@ async def test_explore_executor_killed_overtime_no_tput_no_keep( monkeypatch, ): """A fired soft deadline records KILLED_OVERTIME (no tput, no KEEP/REVERT, stack unchanged).""" - monkeypatch.setenv("INFERENCE_OPTIMIZER_EXPLORE_WARM_DECISION", "0") + _force_cold_decision(monkeypatch) sub, tr, _ = sub_agent_runner base = tmp_path / "base.yaml" _write_baseline_yaml(base) diff --git a/src/hyperloom/inference_optimizer/tests/test_promote_shared_state_lock.py b/src/hyperloom/inference_optimizer/tests/test_promote_shared_state_lock.py index a1a57bdea5..b36f11d005 100644 --- a/src/hyperloom/inference_optimizer/tests/test_promote_shared_state_lock.py +++ b/src/hyperloom/inference_optimizer/tests/test_promote_shared_state_lock.py @@ -153,10 +153,8 @@ async def test_promote_baseline_writes_state_and_audit(session_dir): task=_task("baseline"), ) - # Hot-measure contract: baseline_tput/hot from tput, cold from warmup anchor. + # Hot-measure contract: baseline_tput is the hot round, not the cold warmup. assert s.baseline_tput == 100.0 - assert s.baseline_hot_tput == 100.0 - assert s.baseline_cold_tput == 80.0 assert s.baseline_accuracy == 0.9 assert s.baseline_runtime_sec == 30.0 assert s.current_best["action"] == "baseline" @@ -930,3 +928,41 @@ def test_lift_accepts_winner_that_beats_current_best(session_dir): assert lifted is True assert s.current_best["tput"] == 1100.0 assert s.optimization_stack[-1]["variant_name"] == "real-win" + + +async def test_integrate_keep_carries_the_stack_env_layer(session_dir): + """A kernel integrate publishes args and envs from the same config. + + Writing ``current_best`` without ``extra_envs`` published a config whose args + and envs came from different layers, and every dispatch site seeded from it. + """ + coord = _coord(session_dir) + s = coord.shared_state + s.baseline_tput = 1000.0 + s.current_best = { + "action": "explore", + "tput": 1000.0, + "extra_server_args": "--kv-cache-dtype fp8_e4m3", + "extra_envs": {"VLLM_ROCM_USE_AITER": "1"}, + } + + await coord.writeback._record_integrate_keep( + {"new_tput": 1200.0, "kernel_id": "k001", "integration_id": "i1"}, + ) + + assert s.current_best["extra_envs"] == {"VLLM_ROCM_USE_AITER": "1"} + assert s.current_best["extra_server_args"] == "--kv-cache-dtype fp8_e4m3" + + +async def test_integrate_keep_lets_a_tuning_env_delta_win(session_dir): + """A forge-GEMM KEEP ships ``result['extra_envs']``; it must survive the promote.""" + coord = _coord(session_dir) + s = coord.shared_state + s.baseline_tput = 1000.0 + s.current_best = {"action": "explore", "tput": 1000.0, "extra_envs": {"KEEP_ME": "1", "TUNED": "old"}} + + await coord.writeback._record_integrate_keep( + {"new_tput": 1200.0, "kernel_id": "k002", "extra_envs": {"TUNED": "new"}}, + ) + + assert s.current_best["extra_envs"] == {"KEEP_ME": "1", "TUNED": "new"} diff --git a/src/hyperloom/inference_optimizer/tests/test_shared_state_persistence.py b/src/hyperloom/inference_optimizer/tests/test_shared_state_persistence.py index b0089f58f6..c98b11048b 100644 --- a/src/hyperloom/inference_optimizer/tests/test_shared_state_persistence.py +++ b/src/hyperloom/inference_optimizer/tests/test_shared_state_persistence.py @@ -57,8 +57,6 @@ def test_save_load_round_trip(tmp_path): session_id="abc", model_name="meta-llama/Llama-3.1-8B-Instruct", baseline_tput=1840.0, - baseline_cold_tput=1600.0, - baseline_hot_tput=1840.0, cumulative_gain=12.5, pruned_families=["deep_kernel"], current_best={"action": "backends", "tput": 2010.0}, @@ -70,8 +68,6 @@ def test_save_load_round_trip(tmp_path): assert s2.session_id == "abc" assert s2.model_name == "meta-llama/Llama-3.1-8B-Instruct" assert s2.baseline_tput == 1840.0 - assert s2.baseline_cold_tput == 1600.0 - assert s2.baseline_hot_tput == 1840.0 assert s2.cumulative_gain == 12.5 assert s2.pruned_families == ["deep_kernel"] assert s2.current_best == {"action": "backends", "tput": 2010.0} diff --git a/src/hyperloom/inference_optimizer/tests/test_shared_state_units.py b/src/hyperloom/inference_optimizer/tests/test_shared_state_units.py index 4f5c723f05..1e3166ace3 100644 --- a/src/hyperloom/inference_optimizer/tests/test_shared_state_units.py +++ b/src/hyperloom/inference_optimizer/tests/test_shared_state_units.py @@ -11,6 +11,7 @@ _DEFAULT_ATTEMPTS_HISTORY, _DEFAULT_LAST_FAILURES, SharedState, + inject_stack_base_params, resolve_grading_anchor_tput, ) @@ -41,6 +42,73 @@ def test_zero_when_nothing_established(self): assert resolve_grading_anchor_tput(SharedState()) == 0.0 +class TestInjectStackBaseParams: + @staticmethod + def _state(): + s = SharedState() + s.baseline_tput = 800.0 + s.current_best = { + "tput": 1000.0, + "extra_server_args": "--live-layer 1", + "extra_envs": {"LIVE_ENV": "1"}, + "remove_args": "--dropped", + "args_mode": "replace", + } + return s + + def test_seeds_the_anchor_together_with_its_config(self): + params: dict = {} + inject_stack_base_params(params, self._state(), anchor=True) + assert params == { + "base_tput": 1000.0, + "base_extra_args": "--live-layer 1", + "base_extra_envs": {"LIVE_ENV": "1"}, + "base_remove_args": ["--dropped"], + "base_args_mode": "replace", + } + + def test_omits_the_anchor_unless_asked(self): + params: dict = {} + inject_stack_base_params(params, self._state()) + assert "base_tput" not in params + assert params["base_extra_args"] == "--live-layer 1" + + def test_keeps_a_caller_supplied_value(self): + params = {"base_extra_args": "--operator-pinned"} + inject_stack_base_params(params, self._state(), anchor=True) + assert params["base_extra_args"] == "--operator-pinned" + + def test_overwrite_replaces_a_superseded_layer(self): + params = {"base_extra_args": "--stale-layer 1", "base_tput": 800.0} + state = self._state() + state.current_best = {"tput": 1000.0, "extra_server_args": "", "extra_envs": {}} + inject_stack_base_params(params, state, anchor=True, overwrite=True) + assert params["base_tput"] == 1000.0 + assert params["base_extra_args"] == "" + assert params["base_extra_envs"] == {} + + def test_skips_fields_current_best_does_not_carry(self): + params: dict = {} + state = SharedState() + state.baseline_tput = 800.0 + state.current_best = {"action": "baseline", "tput": 800.0} + inject_stack_base_params(params, state, anchor=True) + assert params == {"base_tput": 800.0} + + def test_falls_back_to_the_baseline_anchor_with_no_stack(self): + params: dict = {} + state = SharedState() + state.baseline_tput = 800.0 + inject_stack_base_params(params, state, anchor=True) + assert params == {"base_tput": 800.0} + + @pytest.mark.parametrize("state", [None, object()]) + def test_tolerates_missing_state(self, state): + params: dict = {} + inject_stack_base_params(params, state, anchor=True) + assert params == {} + + class TestGridSessionDeadline: def test_returns_none_when_budget_unbounded(self): s = SharedState() diff --git a/src/hyperloom/inference_optimizer/tests/test_sweep_phase_auto.py b/src/hyperloom/inference_optimizer/tests/test_sweep_phase_auto.py index 1d03830d3d..ed44acf9c4 100644 --- a/src/hyperloom/inference_optimizer/tests/test_sweep_phase_auto.py +++ b/src/hyperloom/inference_optimizer/tests/test_sweep_phase_auto.py @@ -777,6 +777,25 @@ async def test_enqueue_internal_sweep_task_omits_empty_strings(coord): assert "benchmark_script" not in task.params +@pytest.mark.asyncio +async def test_enqueue_internal_sweep_task_carries_the_arg_mode_controls(coord): + """A ``replace`` current_best reaches the sweep, which assembles variants from it. + + ``_build_grid`` honours ``base_args_mode`` / ``base_remove_args``, so omitting + them made the sweep append onto flags the champion had deliberately replaced. + """ + coord.shared_state.current_best = { + "tput": 1000.0, + "extra_server_args": "--mla 1", + "remove_args": ["--chunked-prefill-size"], + "args_mode": "replace", + } + task = await coord._enqueue_internal_sweep_task(reason="phase_entry") + assert task.params["base_extra_args"] == "--mla 1" + assert task.params["base_remove_args"] == ["--chunked-prefill-size"] + assert task.params["base_args_mode"] == "replace" + + @pytest.mark.asyncio async def test_enqueue_internal_sweep_task_recipe_kb_recipe_propagates(coord): """Recipe-driven grid surfaces as source='recipe_kb' on the task.""" diff --git a/src/hyperloom/orchestrator/actions/executors/explore.py b/src/hyperloom/orchestrator/actions/executors/explore.py index 2aff741050..97a8af2101 100644 --- a/src/hyperloom/orchestrator/actions/executors/explore.py +++ b/src/hyperloom/orchestrator/actions/executors/explore.py @@ -40,7 +40,7 @@ from hyperloom.common.gain_math import gain_pct from hyperloom.common.timeutil import now_iso from hyperloom.inference_optimizer.session.session_paths import runs_dir -from ...state.shared_state import resolve_grading_anchor_tput +from ...state.shared_state import first_positive_tput, resolve_grading_anchor_tput, stack_base_params from ._accuracy_gate import ( accuracy_passed, is_high_accuracy_risk, @@ -589,24 +589,27 @@ async def __call__(self, ctx) -> dict[str, Any]: } # ----- Inputs ------------------------------------------------------ + # Params snapshot the anchor and the stack it was measured on together; + # a KEEP landing while this task queued invalidates both, so refresh them + # as a pair. Revalidation reproduces the saved stack, so it never re-reads. + ss = extra.get("shared_state") or extra.get("state") + snapshot_tput = float(params.get("base_tput") or 0.0) + live_anchor = 0.0 if params.get("source") == "resume_stack_revalidate" else resolve_grading_anchor_tput(ss) + if live_anchor > snapshot_tput: + if snapshot_tput > 0: + log.warning("explore: anchor drift %.1f -> %.1f; re-reading base args", snapshot_tput, live_anchor) + params["base_tput"] = live_anchor + cb = getattr(ss, "current_best", None) + # Only current_best carries args; a baseline_tput anchor leaves the + # params stack (seeded from the baseline record) authoritative. + if first_positive_tput(cb) > 0: + params.update(stack_base_params(cb)) base_extra_args = str(params.get("base_extra_args") or "").strip() base_extra_envs = dict(params.get("base_extra_envs") or {}) base_remove_args = to_str_list(params.get("base_remove_args")) base_unset_envs = to_str_list(params.get("base_unset_envs")) base_args_mode = str(params.get("base_args_mode") or "append").strip().lower() base_tput = float(params.get("base_tput") or 0.0) - # Grade candidates against the live anchor; revalidation uses baseline. - ss = extra.get("shared_state") or extra.get("state") - live_anchor = resolve_grading_anchor_tput(ss) - revalidating_stack = params.get("source") == "resume_stack_revalidate" - if not revalidating_stack and live_anchor > base_tput: - if base_tput > 0: - log.warning( - "explore: anchor drift, params base_tput=%.1f but live anchor is %.1f; grading against live", - base_tput, - live_anchor, - ) - base_tput = live_anchor baseline_accuracy = float(params.get("accuracy_baseline") or 0.0) or float( params.get("baseline_accuracy") or 0.0 ) @@ -958,8 +961,6 @@ def _is_blocked(entry: Any) -> bool: # In-batch KEEP'd entries (for full vs incremental stack recompose). in_batch_keeps: list[dict[str, Any]] = [] - last_run_tput: float | None = None # rebench/single-variant tput - # Single-node server_lifecycle eligibility (multi-node / non-builtin # script / profiler-on falls back to a fresh cold boot for round 2). lifecycle = resolve_lifecycle_params(config_path) @@ -969,13 +970,11 @@ def _is_blocked(entry: Any) -> bool: # Warm-decision mode. Run a discarded cold warmup round first so the # decision round reuses the hot server (client-only) and is measured - # warm — apples-to-apples with ``baseline_tput``. Requires - # server_lifecycle reuse; otherwise fall back to a single cold-decision - # run. Opt out with INFERENCE_OPTIMIZER_EXPLORE_WARM_DECISION=0. - warm_decision_enabled = os.environ.get( - "INFERENCE_OPTIMIZER_EXPLORE_WARM_DECISION", "1" - ).strip().lower() not in {"0", "false", "no", "off"} - use_warm_decision = warm_decision_enabled and lifecycle_eligible + # warm — apples-to-apples with ``baseline_tput``. Mirrors both conjuncts + # the baseline gates its cold+hot double-run on, so the two sides measure + # hot together or cold together: a session that opted out of the double + # run has a COLD ``baseline_tput`` and must be graded cold. + use_warm_decision = lifecycle_eligible and bool(getattr(ss, "baseline_double_run", True)) # Decision-round overtime anchor: the WARM measure time when warm-decision # is active and available, else the cold baseline wall-clock (legacy). decision_anchor_sec = ( @@ -1344,7 +1343,7 @@ def _is_blocked(entry: Any) -> bool: else: outcome = "KEEP" - cold_tput = r.output_throughput + decision_tput = r.output_throughput tested_update[fp] = { "fingerprint": fp, "name": gv.name, @@ -1354,8 +1353,8 @@ def _is_blocked(entry: Any) -> bool: "note": gv.note, "outcome": outcome, "status": r.status, - "tput": cold_tput, - "cold_tput": cold_tput, + "tput": decision_tput, + "decision_tput": decision_tput, "gain_pct": gain, "base_tput": running_base_tput, "round_id": round_id, @@ -1418,8 +1417,8 @@ def _is_blocked(entry: Any) -> bool: "note": gv.note, "provenance": provenance, "gain_pct": gain, - "tput": cold_tput, - "cold_tput": cold_tput, + "tput": decision_tput, + "decision_tput": decision_tput, "single_workspace": r.workspace, "round_id": round_id, "accepted_at_round": round_id, @@ -1431,7 +1430,7 @@ def _is_blocked(entry: Any) -> bool: stack_rebench_workspace: str | None = None stack_rebench_warnings: list[str] = [] - if enable_stack_rebench and base_tput > 0: + if enable_stack_rebench and running_base_tput > 0: # Round 2: same config as round 1. When eligible, # reuse round 1's hot server (cleanup=true tears it # down) so the measurement is warm and baseline- @@ -1466,7 +1465,9 @@ def _is_blocked(entry: Any) -> bool: config_path=config_path, base_extra_args=stack_extra_args, variant=rebench_variant, - base_tput=base_tput, + # Floor sits on the anchor round 1 graded against, + # which advances with each in-batch KEEP. + base_tput=running_base_tput, stable_threshold_pct=stack_stable_threshold_pct, output_slot=slot / "stack_rebench", variant_timeout_sec=timeout_sec, @@ -1490,11 +1491,11 @@ def _is_blocked(entry: Any) -> bool: log.warning( "explore: variant %s KEEP -> KEEP_UNSTABLE " "(stack_rebench_tput=%s vs stable_floor=%.2f " - "with base_tput=%.2f * (1+%.2f%%))", + "with running_base_tput=%.2f * (1+%.2f%%))", gv.name, stack_rebench_tput, stable_floor, - base_tput, + running_base_tput, stack_stable_threshold_pct, ) tested_update[fp]["outcome"] = "KEEP_UNSTABLE" @@ -1519,7 +1520,7 @@ def _is_blocked(entry: Any) -> bool: "note": gv.note, "reason": "stack_unstable", "gain_pct": gain, - "tput": cold_tput, + "tput": decision_tput, "stack_rebench_tput": stack_rebench_tput, "round_id": round_id, "ts": _now_iso(), @@ -1540,7 +1541,6 @@ def _is_blocked(entry: Any) -> bool: stack_unset_envs = list(run_unset_envs) stack_base_args_mode = "replace" if persist_effective_args else "append" running_base_tput = stack_rebench_tput - last_run_tput = stack_rebench_tput keep_entry["gain_pct"] = gain keep_entry["tput"] = stack_rebench_tput keep_entry["stack_rebench_tput"] = stack_rebench_tput @@ -1559,8 +1559,7 @@ def _is_blocked(entry: Any) -> bool: stack_remove_args = list(run_remove_args) stack_unset_envs = list(run_unset_envs) stack_base_args_mode = "replace" if persist_effective_args else "append" - running_base_tput = cold_tput or running_base_tput - last_run_tput = cold_tput + running_base_tput = decision_tput or running_base_tput winners.append(keep_entry) winners_history_update.append( @@ -1590,7 +1589,7 @@ def _is_blocked(entry: Any) -> bool: "note": gv.note, "reason": reason or "not_keep", "gain_pct": gain, - "tput": cold_tput, + "tput": decision_tput, "round_id": round_id, "ts": _now_iso(), "provenance": provenance, @@ -1605,13 +1604,11 @@ def _is_blocked(entry: Any) -> bool: **control_fields, "provenance": provenance, "gain_pct": gain, - "tput": cold_tput, + "tput": decision_tput, "reason": reason or "not_keep", "workspace": r.workspace, } ) - if cold_tput: - last_run_tput = cold_tput finally: # Reap THIS variant's persistent server on every exit path # (idempotent + no-op when reuse was ineligible). This is a @@ -1753,10 +1750,8 @@ def _is_blocked(entry: Any) -> bool: ) best_gain_pct = float(best_winner.get("gain_pct") or 0.0) if best_winner else 0.0 - if winners: - output_throughput = float(running_base_tput) if last_run_tput is not None else None - else: - output_throughput = None + # Each KEEP advances ``running_base_tput``, so this is the final stack. + output_throughput = float(running_base_tput) if winners else None # Successful = at least one bench produced a measurement or was reaped # by the overtime gate (KILLED_OVERTIME is a real signal). diff --git a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py index d9fc0f0f26..dd4e93875b 100644 --- a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py @@ -24,7 +24,7 @@ from hyperloom.inference_optimizer.session.session_paths import runs_dir from ...framework.paths import resolve_source_file_allowlist from ...specialists.patch_safety import patch_file_targets, patch_targets_missing -from ...state.shared_state import resolve_grading_anchor_tput +from ...state.shared_state import inject_stack_base_params, resolve_grading_anchor_tput from ._accuracy_gate import ( DEFAULT_ENABLEMENT_ACCURACY_FLOOR, accuracy_keep_block, @@ -1548,25 +1548,10 @@ async def _stage_resolve( "the patches to integrate)" ), } - # Rebind execution-time base from live current_best so a task queued - # before an Explore KEEP always measures against the real stack top. - # (``shared_state`` and the accuracy_baseline fill are already resolved above.) + # Rebind from live current_best so a task queued before an Explore KEEP + # still measures against the real stack top. if shared_state is not None: - cb = getattr(shared_state, "current_best", None) - if isinstance(cb, dict): - cb_tput = cb.get("tput") - if isinstance(cb_tput, (int, float)) and cb_tput > 0: - params["base_tput"] = float(cb_tput) - cb_args = str(cb.get("extra_server_args") or "").strip() - if cb_args: - params["base_extra_args"] = cb_args - cb_envs = {str(k): str(v) for k, v in (cb.get("extra_envs") or {}).items()} - if cb_envs: - params["base_extra_envs"] = cb_envs - for _ctrl in ("remove_args", "unset_envs", "args_mode"): - cb_ctrl = cb.get(_ctrl) - if cb_ctrl and not params.get(f"base_{_ctrl}"): - params[f"base_{_ctrl}"] = cb_ctrl + inject_stack_base_params(params, shared_state, anchor=True, overwrite=True) # Specialist workspace conventionally at runs/specialist//. specialist_workspace = runs_dir(self.session_dir, "specialist", specialist_task_id) if not specialist_workspace.is_dir(): diff --git a/src/hyperloom/orchestrator/loop/intent_router.py b/src/hyperloom/orchestrator/loop/intent_router.py index 2e7c524587..ed0d04fe7d 100644 --- a/src/hyperloom/orchestrator/loop/intent_router.py +++ b/src/hyperloom/orchestrator/loop/intent_router.py @@ -38,7 +38,7 @@ PRUNE_BRANCH_SCOPE_QUEUED, SPECIALIST_FROM_AGENT_PREFIX, ) -from ..state.shared_state import resolve_grading_anchor_tput +from ..state.shared_state import inject_stack_base_params from ..state.task_registry import IllegalTransition, TaskNotFound from ..kernel.request_handlers import get_handler @@ -406,10 +406,12 @@ async def _handle_delegate(self, source: str, intent: Intent) -> None: # Plumb baseline's materialized YAML into grid-style tasks (delegator may override). if action_name in ("sweep", "explore") and self.shared_state.baseline_config_path: params.setdefault("config_path", self.shared_state.baseline_config_path) - # Parity with _materialize_approved_proposal: direct delegates need the same knobs. + # Delegates skip _materialize_approved_proposal, so seed the same params here. + # Both grid actions launch on top of current_best per their action contract. + if action_name in ("sweep", "explore"): + inject_stack_base_params(params, self.shared_state, anchor=True) if action_name == "explore": self._inject_explore_runtime_params(params) - params.setdefault("base_tput", resolve_grading_anchor_tput(self.shared_state)) # Wave sugar: a specialist delegate carrying params.tasks=[...] fans out # into N standard freeform specialist tasks, each dispatched through the # normal SpecialistRunner + TaskRegistry + lease + reap path. diff --git a/src/hyperloom/orchestrator/loop/proposals.py b/src/hyperloom/orchestrator/loop/proposals.py index 23214ef74d..1e6d6678c7 100644 --- a/src/hyperloom/orchestrator/loop/proposals.py +++ b/src/hyperloom/orchestrator/loop/proposals.py @@ -11,7 +11,7 @@ from ..phases import machine_state as _phase_state from ..bus.message_bus import Message from .coordinator_helpers import approved_proposal_idempotency_key -from ..state.shared_state import resolve_grading_anchor_tput +from ..state.shared_state import inject_stack_base_params if TYPE_CHECKING: from ..state.task_registry import Task @@ -395,7 +395,7 @@ async def _materialize_approved_proposal( *, approved_variant_names: set[str] | None = None, ) -> None: - """Promote an approved proposal into a TaskRegistry entry. Grid executors get current best tput as base_tput; approved_variant_names filters the explore grid (None keeps full). + """Promote an approved proposal into a TaskRegistry entry. Stack-aware actions get current_best's anchor and the base config it was measured on; approved_variant_names filters the explore grid (None keeps full). Args: pending: The approved proposal to materialise into a task. @@ -437,52 +437,16 @@ async def _materialize_approved_proposal( 0, original_grid_len - len(approved_variant_names), ) - cb = self.shared_state.current_best or {} - cb_args = str(cb.get("extra_server_args") or "") if isinstance(cb, dict) else "" - # Cumulative env base for stack-aware actions; without it explore stacks - # args but not envs and current_best.extra_envs collapses to the last delta. - cb_envs = {str(k): str(v) for k, v in (cb.get("extra_envs") or {}).items()} if isinstance(cb, dict) else {} - - def _list_control(value: Any) -> list[str]: - if isinstance(value, str): - return [value] if value.strip() else [] - return [str(v) for v in (value or []) if str(v).strip()] - - cb_remove_args = _list_control(cb.get("remove_args")) if isinstance(cb, dict) else [] - cb_unset_envs = _list_control(cb.get("unset_envs")) if isinstance(cb, dict) else [] - cb_args_mode = str(cb.get("args_mode") or "").strip().lower() if isinstance(cb, dict) else "" if pending.action_name == "profile": # Stamp the server config that produced this trace. - params.setdefault("base_extra_args", cb_args) - if cb_remove_args: - params.setdefault("base_remove_args", cb_remove_args) - if cb_unset_envs: - params.setdefault("base_unset_envs", cb_unset_envs) - if cb_args_mode == "replace": - params.setdefault("base_args_mode", "replace") + inject_stack_base_params(params, self.shared_state) if pending.action_name == "sweep": - params.setdefault("base_tput", resolve_grading_anchor_tput(self.shared_state)) - params.setdefault("base_extra_args", cb_args) - if cb_remove_args: - params.setdefault("base_remove_args", cb_remove_args) - if cb_unset_envs: - params.setdefault("base_unset_envs", cb_unset_envs) - if cb_args_mode == "replace": - params.setdefault("base_args_mode", "replace") + inject_stack_base_params(params, self.shared_state, anchor=True) if self.shared_state.baseline_config_path: params.setdefault("config_path", self.shared_state.baseline_config_path) if pending.action_name == "explore": self._inject_explore_runtime_params(params) - params.setdefault("base_tput", resolve_grading_anchor_tput(self.shared_state)) - params.setdefault("base_extra_args", cb_args) - if cb_envs: - params.setdefault("base_extra_envs", dict(cb_envs)) - if cb_remove_args: - params.setdefault("base_remove_args", cb_remove_args) - if cb_unset_envs: - params.setdefault("base_unset_envs", cb_unset_envs) - if cb_args_mode == "replace": - params.setdefault("base_args_mode", "replace") + inject_stack_base_params(params, self.shared_state, anchor=True) if pending.action_name == "integrate_patch": keep = self._decaying_keep_threshold_pct() if keep is not None: @@ -490,14 +454,7 @@ def _list_control(value: Any) -> list[str]: # Seed the patched-eval server with the same base args/config every # other eval server uses, else it launches on bare framework defaults # and crashes at startup regardless of the patch. - params.setdefault("base_tput", resolve_grading_anchor_tput(self.shared_state)) - params.setdefault("base_extra_args", cb_args) - if cb_remove_args: - params.setdefault("base_remove_args", cb_remove_args) - if cb_unset_envs: - params.setdefault("base_unset_envs", cb_unset_envs) - if cb_args_mode == "replace": - params.setdefault("base_args_mode", "replace") + inject_stack_base_params(params, self.shared_state, anchor=True) if self.shared_state.baseline_config_path: params.setdefault("config_path", self.shared_state.baseline_config_path) lanes, ttl = self._registry_lanes_ttl(pending.action_name) diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index d770683a09..45707043f1 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -268,6 +268,13 @@ async def _record_integrate_keep(self, result: dict[str, Any]) -> None: str(result.get("extra_server_args") or "") or (str(cb.get("extra_server_args") or "") if isinstance(cb, dict) else "") ).strip() + # An integrate carries no env delta of its own except a forge-GEMM tuning + # KEEP, so inherit the stack's env layer and let that delta win. Dropping + # it published a current_best whose args and envs came from different + # configs, which every dispatch site then seeded from. + extra_envs = dict((cb.get("extra_envs") or {}) if isinstance(cb, dict) else {}) + if isinstance(result.get("extra_envs"), dict): + extra_envs.update({str(k): str(v) for k, v in result["extra_envs"].items()}) apply_result = result.get("apply_result") or {} backup_manifest = apply_result.get("manifest_path") if isinstance(apply_result, dict) else None if not backup_manifest and isinstance(apply_result, dict): @@ -331,6 +338,7 @@ async def _record_integrate_keep(self, result: dict[str, Any]) -> None: "integration_id": result.get("integration_id"), "kernel_id": result.get("kernel_id"), "extra_server_args": extra_args, + "extra_envs": extra_envs, "optimization_stack": list(self.shared_state.optimization_stack), "ttft_mean_ms": result.get("ttft_mean_ms"), "e2el_mean_ms": result.get("e2el_mean_ms"), @@ -2383,22 +2391,9 @@ async def _promote_baseline( ) if isinstance(tput, (int, float)) and tput > 0: if anchor_accepted: - # Baseline's conclusion contract is the hot measure round; the - # discarded cold round is kept only as an audit field so gain math - # never mixes cold-before with hot-after. - if isinstance(warmup_anchor, (int, float)) and warmup_anchor > 0: - self.shared_state.baseline_tput = float(tput) - self.shared_state.baseline_cold_tput = float(warmup_anchor) - self.shared_state.baseline_hot_tput = float(tput) - log.info( - "baseline anchor: using hot measure tput %.1f as " - "baseline_tput (discarded cold warmup %.1f kept as " - "baseline_cold_tput)", - float(tput), - float(warmup_anchor), - ) - else: - self.shared_state.baseline_tput = float(tput) + # The anchor is the hot measure round; the cold warmup round is + # discarded so gain math never mixes cold-before with hot-after. + self.shared_state.baseline_tput = float(tput) else: log.info( "baseline anchor: keeping %.1f; re-baseline measured %.1f " diff --git a/src/hyperloom/orchestrator/phases/explore.py b/src/hyperloom/orchestrator/phases/explore.py index 651c43efc9..adeaad1740 100644 --- a/src/hyperloom/orchestrator/phases/explore.py +++ b/src/hyperloom/orchestrator/phases/explore.py @@ -21,7 +21,7 @@ ) from ..loop.sub_agent_runner import SubAgentResult from ..specialists.runner import SpecialistFailureType -from ..state.shared_state import resolve_grading_anchor_tput +from ..state.shared_state import inject_stack_base_params from ..state.task_registry import Task from ..loop.coordinator import ( FORCE_STALLED_KEEP_ROUNDS, @@ -1410,32 +1410,7 @@ async def _maybe_materialize_mn_explore( } if state.baseline_config_path: params["config_path"] = state.baseline_config_path - cb = state.current_best or {} - if isinstance(cb, dict): - cb_args = str(cb.get("extra_server_args") or "") - if cb_args: - params["base_extra_args"] = cb_args - _raw_remove = cb.get("remove_args") - _raw_unset = cb.get("unset_envs") - cb_remove = ( - [_raw_remove] - if isinstance(_raw_remove, str) and _raw_remove.strip() - else [str(v) for v in (_raw_remove or []) if str(v).strip()] - ) - cb_unset = ( - [_raw_unset] - if isinstance(_raw_unset, str) and _raw_unset.strip() - else [str(v) for v in (_raw_unset or []) if str(v).strip()] - ) - if cb_remove: - params["base_remove_args"] = cb_remove - if cb_unset: - params["base_unset_envs"] = cb_unset - if str(cb.get("args_mode") or "").strip().lower() == "replace": - params["base_args_mode"] = "replace" - base_tput = resolve_grading_anchor_tput(state) - if base_tput: - params["base_tput"] = base_tput + inject_stack_base_params(params, state, anchor=True) last_bl = state.last_baseline or {} if isinstance(last_bl, dict): bs = str(last_bl.get("benchmark_script") or "").strip() diff --git a/src/hyperloom/orchestrator/phases/framework.py b/src/hyperloom/orchestrator/phases/framework.py index d8af2f874c..ab034cb010 100644 --- a/src/hyperloom/orchestrator/phases/framework.py +++ b/src/hyperloom/orchestrator/phases/framework.py @@ -18,7 +18,7 @@ from . import machine_state as _phase_state from ..bus.message_bus import Message from ..actions.executors._accuracy_gate import ENABLEMENT_REVALIDATION_REASON -from ..state.shared_state import resolve_grading_anchor_tput +from ..state.shared_state import inject_stack_base_params, resolve_grading_anchor_tput if TYPE_CHECKING: from ..state.task_registry import Task @@ -5101,32 +5101,7 @@ def _framework_config_explore_params( } if state.baseline_config_path: params["config_path"] = state.baseline_config_path - cb = state.current_best or {} - if isinstance(cb, dict): - cb_args = str(cb.get("extra_server_args") or "") - if cb_args: - params["base_extra_args"] = cb_args - _raw_remove = cb.get("remove_args") - _raw_unset = cb.get("unset_envs") - cb_remove = ( - [_raw_remove] - if isinstance(_raw_remove, str) and _raw_remove.strip() - else [str(v) for v in (_raw_remove or []) if str(v).strip()] - ) - cb_unset = ( - [_raw_unset] - if isinstance(_raw_unset, str) and _raw_unset.strip() - else [str(v) for v in (_raw_unset or []) if str(v).strip()] - ) - if cb_remove: - params["base_remove_args"] = cb_remove - if cb_unset: - params["base_unset_envs"] = cb_unset - if str(cb.get("args_mode") or "").strip().lower() == "replace": - params["base_args_mode"] = "replace" - base_tput = resolve_grading_anchor_tput(state) - if base_tput: - params["base_tput"] = base_tput + inject_stack_base_params(params, state, anchor=True) last_bl = state.last_baseline or {} if isinstance(last_bl, dict): bs = str(last_bl.get("benchmark_script") or "").strip() diff --git a/src/hyperloom/orchestrator/phases/prelude.py b/src/hyperloom/orchestrator/phases/prelude.py index bdd8cd0fe6..482f28cb1f 100644 --- a/src/hyperloom/orchestrator/phases/prelude.py +++ b/src/hyperloom/orchestrator/phases/prelude.py @@ -12,6 +12,7 @@ from ..state.optimization_journal import ( JournalEntry, ) +from ..state.shared_state import inject_stack_base_params from ..state.task_registry import Task from ..loop.coordinator import ( _DEFAULT_WARM_REPLAY_MIN_CONFIDENCE, @@ -668,22 +669,7 @@ async def _enqueue_internal_analysis_task(self, *, reason: str) -> Task: "reason": str(reason), } if reason != "prelude_initial": - cb = state.current_best or {} - if isinstance(cb, dict): - cb_args = str(cb.get("extra_server_args") or "") - if cb_args: - params["base_extra_args"] = cb_args - cb_envs = cb.get("extra_envs") - if isinstance(cb_envs, dict) and cb_envs: - params["base_extra_envs"] = dict(cb_envs) - for source, target in ( - ("remove_args", "base_remove_args"), - ("unset_envs", "base_unset_envs"), - ("args_mode", "base_args_mode"), - ): - value = cb.get(source) - if value not in (None, "", [], ()): - params[target] = value + inject_stack_base_params(params, state) else: # PRELUDE roofline profiles the baseline arm: inject baseline's own # server args (never current_best's) so a later warm-replay can't diff --git a/src/hyperloom/orchestrator/phases/sweep.py b/src/hyperloom/orchestrator/phases/sweep.py index 865671ffa3..a4872d2958 100644 --- a/src/hyperloom/orchestrator/phases/sweep.py +++ b/src/hyperloom/orchestrator/phases/sweep.py @@ -6,7 +6,7 @@ from __future__ import annotations import logging as _logging from typing import Any -from ..state.shared_state import SharedState +from ..state.shared_state import SharedState, inject_stack_base_params from ..state.task_registry import Task from .base import PhaseHandler @@ -215,11 +215,7 @@ async def _enqueue_internal_sweep_task( ps_result = getattr(state, "geak_result", None) or {} if isinstance(ps_result, dict) and ps_result.get("status") == "ok" and ps_result.get("bench_script"): params["geak_result"] = ps_result - cb = state.current_best or {} - if isinstance(cb, dict): - cb_args = str(cb.get("extra_server_args") or "") - if cb_args: - params["base_extra_args"] = cb_args + inject_stack_base_params(params, state) last_bl = state.last_baseline or {} if isinstance(last_bl, dict): # Mirror baseline's benchmark_script so re-launch uses the same wrapper. diff --git a/src/hyperloom/orchestrator/state/shared_state.py b/src/hyperloom/orchestrator/state/shared_state.py index b1b9907868..982fc1aebc 100644 --- a/src/hyperloom/orchestrator/state/shared_state.py +++ b/src/hyperloom/orchestrator/state/shared_state.py @@ -46,11 +46,13 @@ import os import shlex import time +from collections.abc import Callable from dataclasses import asdict, dataclass, field from datetime import datetime, timezone from pathlib import Path from typing import Any +from hyperloom.common.coerce import to_str_list from hyperloom.common.env_safety import redact_secret_values from hyperloom.common.io import atomic_write_json from hyperloom.common.profile_args import sanitize_profile_server_args @@ -71,7 +73,7 @@ resolve_kernel_opt_max_failures = _kernel_decision_settings.resolve_kernel_opt_max_failures -def _first_positive_tput(d: Any) -> float: +def first_positive_tput(d: Any) -> float: """Return the first positive ``tput``/``output_throughput`` from a dict. Args: @@ -109,13 +111,82 @@ def resolve_grading_anchor_tput(state: Any) -> float: """ if state is None: return 0.0 - best = _first_positive_tput(getattr(state, "current_best", None)) + best = first_positive_tput(getattr(state, "current_best", None)) if best > 0: return best baseline = getattr(state, "baseline_tput", 0.0) return float(baseline) if isinstance(baseline, (int, float)) and baseline > 0 else 0.0 +def _normalize_envs(value: Any) -> dict[str, str]: + """Coerce an env mapping to ``str -> str``; a resumed non-dict reads as empty.""" + return {str(k): str(v) for k, v in value.items()} if isinstance(value, dict) else {} + + +# ``base_*`` task param -> the ``current_best`` field it mirrors and its normalizer. +_STACK_BASE_FIELDS: tuple[tuple[str, str, Callable[[Any], Any]], ...] = ( + ("base_extra_args", "extra_server_args", lambda v: str(v or "").strip()), + ("base_extra_envs", "extra_envs", _normalize_envs), + ("base_remove_args", "remove_args", to_str_list), + ("base_unset_envs", "unset_envs", to_str_list), + ("base_args_mode", "args_mode", lambda v: "replace" if str(v or "").strip().lower() == "replace" else ""), +) + + +def stack_base_params(current_best: Any) -> dict[str, Any]: + """``base_*`` params projected from the fields ``current_best`` carries. + + Args: + current_best: A ``current_best`` snapshot (non-dicts read as empty). + + Returns: + The normalized ``base_*`` params; keys absent from ``current_best`` are + omitted rather than defaulted, so a caller can tell "no config" from + "empty config". + """ + cb = current_best if isinstance(current_best, dict) else {} + return {key: normalize(cb[source]) for key, source, normalize in _STACK_BASE_FIELDS if source in cb} + + +def inject_stack_base_params( + params: dict[str, Any], + state: Any, + *, + anchor: bool = False, + overwrite: bool = False, +) -> None: + """Seed a task's base config from ``current_best``, in place. + + A candidate is graded against an anchor while being launched on top of that + anchor's args/envs, so the two are one unit; ``anchor=True`` takes both from + the same snapshot rather than leaving a caller to seed one and omit the other. + + Args: + params: Task params, mutated in place. + state: Any object exposing ``current_best`` / ``baseline_tput`` + (``None`` and partial test doubles are tolerated). + anchor: Also seed ``base_tput``. + overwrite: Replace keys already present in ``params``, for an + execution-time rebind; dispatch-time seeding must not clobber an + operator- or LLM-supplied value. + """ + + def _put(key: str, value: Any) -> None: + if overwrite: + params[key] = value + else: + params.setdefault(key, value) + + if anchor: + anchor_tput = resolve_grading_anchor_tput(state) + if anchor_tput > 0: + _put("base_tput", anchor_tput) + for key, value in stack_base_params(getattr(state, "current_best", None)).items(): + # Empty means "no config"; on a rebind it is what clears a superseded layer. + if value or overwrite: + _put(key, value) + + # Ordered (key, label) projection for advisory ``model_arch``; empty/None keys dropped. _MODEL_ARCH_STRUCTURED_FIELDS: tuple[tuple[str, str], ...] = ( ("decoder_type", "decoder"), @@ -396,12 +467,6 @@ class SharedState(_RenderMixin, _ExploreStateMixin): # Internal-only baseline cold+hot double-run switch; default-on keeps EXPLORE # warm-decision apples-to-apples with the baseline measurement basis. baseline_double_run: bool = True - # Discarded first-round tput from the baseline cold-start double-run - # (audit/debugging only; gain math uses the hot ``baseline_tput``). - baseline_cold_tput: float = 0.0 - # Mirror of the hot measure-round tput; matches ``baseline_tput`` when the - # double-run path is eligible. - baseline_hot_tput: float = 0.0 baseline_accuracy: float = 0.0 # Standalone baseline-arm roofline ceiling computed right after baseline # lands; backs up snapshot ceiling so the frontend has data even when the @@ -2273,7 +2338,7 @@ def _resolve_baseline_achieved_tput(self) -> float: """ if isinstance(self.baseline_tput, (int, float)) and self.baseline_tput > 0: return float(self.baseline_tput) - return _first_positive_tput(self.last_baseline) + return first_positive_tput(self.last_baseline) def _resolve_current_best_achieved_tput(self) -> float: """Optimized-arm throughput for a current_best roofline snapshot. @@ -2286,7 +2351,7 @@ def _resolve_current_best_achieved_tput(self) -> float: float: The resolved current_best throughput, or ``0.0`` when none is available. """ - return _first_positive_tput(self.current_best) + return first_positive_tput(self.current_best) def _locate_diffusion_roofline_sidecar(self, kernel_roofline_path: Any) -> Path | None: """Locate the ``diffusion_roofline.json`` sidecar for the latest trace run.