diff --git a/src/hyperloom/inference_optimizer/tests/test_integrate_payload_defaults.py b/src/hyperloom/inference_optimizer/tests/test_integrate_payload_defaults.py index 00f1bf8a78..321b96b2c5 100644 --- a/src/hyperloom/inference_optimizer/tests/test_integrate_payload_defaults.py +++ b/src/hyperloom/inference_optimizer/tests/test_integrate_payload_defaults.py @@ -54,7 +54,9 @@ def test_all_three_defaults_fired(self, session_dir): session_dir=session_dir, ) - assert out["base_tput"] == 800.0 + # base_tput prefers current_best.tput (900) over the raw baseline (800): + # the candidate must beat the recipe it stacks onto, not just baseline. + assert out["base_tput"] == 900.0 assert out["config_path"] == "/tmp/base.yaml" assert out["extra_server_args"] == "--page-size 16" assert out["kernel_id"] == "k_abc" @@ -137,6 +139,40 @@ def test_zero_state_does_not_overwrite_explicit_payload(self, session_dir): assert out["base_tput"] == 750.0 + def test_base_tput_prefers_current_best_over_baseline(self, session_dir): + """A candidate must be judged against the CURRENT BEST recipe it stacks + onto (current_best.tput), not the raw baseline. + + Otherwise a kernel/fusion that beats baseline but regresses vs the + established best (e.g. a warm-replay recipe) is wrongly KEEP'd instead of + REVERT'd (observed: forge_fusion adopted at negative gain vs current_best + while still positive vs baseline, dragging the final recipe down). + """ + _seed_state( + session_dir, + baseline_tput=800.0, + current_best_args="--page-size 16", # _seed_state sets current_best.tput=900 + ) + + out = krh._fill_integrate_defaults_from_state( + {"kernel_id": "k_abc"}, + session_dir=session_dir, + ) + + assert out["base_tput"] == 900.0 # current_best, NOT baseline 800 + + def test_base_tput_falls_back_to_baseline_without_current_best(self, session_dir): + # No current_best recorded yet (early kernel phase) -> baseline is the + # only reference available. + _seed_state(session_dir, baseline_tput=800.0) + + out = krh._fill_integrate_defaults_from_state( + {"kernel_id": "k_abc"}, + session_dir=session_dir, + ) + + assert out["base_tput"] == 800.0 + class TestIntegrateHandlerHonoursStateDefault: @pytest.mark.asyncio diff --git a/src/hyperloom/orchestrator/kernel/request_handlers.py b/src/hyperloom/orchestrator/kernel/request_handlers.py index 05e20b629c..487b03e130 100644 --- a/src/hyperloom/orchestrator/kernel/request_handlers.py +++ b/src/hyperloom/orchestrator/kernel/request_handlers.py @@ -1264,8 +1264,19 @@ def _fill_integrate_defaults_from_state( str(pending_record.get("identity_route") or ""), ) + current_best = getattr(state, "current_best", None) or {} + if float(resolved.get("base_tput", 0.0) or 0.0) <= 0: - bt = float(getattr(state, "baseline_tput", 0.0) or 0.0) + # Judge the candidate against the CURRENT BEST recipe it stacks onto, + # not the raw baseline. ``extra_server_args`` below is filled from + # current_best, so the candidate is re-benched on top of that recipe; + # comparing the result to the raw baseline lets a kernel/fusion that + # beats baseline but REGRESSES vs the established best (e.g. a + # warm-replay recipe) get KEEP'd and drag the recipe down. Mirrors + # integrate_patch's rebind to current_best. Baseline is the fallback + # only before any current_best exists. + cb_tput = float(current_best.get("tput") or 0.0) if isinstance(current_best, dict) else 0.0 + bt = cb_tput if cb_tput > 0 else float(getattr(state, "baseline_tput", 0.0) or 0.0) if bt > 0: resolved["base_tput"] = bt @@ -1274,7 +1285,6 @@ def _fill_integrate_defaults_from_state( if cfg: resolved["config_path"] = cfg - current_best = getattr(state, "current_best", None) or {} if not resolved.get("extra_server_args") and isinstance(current_best, dict): cb_args = current_best.get("extra_server_args") or "" if cb_args: