From f52bd2e12194ebdba14023307e132a26da8f261d Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Mon, 3 Aug 2026 04:22:05 +0000 Subject: [PATCH 01/22] refactor(state): resolve the grading anchor in one place A candidate is launched with current_best's args/envs but several dispatch paths graded it against the bare baseline_tput, so anything that beat the baseline while regressing against the established recipe read as a win. The same fix had already been applied twice in isolation (integrate_patch, then kernel integrate in #1074) without the other seeding sites following. Add resolve_grading_anchor_tput() next to _first_positive_tput and route the explore/framework phase dispatch, the kernel integrate defaults, the approved proposal and delegate paths, and the kernel-stack keep decision through it. A free function rather than a SharedState method: half the callers hold a possibly-None state and the phase unit tests pass duck-typed doubles. The two resume/geak revalidation tasks keep baseline_tput on purpose - they reproduce the whole stack, so their gain is cumulative rather than a delta over current_best. Co-authored-by: Cursor --- .../tests/test_shared_state_units.py | 27 +++++++++++++++++++ .../orchestrator/kernel/request_handlers.py | 15 +++-------- .../orchestrator/loop/intent_router.py | 11 ++------ src/hyperloom/orchestrator/loop/proposals.py | 14 +++------- src/hyperloom/orchestrator/phases/explore.py | 3 ++- .../orchestrator/phases/framework.py | 5 ++-- .../orchestrator/phases/kernel_stack.py | 5 ++-- .../orchestrator/state/shared_state.py | 27 +++++++++++++++++++ 8 files changed, 71 insertions(+), 36 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_shared_state_units.py b/src/hyperloom/inference_optimizer/tests/test_shared_state_units.py index d308928f64..7ac267cb48 100644 --- a/src/hyperloom/inference_optimizer/tests/test_shared_state_units.py +++ b/src/hyperloom/inference_optimizer/tests/test_shared_state_units.py @@ -11,9 +11,36 @@ _DEFAULT_ATTEMPTS_HISTORY, _DEFAULT_LAST_FAILURES, SharedState, + resolve_grading_anchor_tput, ) +class TestResolveGradingAnchorTput: + def test_prefers_current_best_over_baseline(self): + s = SharedState() + s.baseline_tput = 2195.86 + s.current_best = {"action": "replay_warm_recipe", "tput": 2358.80} + assert resolve_grading_anchor_tput(s) == 2358.80 + + def test_falls_back_to_baseline_before_any_validated_layer(self): + s = SharedState() + s.baseline_tput = 2195.86 + assert resolve_grading_anchor_tput(s) == 2195.86 + + def test_reads_output_throughput_when_tput_absent(self): + s = SharedState() + s.baseline_tput = 800.0 + s.current_best = {"action": "explore", "output_throughput": 900.0} + assert resolve_grading_anchor_tput(s) == 900.0 + + @pytest.mark.parametrize("state", [None, object()]) + def test_tolerates_missing_state(self, state): + assert resolve_grading_anchor_tput(state) == 0.0 + + def test_zero_when_nothing_established(self): + assert resolve_grading_anchor_tput(SharedState()) == 0.0 + + class TestGridSessionDeadline: def test_returns_none_when_budget_unbounded(self): s = SharedState() diff --git a/src/hyperloom/orchestrator/kernel/request_handlers.py b/src/hyperloom/orchestrator/kernel/request_handlers.py index 487b03e130..2fd135f97a 100644 --- a/src/hyperloom/orchestrator/kernel/request_handlers.py +++ b/src/hyperloom/orchestrator/kernel/request_handlers.py @@ -1215,7 +1215,7 @@ def _fill_integrate_defaults_from_state( Returns: A shallow copy of ``payload`` with defaults filled from state. """ - from ..state.shared_state import SharedState + from ..state.shared_state import SharedState, resolve_grading_anchor_tput resolved = dict(payload) state = SharedState.load_or_init(session_dir) @@ -1267,16 +1267,9 @@ def _fill_integrate_defaults_from_state( current_best = getattr(state, "current_best", None) or {} if float(resolved.get("base_tput", 0.0) or 0.0) <= 0: - # Judge the candidate against the CURRENT BEST recipe it stacks onto, - # not the raw baseline. ``extra_server_args`` below is filled from - # current_best, so the candidate is re-benched on top of that recipe; - # comparing the result to the raw baseline lets a kernel/fusion that - # beats baseline but REGRESSES vs the established best (e.g. a - # warm-replay recipe) get KEEP'd and drag the recipe down. Mirrors - # integrate_patch's rebind to current_best. Baseline is the fallback - # only before any current_best exists. - cb_tput = float(current_best.get("tput") or 0.0) if isinstance(current_best, dict) else 0.0 - bt = cb_tput if cb_tput > 0 else float(getattr(state, "baseline_tput", 0.0) or 0.0) + # ``extra_server_args`` below is filled from current_best, so the + # candidate must be graded against that recipe too. + bt = resolve_grading_anchor_tput(state) if bt > 0: resolved["base_tput"] = bt diff --git a/src/hyperloom/orchestrator/loop/intent_router.py b/src/hyperloom/orchestrator/loop/intent_router.py index 0eab0e0b41..15db15dbf5 100644 --- a/src/hyperloom/orchestrator/loop/intent_router.py +++ b/src/hyperloom/orchestrator/loop/intent_router.py @@ -33,6 +33,7 @@ from hyperloom.inference_optimizer.session.session_paths import runs_dir from ..bus.message_bus import Message from ..policy.gate import PolicyDenied, SPECIALIST_FROM_AGENT_PREFIX +from ..state.shared_state import resolve_grading_anchor_tput from ..state.task_registry import IllegalTransition, TaskNotFound from ..kernel.request_handlers import get_handler @@ -394,15 +395,7 @@ async def _handle_delegate(self, source: str, intent: Intent) -> None: # Parity with _materialize_approved_proposal: direct delegates need the same knobs. if action_name == "explore": self._inject_explore_runtime_params(params) - # Inject base_tput tied to current_best (or baseline_tput). - cb = getattr(self.shared_state, "current_best", None) or {} - cb_tput = cb.get("tput") if isinstance(cb, dict) else None - base = ( - cb_tput - if isinstance(cb_tput, (int, float)) and cb_tput > 0 - else getattr(self.shared_state, "baseline_tput", 0.0) - ) - params.setdefault("base_tput", float(base or 0.0)) + params.setdefault("base_tput", resolve_grading_anchor_tput(self.shared_state)) # Wave sugar: a specialist delegate carrying params.tasks=[...] fans out # into N standard freeform specialist tasks, each dispatched through the # normal SpecialistRunner + TaskRegistry + lease + reap path. diff --git a/src/hyperloom/orchestrator/loop/proposals.py b/src/hyperloom/orchestrator/loop/proposals.py index 7dd261d944..23214ef74d 100644 --- a/src/hyperloom/orchestrator/loop/proposals.py +++ b/src/hyperloom/orchestrator/loop/proposals.py @@ -11,6 +11,7 @@ from ..phases import machine_state as _phase_state from ..bus.message_bus import Message from .coordinator_helpers import approved_proposal_idempotency_key +from ..state.shared_state import resolve_grading_anchor_tput if TYPE_CHECKING: from ..state.task_registry import Task @@ -460,9 +461,7 @@ def _list_control(value: Any) -> list[str]: if cb_args_mode == "replace": params.setdefault("base_args_mode", "replace") if pending.action_name == "sweep": - cb_tput = cb.get("tput") if isinstance(cb, dict) else None - base = cb_tput if isinstance(cb_tput, (int, float)) and cb_tput > 0 else self.shared_state.baseline_tput - params.setdefault("base_tput", float(base or 0.0)) + params.setdefault("base_tput", resolve_grading_anchor_tput(self.shared_state)) params.setdefault("base_extra_args", cb_args) if cb_remove_args: params.setdefault("base_remove_args", cb_remove_args) @@ -474,10 +473,7 @@ def _list_control(value: Any) -> list[str]: params.setdefault("config_path", self.shared_state.baseline_config_path) if pending.action_name == "explore": self._inject_explore_runtime_params(params) - # Inject base_tput/base_extra_args tied to current_best (or baseline_tput) so _gain_pct resolves. - cb_tput = cb.get("tput") if isinstance(cb, dict) else None - base = cb_tput if isinstance(cb_tput, (int, float)) and cb_tput > 0 else self.shared_state.baseline_tput - params.setdefault("base_tput", float(base or 0.0)) + params.setdefault("base_tput", resolve_grading_anchor_tput(self.shared_state)) params.setdefault("base_extra_args", cb_args) if cb_envs: params.setdefault("base_extra_envs", dict(cb_envs)) @@ -494,9 +490,7 @@ def _list_control(value: Any) -> list[str]: # Seed the patched-eval server with the same base args/config every # other eval server uses, else it launches on bare framework defaults # and crashes at startup regardless of the patch. - cb_tput = cb.get("tput") if isinstance(cb, dict) else None - base = cb_tput if isinstance(cb_tput, (int, float)) and cb_tput > 0 else self.shared_state.baseline_tput - params.setdefault("base_tput", float(base or 0.0)) + params.setdefault("base_tput", resolve_grading_anchor_tput(self.shared_state)) params.setdefault("base_extra_args", cb_args) if cb_remove_args: params.setdefault("base_remove_args", cb_remove_args) diff --git a/src/hyperloom/orchestrator/phases/explore.py b/src/hyperloom/orchestrator/phases/explore.py index 1b52f5c6bf..e7d16f32e9 100644 --- a/src/hyperloom/orchestrator/phases/explore.py +++ b/src/hyperloom/orchestrator/phases/explore.py @@ -21,6 +21,7 @@ ) from ..loop.sub_agent_runner import SubAgentResult from ..specialists.runner import SpecialistFailureType +from ..state.shared_state import resolve_grading_anchor_tput from ..state.task_registry import Task from ..loop.coordinator import ( FORCE_STALLED_KEEP_ROUNDS, @@ -1389,7 +1390,7 @@ async def _maybe_materialize_mn_explore( params["base_unset_envs"] = cb_unset if str(cb.get("args_mode") or "").strip().lower() == "replace": params["base_args_mode"] = "replace" - base_tput = float(getattr(state, "baseline_tput", 0.0) or 0.0) + base_tput = resolve_grading_anchor_tput(state) if base_tput: params["base_tput"] = base_tput last_bl = state.last_baseline or {} diff --git a/src/hyperloom/orchestrator/phases/framework.py b/src/hyperloom/orchestrator/phases/framework.py index 3c591bb456..d7e8ed3167 100644 --- a/src/hyperloom/orchestrator/phases/framework.py +++ b/src/hyperloom/orchestrator/phases/framework.py @@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any from . import machine_state as _phase_state from ..bus.message_bus import Message +from ..state.shared_state import resolve_grading_anchor_tput if TYPE_CHECKING: from ..state.task_registry import Task @@ -3544,7 +3545,7 @@ async def _enqueue_framework_agent_task(self, candidate: dict[str, Any]) -> None params = { "candidate": candidate, "batch_id": candidate.get("batch_id") or "", - "base_tput": float(getattr(state, "baseline_tput", 0.0) or 0.0), + "base_tput": resolve_grading_anchor_tput(state), "framework": str(candidate.get("framework") or getattr(state, "framework", "") or "").strip().lower(), # Source patches require the accuracy gate for KEEP. "require_accuracy_for_keep": True, @@ -5118,7 +5119,7 @@ def _framework_config_explore_params( params["base_unset_envs"] = cb_unset if str(cb.get("args_mode") or "").strip().lower() == "replace": params["base_args_mode"] = "replace" - base_tput = float(getattr(state, "baseline_tput", 0.0) or 0.0) + base_tput = resolve_grading_anchor_tput(state) if base_tput: params["base_tput"] = base_tput last_bl = state.last_baseline or {} diff --git a/src/hyperloom/orchestrator/phases/kernel_stack.py b/src/hyperloom/orchestrator/phases/kernel_stack.py index ae6729d9cc..01ce7af9da 100644 --- a/src/hyperloom/orchestrator/phases/kernel_stack.py +++ b/src/hyperloom/orchestrator/phases/kernel_stack.py @@ -9,6 +9,7 @@ from datetime import datetime, timezone from typing import Any from ..bus.message_bus import Message +from ..state.shared_state import resolve_grading_anchor_tput from ..state.task_registry import Task from .base import PhaseHandler @@ -499,9 +500,7 @@ async def _run_kernel_stack_validation_e2e( # The stack is applied on top of current_best, so the KEEP # decision uses the incremental gain over current_best, not the # total gain over the original baseline. - current_best = self.shared_state.current_best or {} - current_best_tput = float(current_best.get("tput") or 0.0) - decision_base = current_best_tput if current_best_tput > 0 else base_tput + decision_base = resolve_grading_anchor_tput(self.shared_state) new_tput = float(bench_result.get("output_throughput") or 0.0) gain_pct = (new_tput - base_tput) / base_tput * 100.0 if base_tput > 0 else 0.0 incremental_gain_pct = (new_tput - decision_base) / decision_base * 100.0 if decision_base > 0 else 0.0 diff --git a/src/hyperloom/orchestrator/state/shared_state.py b/src/hyperloom/orchestrator/state/shared_state.py index ed5870eea6..3ad37f31dc 100644 --- a/src/hyperloom/orchestrator/state/shared_state.py +++ b/src/hyperloom/orchestrator/state/shared_state.py @@ -86,6 +86,33 @@ def _first_positive_tput(d: Any) -> float: return 0.0 +def resolve_grading_anchor_tput(state: Any) -> float: + """Throughput a new candidate must beat: the recipe it is composed on top of. + + Candidates are launched with ``current_best``'s args/envs, so grading them + against ``baseline_tput`` compares a measurement to a configuration it was + never taken on: anything that beats the bare baseline but regresses against + the established recipe (e.g. a warm-replay bundle) reads as a win and drags + ``current_best`` down. ``baseline_tput`` is the fallback only before any + validated layer exists. + + Args: + state: Any object exposing ``current_best`` / ``baseline_tput`` + (``None`` and partial test doubles are tolerated). + + Returns: + ``current_best``'s throughput when positive, else ``baseline_tput``; + ``0.0`` when neither is established. + """ + if state is None: + return 0.0 + best = _first_positive_tput(getattr(state, "current_best", None)) + if best > 0: + return best + baseline = getattr(state, "baseline_tput", 0.0) + return float(baseline) if isinstance(baseline, (int, float)) and baseline > 0 else 0.0 + + # Ordered (key, label) projection for advisory ``model_arch``; empty/None keys dropped. _MODEL_ARCH_STRUCTURED_FIELDS: tuple[tuple[str, str], ...] = ( ("decoder_type", "decoder"), From b8e6308b5c0616d2fd710f7062e2349e30cbce11 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Mon, 3 Aug 2026 04:22:19 +0000 Subject: [PATCH 02/22] fix(writeback): never lower current_best when lifting a winner _lift_to_current_best overwrote current_best unconditionally, so a winner whose executor called it a KEEP against a stale task-level base_tput could regress the recipe. Session MiniMax-M3-MXFP8 lost 0.14% that way: a warm replay had reached 2358.8 and an explore variant measuring 2355.5 was still lifted, leaving the run reporting less gain than it had already achieved. Refuse the lift when the winner does not beat the anchor it was composed on, and return whether it landed so the callers stop stamping a promotion, advancing cumulative_gain_validated or firing a watermark roofline for work that was refused. The audit row records no_promote rather than discarded, matching _promote_baseline, so orchestration reads it as measured-but-flat instead of retrying it as a failure. No allow_regression escape: all four call sites were audited and none needs one. Revalidation is skipped earlier, geak 2b has its own no_promote branch, and the framework/integrate keep flags already come from an executor that compared against current_best. Co-authored-by: Cursor --- .../tests/test_promote_shared_state_lock.py | 66 +++++++++++++++++++ src/hyperloom/orchestrator/loop/writeback.py | 64 ++++++++++++++---- 2 files changed, 118 insertions(+), 12 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_promote_shared_state_lock.py b/src/hyperloom/inference_optimizer/tests/test_promote_shared_state_lock.py index d263fb5f93..3f2221d8c4 100644 --- a/src/hyperloom/inference_optimizer/tests/test_promote_shared_state_lock.py +++ b/src/hyperloom/inference_optimizer/tests/test_promote_shared_state_lock.py @@ -720,3 +720,69 @@ async def test_lift_copies_source_snapshot_into_stack_entry(session_dir): assert top.get("source_snapshot") == "/session/optimization_stack/src/abc123" assert top.get("framework_root") == "/opt/vllm" assert top.get("base_sha") == "deadbeef" + + +def test_lift_refuses_winner_that_does_not_beat_current_best(session_dir): + """current_best never moves down, even for a winner its executor called a KEEP.""" + coord = _coord(session_dir) + s = coord.shared_state + s.baseline_tput = 2195.86 + s.current_best = { + "action": "replay_warm_recipe", + "tput": 2358.80, + "extra_server_args": "--enable-aiter-allreduce-fusion", + "extra_envs": {"SGLANG_USE_AITER": "1"}, + } + s.optimization_stack = [{"action": "replay_warm_recipe", "variant_name": "warm_replay"}] + s.gain_per_stack_entry = [7.908] + + lifted = coord._lift_to_current_best( + "explore", + 2355.46, + { + "name": "minimax-fused-swiglu+moe-combine", + "candidate_extra_server_args": "--trust-remote-code", + "extra_envs": {"SGLANG_MINIMAX_M3_FUSED_MOE_COMBINE": "1"}, + "tput": 2355.46, + }, + ) + + assert lifted is False + assert s.current_best["tput"] == 2358.80 + assert len(s.optimization_stack) == 1 + assert s.gain_per_stack_entry == [7.908] + + +def test_lift_refuses_winner_below_baseline_when_stack_is_empty(session_dir): + """Before any validated layer the baseline is the anchor, and it holds too.""" + coord = _coord(session_dir) + s = coord.shared_state + s.baseline_tput = 1000.0 + + lifted = coord._lift_to_current_best( + "explore", + 900.0, + {"name": "regression", "candidate_extra_server_args": "--slow", "extra_envs": {}}, + ) + + assert lifted is False + assert not s.current_best + assert s.optimization_stack == [] + + +def test_lift_accepts_winner_that_beats_current_best(session_dir): + """The guard only blocks regressions; a genuine win still lifts.""" + coord = _coord(session_dir) + s = coord.shared_state + s.baseline_tput = 1000.0 + s.current_best = {"action": "baseline", "tput": 1000.0, "extra_server_args": "", "extra_envs": {}} + + lifted = coord._lift_to_current_best( + "explore", + 1100.0, + {"name": "real-win", "candidate_extra_server_args": "--fast", "extra_envs": {}}, + ) + + assert lifted is True + assert s.current_best["tput"] == 1100.0 + assert s.optimization_stack[-1]["variant_name"] == "real-win" diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index 64146dc1e2..d88225299f 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -22,7 +22,7 @@ operation_kind_for, summarize_change, ) -from ..state.shared_state import SharedState +from ..state.shared_state import SharedState, resolve_grading_anchor_tput from hyperloom.inference_optimizer.protocol.intent import Intent from ..bus.message_bus import Message from .coordinator_helpers import ( @@ -2028,9 +2028,15 @@ def _lift_to_current_best( bv: dict[str, Any], *, gap_canonical_id: str = "", - ) -> None: + ) -> bool: """Update SharedState.current_best + recompute cumulative_gain; gap_canonical_id (when known) is stamped onto the stack entry so provenance resolves by gap id not name. + ``current_best`` never moves down: a winner that does not beat the + anchor it was composed on top of is refused here even if its own + executor called it a KEEP, so a stale task-level ``base_tput`` cannot + regress the recipe. Revalidation flows confirm the existing stack and + never reach this method. + Args: task_kind: The action kind that produced the winner (stamped on the stack entry / current_best). @@ -2038,7 +2044,20 @@ def _lift_to_current_best( bv: The winning variant dict (args, envs, metrics, provenance). gap_canonical_id: When known, stamped onto the stack entry so provenance resolves by gap id rather than name. + + Returns: + ``True`` when the winner was lifted, ``False`` when it was refused + for not beating the current anchor. """ + anchor = resolve_grading_anchor_tput(self.shared_state) + if anchor > 0 and float(best_tput) <= anchor: + log.info( + "current_best held at %.1f: %s winner measured %.1f (no lift)", + anchor, + task_kind, + float(best_tput), + ) + return False previous = self.shared_state.current_best or {} base_args = "" if isinstance(previous, dict): @@ -2187,6 +2206,7 @@ def _lift_to_current_best( self.shared_state.cumulative_gain = ( (float(best_tput) - self.shared_state.baseline_tput) / self.shared_state.baseline_tput * 100.0 ) + return True def _should_run_prelude_bootstrap(self, tput: Any) -> bool: """Whether to enqueue the post-baseline PRELUDE bootstrap chain. @@ -2970,13 +2990,12 @@ async def _promote_explore( explore_gap_cid = ( str((task.params or {}).get("gap_canonical_id") or "").strip() if task is not None else "" ) - self._lift_to_current_best( + promoted = self._lift_to_current_best( "explore", float(best_tput), best_winner, gap_canonical_id=explore_gap_cid, ) - promoted = True changed = True try: self.shared_state.note_explore_outcome(promoted=promoted) @@ -2993,7 +3012,12 @@ async def _promote_explore( ) else: changed = True - audit_decision = "promoted" if promoted else "discarded" + if promoted: + audit_decision = "promoted" + elif winners and not is_revalidation_task: + audit_decision = "no_promote" + else: + audit_decision = "discarded" audit_extras = { "round_id": round_id, "winners_count": (len(winners) if isinstance(winners, list) else 0), @@ -3022,6 +3046,7 @@ async def _promote_integrate_patch( status = str(result.get("status") or "") new_tput = result.get("output_throughput") kept_flag = status == "kept" and isinstance(new_tput, (int, float)) and float(new_tput) > 0 + lifted = False if kept_flag: specialist_task_id = str(result.get("specialist_task_id") or "") lift = { @@ -3043,8 +3068,8 @@ async def _promote_integrate_patch( "framework_root": result.get("framework_root") or "", "base_sha": result.get("base_sha") or "", } - self._lift_to_current_best("integrate_patch", float(new_tput), lift) - if self.shared_state.baseline_tput > 0: + lifted = self._lift_to_current_best("integrate_patch", float(new_tput), lift) + if lifted and self.shared_state.baseline_tput > 0: self._update_cumulative_gain_validated(new_tput) self.shared_state.resume_pending_revalidation = False await self._maybe_enqueue_watermark_roofline( @@ -3060,7 +3085,12 @@ async def _promote_integrate_patch( }: self.shared_state.pending_integrate = {} changed = True - audit_decision = "promoted" if kept_flag else "discarded" + if lifted: + audit_decision = "promoted" + elif kept_flag: + audit_decision = "no_promote" + else: + audit_decision = "discarded" audit_extras = { "status": status, "specialist_task_id": result.get("specialist_task_id"), @@ -3154,6 +3184,7 @@ async def _promote_framework_agent( entry["max_gain_pct_observed_in_batch"] = gain break changed = True + lifted = False if kept_flag and isinstance(new_tput, (int, float)) and new_tput > 0: lift = { "name": f"framework:{cand_id}", @@ -3163,13 +3194,18 @@ async def _promote_framework_agent( "extra_envs": {}, "workspace": result.get("workspace"), } - self._lift_to_current_best("framework", float(new_tput), lift) - if self.shared_state.baseline_tput > 0: + lifted = self._lift_to_current_best("framework", float(new_tput), lift) + if lifted and self.shared_state.baseline_tput > 0: self._update_cumulative_gain_validated(new_tput) await self._maybe_enqueue_watermark_roofline( reason="framework_keep_watermark", ) - audit_decision = "promoted" if kept_flag else "discarded" + if lifted: + audit_decision = "promoted" + elif kept_flag: + audit_decision = "no_promote" + else: + audit_decision = "discarded" audit_extras = { "candidate_id": cand_id, "batch_id": batch_id, @@ -3611,7 +3647,8 @@ def _replay_keep_from_result(self, kind: str, result: dict[str, Any]) -> bool: else: return False before = len(self.shared_state.optimization_stack or []) - self._lift_to_current_best(kind, float(tput), bv) + if not self._lift_to_current_best(kind, float(tput), bv): + return False return len(self.shared_state.optimization_stack or []) > before def _resume_rollback_pending_integrate(self, pending: dict[str, Any]) -> dict[str, Any]: @@ -3985,6 +4022,8 @@ async def _enqueue_internal_stack_rebench(self, *, reason: str) -> dict[str, Any "note": "same-harness config-identity revalidation of the geak e2e win", } ], + # Revalidation reproduces the whole stack, so its gain is + # cumulative-vs-baseline, not a delta over current_best. "base_tput": float(getattr(self.shared_state, "baseline_tput", 0.0) or 0.0), "enable_stack_rebench": False, } @@ -4047,6 +4086,7 @@ async def _enqueue_internal_stack_rebench(self, *, reason: str) -> dict[str, Any "note": "post-resume full-stack end-to-end revalidation", } ], + # Cumulative-vs-baseline, same as the geak revalidation above. "base_tput": float(getattr(self.shared_state, "baseline_tput", 0.0) or 0.0), "enable_stack_rebench": False, } From 959c4200ed153882d28e60bf5d156fc1764fb404 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Mon, 3 Aug 2026 04:22:32 +0000 Subject: [PATCH 03/22] fix(explore,integrate): re-resolve the grading anchor at execution time base_tput is snapshotted into task params when the task is queued, and the backstop only consulted live state when that value was missing. A task that waits behind others - PRELUDE ran six hours in the session that surfaced this - therefore grades against an anchor that current_best has since moved past, and the regression reads as a win. Take the live anchor whenever it exceeds the snapshot, warning on the drift so the next occurrence is one grep away rather than an investigation. This also feeds the stack rebench a correct stability floor, which on its own would have rejected the variant that started this. Co-authored-by: Cursor --- .../tests/test_explore_executor.py | 69 +++++++++++++++++++ .../orchestrator/actions/executors/explore.py | 27 ++++---- .../actions/executors/integrate_patch.py | 21 +++--- 3 files changed, 96 insertions(+), 21 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_explore_executor.py b/src/hyperloom/inference_optimizer/tests/test_explore_executor.py index 297ab6d61e..b594fdd58b 100644 --- a/src/hyperloom/inference_optimizer/tests/test_explore_executor.py +++ b/src/hyperloom/inference_optimizer/tests/test_explore_executor.py @@ -739,6 +739,75 @@ def _fake_run(cmd, *args, **kwargs): assert tested["outcome"] == "REVERT" +@pytest.mark.asyncio +async def test_explore_executor_supersedes_stale_params_base_tput( + sub_agent_runner, + tmp_path, +): + """A queued task's stale ``base_tput`` is superseded by the live anchor. + + Reproduces MiniMax-M3-MXFP8 session 96879: the task was queued against the + bare baseline (2192.5) while a warm replay had already lifted current_best + to 2358.8, so a 2355.5 variant read as +7.4% and was KEEP'd even though it + regressed the recipe by 0.14%. + """ + sub, tr, _ = sub_agent_runner + state = SharedState() + state.baseline_tput = 2195.86 + state.current_best = {"action": "replay_warm_recipe", "tput": 2358.80} + sub.shared_state = state + + base = tmp_path / "base.yaml" + _write_baseline_yaml(base) + output_dir = tmp_path / "explore-stale-anchor" + + def _fake_run(cmd, *args, **kwargs): + out_idx = cmd.index("--output-dir") + slot = Path(cmd[out_idx + 1]) + _fake_workspace(slot, tput=2355.46) + return subprocess.CompletedProcess( + args=cmd, + returncode=0, + stdout="ok", + stderr="", + ) + + grid = [ + { + "name": "minimax-fused-swiglu+moe-combine", + "extra_args": "--fused-flag", + "extra_envs": {}, + "provenance": "specialist:research_scout", + } + ] + task = await tr.create( + kind="explore", + params={ + "config_path": str(base), + "output_dir": str(output_dir), + # Snapshotted when the task was queued, before the warm replay landed. + "base_tput": 2192.52, + "grid": grid, + "variant_timeout_sec": 10, + }, + idempotency_key="ex-stale-anchor", + ) + sub.register_executor("explore", ExploreExecutor(session_dir=tmp_path)) + with patch( + "hyperloom.orchestrator.actions.executors._grid_runner.run_with_session_kill", + side_effect=_fake_run, + ): + res = await sub.run_task(task) + + out = res.result + assert out["status"] == "succeeded" + assert out["winners"] == [] + fp = canonical_fingerprint("--fused-flag", {}) + tested = out["explore_search_update"]["tested"][fp] + assert tested["base_tput"] == 2358.80 + assert tested["outcome"] == "REVERT" + + @pytest.mark.asyncio async def test_explore_executor_dedups_against_ledger(sub_agent_runner, tmp_path, monkeypatch): """A variant whose fingerprint already lives in explore_search.tested lands in ``skipped_dup``, not re-benched.""" diff --git a/src/hyperloom/orchestrator/actions/executors/explore.py b/src/hyperloom/orchestrator/actions/executors/explore.py index 28e21488a3..73a9da9873 100644 --- a/src/hyperloom/orchestrator/actions/executors/explore.py +++ b/src/hyperloom/orchestrator/actions/executors/explore.py @@ -40,6 +40,7 @@ from hyperloom.common.gain_math import gain_pct from hyperloom.common.timeutil import now_iso from hyperloom.inference_optimizer.session.session_paths import runs_dir +from ...state.shared_state import resolve_grading_anchor_tput from ._accuracy_gate import ( accuracy_passed, is_high_accuracy_risk, @@ -598,18 +599,20 @@ async def __call__(self, ctx) -> dict[str, Any]: base_unset_envs = to_str_list(params.get("base_unset_envs")) base_args_mode = str(params.get("base_args_mode") or "append").strip().lower() base_tput = float(params.get("base_tput") or 0.0) - # Backstop: recover the comparison anchor from live SharedState when - # params carries no positive ``base_tput``. Prefer running best, fall - # back to the original baseline. - if base_tput <= 0: - ss = extra.get("shared_state") or extra.get("state") - if ss is not None: - cb = getattr(ss, "current_best", None) or {} - cb_tput = cb.get("tput") if isinstance(cb, dict) else None - if isinstance(cb_tput, (int, float)) and cb_tput > 0: - base_tput = float(cb_tput) - else: - base_tput = float(getattr(ss, "baseline_tput", 0.0) or 0.0) + # ``base_tput`` is snapshotted when the task is queued, so it goes stale + # whenever current_best advances before the task runs. Grading against + # the stale value lets a regression read as a win, so always take the + # live anchor when it is higher (this also covers an absent param). + ss = extra.get("shared_state") or extra.get("state") + live_anchor = resolve_grading_anchor_tput(ss) + if live_anchor > base_tput: + if base_tput > 0: + log.warning( + "explore: anchor drift, params base_tput=%.1f but live anchor is %.1f; grading against live", + base_tput, + live_anchor, + ) + base_tput = live_anchor baseline_accuracy = float(params.get("accuracy_baseline") or 0.0) or float( params.get("baseline_accuracy") or 0.0 ) diff --git a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py index 9f871fdec8..c4079eef3f 100644 --- a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py @@ -24,6 +24,7 @@ from hyperloom.inference_optimizer.session.session_paths import runs_dir from ...framework.paths import resolve_source_file_allowlist from ...specialists.patch_safety import patch_file_targets, patch_targets_missing +from ...state.shared_state import resolve_grading_anchor_tput from ._accuracy_gate import ( DEFAULT_ENABLEMENT_ACCURACY_FLOOR, accuracy_keep_block, @@ -2656,15 +2657,17 @@ async def _gate_perf( ) -> dict[str, Any]: """Throughput KEEP / REVERT decision with optional stack rebench.""" base_tput = float(params.get("base_tput") or 0.0) - if base_tput <= 0 and shared_state is not None: - cb = getattr(shared_state, "current_best", None) - cb_tput = cb.get("tput") if isinstance(cb, dict) else None - if isinstance(cb_tput, (int, float)) and cb_tput > 0: - base_tput = float(cb_tput) - else: - ss_base = getattr(shared_state, "baseline_tput", 0.0) - if isinstance(ss_base, (int, float)) and ss_base > 0: - base_tput = float(ss_base) + # Same stale-snapshot hazard as explore: prefer the live anchor whenever + # it is higher than whatever the task was queued with. + live_anchor = resolve_grading_anchor_tput(shared_state) + if live_anchor > base_tput: + if base_tput > 0: + log.warning( + "integrate_patch: anchor drift, params base_tput=%.1f but live anchor is %.1f; using live", + base_tput, + live_anchor, + ) + base_tput = live_anchor keep_threshold_pct = float(params.get("keep_threshold_pct", self.keep_threshold_pct)) new_tput = bench_result.get("output_throughput") From 05dd00dc14e40efb7755c63e954635c95c85fff3 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Mon, 3 Aug 2026 05:30:10 +0000 Subject: [PATCH 04/22] fix(baseline): drain the queued backlog once the anchor is established baseline is LLM-proposable and nothing retired it after it succeeded, so a run could keep queueing reference measurements while the first was still running. Session MiniMax-M3-MXFP8 executed ten of them, eight after the anchor already existed, at roughly nineteen GPU-minutes each. Cancel the queued baselines when baseline_tput turns positive and report the cancellation as a baseline_drain observation. The enablement revalidation baseline is spared: it re-anchors a stack the specialist changed rather than re-measuring the one the session has, identified the same way _promote_baseline identifies it. cancel_family grows a reason and an exclusion set - it previously hardcoded prune_branch into every cancellation's history evidence, which would have misattributed these. The revalidation params reason becomes a shared constant instead of a literal repeated across four call sites. Co-authored-by: Cursor --- .../tests/test_promote_shared_state_lock.py | 55 ++++++++++++++ .../actions/executors/_accuracy_gate.py | 5 ++ .../orchestrator/loop/coordinator.py | 1 + src/hyperloom/orchestrator/loop/writeback.py | 74 ++++++++++++++++++- .../orchestrator/phases/framework.py | 3 +- .../orchestrator/state/task_registry.py | 18 ++++- 6 files changed, 149 insertions(+), 7 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_promote_shared_state_lock.py b/src/hyperloom/inference_optimizer/tests/test_promote_shared_state_lock.py index 3f2221d8c4..e9f5abe164 100644 --- a/src/hyperloom/inference_optimizer/tests/test_promote_shared_state_lock.py +++ b/src/hyperloom/inference_optimizer/tests/test_promote_shared_state_lock.py @@ -722,6 +722,61 @@ async def test_lift_copies_source_snapshot_into_stack_entry(session_dir): assert top.get("base_sha") == "deadbeef" +@pytest.mark.asyncio +async def test_drain_cancels_queued_baselines_but_spares_revalidation(session_dir): + """A succeeded baseline drains its backlog; the enablement revalidation survives.""" + from hyperloom.orchestrator.actions.executors._accuracy_gate import ( + ENABLEMENT_REVALIDATION_REASON, + ) + + coord = _coord(session_dir) + coord.shared_state.baseline_tput = 2195.86 + + stale_a = await coord.tasks.create(kind="baseline", params={}, idempotency_key="bl-a") + stale_b = await coord.tasks.create(kind="baseline", params={"tag": "x"}, idempotency_key="bl-b") + reval = await coord.tasks.create( + kind="baseline", + params={"reason": ENABLEMENT_REVALIDATION_REASON}, + idempotency_key="bl-reval", + ) + other = await coord.tasks.create(kind="explore", params={}, idempotency_key="ex-a") + + cancelled = await coord._drain_queued_baselines(reason="baseline_established") + + assert set(cancelled) == {stale_a.task_id, stale_b.task_id} + assert (await coord.tasks.get(reval.task_id)).state == "queued" + assert (await coord.tasks.get(other.task_id)).state == "queued" + assert (await coord.tasks.get(stale_a.task_id)).state == "cancelled" + + +@pytest.mark.asyncio +async def test_drain_spares_the_tracked_revalidation_task_id(session_dir): + """The tracked id is honoured even when params carry no revalidation reason.""" + coord = _coord(session_dir) + coord.shared_state.baseline_tput = 1000.0 + reval = await coord.tasks.create(kind="baseline", params={}, idempotency_key="bl-tracked") + coord.shared_state.enablement_revalidation_task_id = reval.task_id + + assert await coord._drain_queued_baselines(reason="baseline_established") == [] + assert (await coord.tasks.get(reval.task_id)).state == "queued" + + +@pytest.mark.asyncio +async def test_promote_baseline_drains_the_backlog(session_dir): + """The drain is wired into promotion, not just available as a helper.""" + coord = _coord(session_dir) + stale = await coord.tasks.create(kind="baseline", params={}, idempotency_key="bl-stale") + + await coord._promote_to_shared_state( + "baseline", + {"status": "succeeded", "output_throughput": 2185.95}, + task=_task("baseline", task_id="t-first"), + ) + + assert coord.shared_state.baseline_tput == 2185.95 + assert (await coord.tasks.get(stale.task_id)).state == "cancelled" + + def test_lift_refuses_winner_that_does_not_beat_current_best(session_dir): """current_best never moves down, even for a winner its executor called a KEEP.""" coord = _coord(session_dir) diff --git a/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py b/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py index ddba6a0649..ecebc4ee09 100644 --- a/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py +++ b/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py @@ -51,6 +51,10 @@ ENABLEMENT_MODE_ALL, ) +# params.reason marking a baseline that re-anchors a stack the enablement +# specialist changed, rather than re-measuring the established one. +ENABLEMENT_REVALIDATION_REASON = "enablement_eval_revalidation" + # Result-dict keys stamped by the baseline executor on an eval-rooted failure and # read by writeback promotion/persistence. BASELINE_EVAL_FAILED_KEY = "baseline_eval_failed" @@ -588,6 +592,7 @@ def accuracy_passed( "DEFAULT_ENABLEMENT_ACCURACY_FLOOR", "ENABLEMENT_MODES", "ENABLEMENT_MODE_ALL", + "ENABLEMENT_REVALIDATION_REASON", "ENABLEMENT_MODE_EVAL", "ENABLEMENT_MODE_LAUNCH", "ENABLEMENT_MODE_OFF", diff --git a/src/hyperloom/orchestrator/loop/coordinator.py b/src/hyperloom/orchestrator/loop/coordinator.py index 4c27d56b56..fa376dfd55 100644 --- a/src/hyperloom/orchestrator/loop/coordinator.py +++ b/src/hyperloom/orchestrator/loop/coordinator.py @@ -868,6 +868,7 @@ def router(self) -> IntentRouter: "_aggregate_research_evidence": "writeback", "_harvest_research_scout": "writeback", "_record_specialist_result": "writeback", + "_drain_queued_baselines": "writeback", # Phase handlers, grouped in the same call-chain order as # _COLLAB_MODULES/the @property block above: # machine -> prelude -> sweep -> close -> internal -> kernel_stack -> diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index d88225299f..b7f4796778 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -22,6 +22,7 @@ operation_kind_for, summarize_change, ) +from ..actions.executors._accuracy_gate import ENABLEMENT_REVALIDATION_REASON from ..state.shared_state import SharedState, resolve_grading_anchor_tput from hyperloom.inference_optimizer.protocol.intent import Intent from ..bus.message_bus import Message @@ -646,7 +647,7 @@ async def _handle_unpromotable_result( # pending state, preserve the frozen trigger identity, increment stall. reval_tid = str(getattr(self.shared_state, "enablement_revalidation_task_id", "") or "").strip() is_revalidation = bool( - (task.params or {}).get("reason") == "enablement_eval_revalidation" + (task.params or {}).get("reason") == ENABLEMENT_REVALIDATION_REASON or (reval_tid and reval_tid == str(task.task_id or "")) ) if is_revalidation and bool(getattr(self.shared_state, "enablement_validation_pending", False)): @@ -2337,7 +2338,7 @@ async def _promote_baseline( promoting_tid = str(getattr(task, "task_id", "") or "").strip() if task is not None else "" task_params = (getattr(task, "params", None) or {}) if task is not None else {} is_revalidation = bool( - task_params.get("reason") == "enablement_eval_revalidation" + task_params.get("reason") == ENABLEMENT_REVALIDATION_REASON or (tracked_tid and tracked_tid == promoting_tid) ) # The anchor is the best measurement of the unmodified stack. A later, @@ -2501,7 +2502,7 @@ async def _promote_baseline( # Stamp canonical params fingerprint for the self-loop denial helper. "fingerprint": _baseline_params_fingerprint(task_params), # Record revalidation context for history. - "is_revalidation": bool(task_params.get("reason") == "enablement_eval_revalidation"), + "is_revalidation": bool(task_params.get("reason") == ENABLEMENT_REVALIDATION_REASON), "enablement_succeeded": bool(getattr(self.shared_state, "enablement_succeeded", False)), "enablement_accuracy_floor": float(getattr(self.shared_state, "enablement_accuracy_floor", 0.0) or 0.0), } @@ -2509,6 +2510,8 @@ async def _promote_baseline( audit_extras["anchor_kept_tput"] = prior_anchor # seed the gaps[] ledger from baseline (best-effort). await self._refresh_gaps(reason="baseline_done") + if self.shared_state.baseline_tput > 0: + await self._drain_queued_baselines(reason="baseline_established") # Standalone baseline-arm roofline ceiling (pure CPU): backs up the # snapshot ceiling in case the later roofline step fails. if isinstance(tput, (int, float)) and tput > 0: @@ -2554,6 +2557,71 @@ async def _promote_baseline( outcome.audit_decision = audit_decision outcome.audit_extras = audit_extras + async def _drain_queued_baselines(self, *, reason: str) -> list[str]: + """Cancel queued baselines that the established anchor has made redundant. + + Baseline is LLM-proposable and nothing stopped a run from queueing more + of them while the first was still measuring, so a succeeded baseline can + leave a backlog that re-measures a number the session already has. The + enablement revalidation baseline is spared: it re-anchors a stack the + specialist changed, so it is not redundant work. + + Args: + reason: Stamped onto the cancellation history and the observation. + + Returns: + The cancelled task ids (empty when the queue held none). + """ + spared = await self._enablement_revalidation_task_ids() + try: + cancelled = await self.tasks.cancel_family( + ["baseline"], + reason=reason, + exclude_task_ids=spared, + ) + except Exception: # noqa: BLE001 — draining is best-effort + log.exception("baseline drain: cancel_family failed") + return [] + if not cancelled: + return [] + log.info( + "baseline drain: cancelled %d queued baseline task(s) (reason=%s, spared=%d)", + len(cancelled), + reason, + len(spared), + ) + await self._record_observation( + "coordinator", + "observation", + { + "kind": "baseline_drain", + "reason": reason, + "cancelled_task_ids": cancelled, + "baseline_tput": float(self.shared_state.baseline_tput or 0.0), + }, + ) + return cancelled + + async def _enablement_revalidation_task_ids(self) -> set[str]: + """Queued baseline task ids that re-anchor an enablement-changed stack. + + Matches the identity ``_promote_baseline`` uses: the tracked task id, or + a params reason recorded before the id was persisted. + """ + spared: set[str] = set() + tracked = str(getattr(self.shared_state, "enablement_revalidation_task_id", "") or "").strip() + if tracked: + spared.add(tracked) + try: + for task in await self.tasks.queued(): + if str(getattr(task, "kind", "") or "") != "baseline": + continue + if (getattr(task, "params", None) or {}).get("reason") == ENABLEMENT_REVALIDATION_REASON: + spared.add(str(getattr(task, "task_id", "") or "")) + except Exception: # noqa: BLE001 — fall back to the tracked id alone + log.exception("baseline drain: queued-task scan failed") + return {t for t in spared if t} + async def _promote_replay_warm_recipe( self, result: dict, diff --git a/src/hyperloom/orchestrator/phases/framework.py b/src/hyperloom/orchestrator/phases/framework.py index d7e8ed3167..dd2c1f2925 100644 --- a/src/hyperloom/orchestrator/phases/framework.py +++ b/src/hyperloom/orchestrator/phases/framework.py @@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any from . import machine_state as _phase_state from ..bus.message_bus import Message +from ..actions.executors._accuracy_gate import ENABLEMENT_REVALIDATION_REASON from ..state.shared_state import resolve_grading_anchor_tput if TYPE_CHECKING: @@ -4596,7 +4597,7 @@ async def _maybe_enqueue_enablement_baseline_revalidation(self) -> str: pass params: dict[str, Any] = { "source": "coordinator_internal", - "reason": "enablement_eval_revalidation", + "reason": ENABLEMENT_REVALIDATION_REASON, "disable_run_eval": False, **_enablement_carrier_params(state), } diff --git a/src/hyperloom/orchestrator/state/task_registry.py b/src/hyperloom/orchestrator/state/task_registry.py index 44e6407484..253b08f8ac 100644 --- a/src/hyperloom/orchestrator/state/task_registry.py +++ b/src/hyperloom/orchestrator/state/task_registry.py @@ -19,6 +19,7 @@ import json import uuid +from collections.abc import Iterable from dataclasses import dataclass, field from datetime import datetime, timezone from typing import Any @@ -520,17 +521,26 @@ def _alive(pid: int) -> bool: reclaimed.append(task_id) return reclaimed - async def cancel_family(self, family_kinds: list[str]) -> list[str]: - """Bulk-cancel queued tasks of the given kinds (Robustness prune_branch); returns cancelled task_ids. + async def cancel_family( + self, + family_kinds: list[str], + *, + reason: str = "prune_branch", + exclude_task_ids: Iterable[str] = (), + ) -> list[str]: + """Bulk-cancel queued tasks of the given kinds; returns cancelled task_ids. Args: family_kinds: Task kinds whose queued tasks should be cancelled. + reason: Stamped onto each cancellation's history evidence. + exclude_task_ids: Task ids to leave queued. Returns: The task ids that were cancelled (empty when none matched). """ if not family_kinds: return [] + spared = {str(t or "").strip() for t in exclude_task_ids if str(t or "").strip()} cancelled: list[str] = [] async with self.db.transaction() as cur: placeholders = ",".join("?" * len(family_kinds)) @@ -541,13 +551,15 @@ async def cancel_family(self, family_kinds: list[str]) -> list[str]: rows = [(r["task_id"], r["history"]) for r in cur.fetchall()] now = _now_iso() for task_id, history_json in rows: + if str(task_id or "").strip() in spared: + continue history = json.loads(history_json) history.append( { "from": "queued", "to": "cancelled", "ts": now, - "evidence": {"reason": "prune_branch"}, + "evidence": {"reason": reason}, } ) cur.execute( From 254fd8bf89bd568f3ef5eb1e44aae82bcc5a3efc Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Mon, 3 Aug 2026 05:30:42 +0000 Subject: [PATCH 05/22] feat(policy): retire baseline once the session has an anchor PRELUDE allows baseline so a run can reach baseline_tput > 0, and nothing retired it afterwards. _sequence_denial_for_action exempts baseline by design - it exists to hold everything else back until the anchor lands - and PolicyGate carried a singleton rule only for sweep, so a repeat baseline was denied by nothing and reviewed against an empty evidence checklist. Add baseline_phase_singleton on the same shape as sweep_phase_singleton: denied on both the propose_action and delegate channels once the anchor is positive, with params.bypass_baseline_singleton as the recorded escape for an operator who genuinely wants a fresh reference. Co-authored-by: Cursor --- .../tests/test_sweep_phase_auto.py | 52 +++++++++++++++ src/hyperloom/orchestrator/policy/gate.py | 63 +++++++++++++++++++ 2 files changed, 115 insertions(+) diff --git a/src/hyperloom/inference_optimizer/tests/test_sweep_phase_auto.py b/src/hyperloom/inference_optimizer/tests/test_sweep_phase_auto.py index 4849682a7f..e513edb43c 100644 --- a/src/hyperloom/inference_optimizer/tests/test_sweep_phase_auto.py +++ b/src/hyperloom/inference_optimizer/tests/test_sweep_phase_auto.py @@ -1219,6 +1219,58 @@ def test_sweep_singleton_bypass_flag_lets_operator_force_second_sweep(): ) +# 6a. PolicyGate baseline_phase_singleton rule +class _BaselineSingletonState: + """SharedState stand-in carrying just the field the ``baseline_phase_singleton`` rule reads.""" + + def __init__(self, baseline_tput: float = 0.0): + self.baseline_tput = baseline_tput + + +@pytest.mark.parametrize("intent_kind", ["delegate", "propose_action"]) +def test_baseline_singleton_denies_once_anchor_is_established(intent_kind): + """Both channels refuse a repeat baseline after baseline_tput turns positive.""" + from hyperloom.orchestrator.policy.gate import PolicyDenied + + gate = _make_policy_gate(shared_state=_BaselineSingletonState(2195.86)) + + with pytest.raises(PolicyDenied) as excinfo: + gate._validate_baseline_singleton( + payload={"action_name": "baseline", "params": {}}, + intent_kind=intent_kind, + ) + assert excinfo.value.rule == "baseline_phase_singleton" + assert "bypass_baseline_singleton" in (excinfo.value.hint or "") + + +def test_baseline_singleton_inert_before_the_anchor_exists(): + """PRELUDE must still be able to reach baseline_tput > 0.""" + gate = _make_policy_gate(shared_state=_BaselineSingletonState(0.0)) + gate._validate_baseline_singleton( + payload={"action_name": "baseline", "params": {}}, + intent_kind="propose_action", + ) + + +def test_baseline_singleton_inert_when_shared_state_is_none(): + gate = _make_policy_gate(shared_state=None) + gate._validate_baseline_singleton( + payload={"action_name": "baseline"}, + intent_kind="delegate", + ) + + +def test_baseline_singleton_bypass_flag_lets_operator_force_a_fresh_anchor(): + gate = _make_policy_gate(shared_state=_BaselineSingletonState(2195.86)) + gate._validate_baseline_singleton( + payload={ + "action_name": "baseline", + "params": {"bypass_baseline_singleton": True}, + }, + intent_kind="delegate", + ) + + # 6b. End-to-end through full validate_intent (delegate / propose_action) def test_validate_intent_denies_llm_sweep_delegate_in_active_sweep_phase(): """Through full ``validate_intent``: a sweep delegate in active SWEEP fires the singleton rule before ``_validate_phase_action``.""" diff --git a/src/hyperloom/orchestrator/policy/gate.py b/src/hyperloom/orchestrator/policy/gate.py index 371ee478a4..c6236ca4e7 100644 --- a/src/hyperloom/orchestrator/policy/gate.py +++ b/src/hyperloom/orchestrator/policy/gate.py @@ -147,6 +147,9 @@ def __init__(self, reason: str, *, rule: str | None = None, hint: str | None = N # action gated via COORDINATOR_INTERNAL_ACTIONS, not a singleton rule here.) SWEEP_ACTION_NAME: str = "sweep" +# Reference measurement action; named constant for the ``baseline_phase_singleton`` rule. +BASELINE_ACTION_NAME: str = "baseline" + # Specialist / Explore parallelism caps — single source of truth across layers. # Research-lane ceiling fallback used when the GPU count cannot be probed. RESEARCH_LANE_CEILING_FALLBACK: int = 2 @@ -849,6 +852,8 @@ def _validate_delegate_body( # conc_sweep as Coordinator-managed (phase_incompatible) below. if action_name == SWEEP_ACTION_NAME: self._validate_sweep_singleton(payload, intent_kind="delegate") + if action_name == BASELINE_ACTION_NAME: + self._validate_baseline_singleton(payload, intent_kind="delegate") self._validate_gemm_tuning_action(action_name, intent_kind="delegate") # Refuse delegate for unknown action names when an ActionRegistry is wired (no registry → fall through). if self.action_registry is not None and self.action_registry.get(action_name) is None: @@ -945,6 +950,11 @@ def _validate_propose_action(self, role: "AgentRole", payload: dict[str, Any]) - payload, intent_kind="propose_action", ) + if action_name == BASELINE_ACTION_NAME: + self._validate_baseline_singleton( + payload, + intent_kind="propose_action", + ) # Per-action source allowlist (e.g. ``recover`` is robustness-only); mirrors the delegate-path guard. allowed_sources = DELEGATE_ACTION_SOURCE_ALLOWLIST.get(action_name) if allowed_sources is not None and role.name not in allowed_sources: @@ -1403,6 +1413,58 @@ def _validate_sweep_singleton( ), ) + # ``baseline_phase_singleton`` + def _validate_baseline_singleton( + self, + payload: dict[str, Any], + *, + intent_kind: str, + ) -> None: + """Deny an LLM ``baseline`` once the session has an anchor. + + PRELUDE allows ``baseline`` so the run can reach ``baseline_tput > 0``, + and nothing retired it afterwards: a run could keep re-measuring a + reference it already had, at roughly twenty GPU-minutes a turn, while + the newest measurement silently redefined every later gain. Escape: + ``params.bypass_baseline_singleton=True``. + + Args: + payload (dict[str, Any]): the intent payload; + ``params.bypass_baseline_singleton`` opts out of the guard. + intent_kind (str): the channel the action arrived on, used in the + error hint. + + Raises: + PolicyDenied: when ``baseline_tput`` is already positive and no + bypass flag is set. + """ + params = payload.get("params") or {} + if isinstance(params, dict) and params.get("bypass_baseline_singleton"): + return + ss = getattr(self, "shared_state", None) + if ss is None: + return + anchor = getattr(ss, "baseline_tput", 0.0) + if not isinstance(anchor, (int, float)) or anchor <= 0: + return + raise PolicyDenied( + ( + f"baseline: the session anchor is already established " + f"(baseline_tput={float(anchor):.1f}); a repeat baseline " + f"re-measures a reference the run already has." + ), + rule="baseline_phase_singleton", + hint=( + "PRELUDE is done with baseline; let the phase advance. " + "A measurement you distrust is a reason to re-measure the " + "candidate, not the reference. " + f"If you genuinely need a fresh anchor, set " + f"params.bypass_baseline_singleton=True on the " + f"{intent_kind} payload (the override is recorded " + f"on the audit trail)." + ), + ) + def _validate_integrate_patch_critic_gate( self, payload: dict[str, Any], @@ -2184,6 +2246,7 @@ def to_policy_denial_summary(state, *, top_k: int = 6) -> str: "DELEGATE_ACTION_REQUIRED_PAYLOAD", "DELEGATE_ACTION_SOURCE_ALLOWLIST", "EXTEND_LEASE_MAX_SEC", + "BASELINE_ACTION_NAME", "INTERNAL_ONLY_ACTION_NAMES", "KERNEL_AGENT_OWNED_ACTIONS", "KILL_TASK_ALLOWED_SCOPES", From 08e8136704a7ae873d4e83c6ab29df198c9905bf Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Mon, 3 Aug 2026 05:30:57 +0000 Subject: [PATCH 06/22] feat(prune): let orchestration drain a queue without retiring the family Orchestration could already emit prune_branch, but the handler always added the family to the persistent pruned set. That is the right move for an action that has to stop and the wrong one for a backlog that merely outlived its purpose - draining stale baselines should not cost the run the ability to re-baseline later. Add a scope field, following the convention kill_task already established. The default "family" keeps today's behaviour; "queued" cancels the queued tasks and leaves the pruned set alone, routing a baseline drain through the same helper the Coordinator uses so the enablement revalidation is spared there too. Documented for the model alongside kill_task / send_message / extend_lease, which are the moves it already reaches for when a queue needs attention. Co-authored-by: Cursor --- .../tests/test_coordinator_runtime.py | 31 +++++++++++++++++++ ...t_orchestration_prune_branch_permission.py | 28 +++++++++++++++++ .../orchestrator/loop/intent_router.py | 26 ++++++++++++---- src/hyperloom/orchestrator/policy/gate.py | 26 ++++++++++++++++ .../orchestrator/prompts/orchestration.md | 12 +++++++ .../orchestrator/roles/mcp_emit_intent.py | 3 +- 6 files changed, 119 insertions(+), 7 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py index 86fe3b9081..195b7d2e3f 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_runtime.py @@ -670,6 +670,37 @@ async def test_coordinator_prune_branch_cancels_family_and_records_advisory(sess await c.stop() +@pytest.mark.asyncio +async def test_coordinator_prune_branch_queued_scope_drains_without_retiring(session_dir): + """Orchestration can drain a backlog and still propose the family afterwards.""" + c = Coordinator(session_dir, backends=_build_backends({})) + try: + a = await c.tasks.create(kind="baseline", params={}, idempotency_key="qa") + b = await c.tasks.create(kind="baseline", params={"tag": "x"}, idempotency_key="qb") + + await c._handle_intent( + "orchestration", + Intent( + type=IntentType.PRUNE_BRANCH, + payload={ + "family": "baseline", + "reason": "anchor already established", + "scope": "queued", + }, + ), + ) + + assert (await c.tasks.get(a.task_id)).state == "cancelled" + assert (await c.tasks.get(b.task_id)).state == "cancelled" + assert "baseline" not in c.shared_state.pruned_families + events = await c.bus.tail(topic="event") + assert any( + m.payload.get("kind") == "prune_branch" and m.payload.get("scope") == "queued" for m in events + ) + finally: + await c.stop() + + @pytest.mark.asyncio async def test_coordinator_policy_denied_surfaces_as_observation(session_dir): bad = Intent(type=IntentType.DELEGATE, payload={"action_name": "baseline"}) diff --git a/src/hyperloom/inference_optimizer/tests/test_orchestration_prune_branch_permission.py b/src/hyperloom/inference_optimizer/tests/test_orchestration_prune_branch_permission.py index db4a933732..0692518ecc 100644 --- a/src/hyperloom/inference_optimizer/tests/test_orchestration_prune_branch_permission.py +++ b/src/hyperloom/inference_optimizer/tests/test_orchestration_prune_branch_permission.py @@ -59,6 +59,34 @@ def test_orchestration_prune_branch_missing_family_key_rejected(gate): assert exc.value.rule == "payload" +def test_orchestration_prune_branch_accepts_queued_scope(gate): + """scope='queued' drains a backlog without retiring the family.""" + gate.validate_intent( + "orchestration", + Intent( + type=IntentType.PRUNE_BRANCH, + payload={ + "family": "baseline", + "reason": "anchor already established", + "scope": "queued", + }, + ), + ) + + +def test_orchestration_prune_branch_unknown_scope_rejected(gate): + """An unrecognised scope is denied rather than silently treated as a full prune.""" + with pytest.raises(PolicyDenied) as exc: + gate.validate_intent( + "orchestration", + Intent( + type=IntentType.PRUNE_BRANCH, + payload={"family": "baseline", "reason": "x", "scope": "running"}, + ), + ) + assert exc.value.rule == "prune_scope" + + # Robustness (pre-existing path) — still works def test_robustness_can_still_emit_prune_branch(gate): """Pre-existing path unchanged — both sources are in the PRUNE_BRANCH allowlist.""" diff --git a/src/hyperloom/orchestrator/loop/intent_router.py b/src/hyperloom/orchestrator/loop/intent_router.py index 15db15dbf5..029393de42 100644 --- a/src/hyperloom/orchestrator/loop/intent_router.py +++ b/src/hyperloom/orchestrator/loop/intent_router.py @@ -32,7 +32,12 @@ from hyperloom.common.timeutil import now_iso from hyperloom.inference_optimizer.session.session_paths import runs_dir from ..bus.message_bus import Message -from ..policy.gate import PolicyDenied, SPECIALIST_FROM_AGENT_PREFIX +from ..policy.gate import ( + PolicyDenied, + PRUNE_BRANCH_SCOPE_FAMILY, + PRUNE_BRANCH_SCOPE_QUEUED, + SPECIALIST_FROM_AGENT_PREFIX, +) from ..state.shared_state import resolve_grading_anchor_tput from ..state.task_registry import IllegalTransition, TaskNotFound from ..kernel.request_handlers import get_handler @@ -889,18 +894,26 @@ async def _handle_extend_lease(self, source: str, intent: Intent) -> None: async def _handle_prune_branch(self, source: str, intent: Intent) -> None: """Prune an action family and cancel its in-flight tasks. - Adds the family to the persistent pruned set, cancels any tasks in that - family, and broadcasts a ``prune_branch`` event. + ``scope="family"`` (the default) adds the family to the persistent + pruned set so it stays retired. ``scope="queued"`` only drains the + backlog, leaving the family available — the move for an action whose + queue outlived its usefulness rather than one that has to stop. Args: source (str): The agent issuing the prune. intent (Intent): The PRUNE_BRANCH intent; ``payload`` carries - ``family`` and optional ``reason``. + ``family``, optional ``reason`` and optional ``scope``. """ family = intent.payload["family"] - if self.shared_state.add_pruned_family(family): + reason = str(intent.payload.get("reason") or "prune_branch") + scope = str(intent.payload.get("scope") or PRUNE_BRANCH_SCOPE_FAMILY).strip() + drain_only = scope == PRUNE_BRANCH_SCOPE_QUEUED + if not drain_only and self.shared_state.add_pruned_family(family): self.shared_state.save(self.session_dir) - cancelled = await self.tasks.cancel_family([family]) + if drain_only and family == "baseline": + cancelled = await self._drain_queued_baselines(reason=reason) + else: + cancelled = await self.tasks.cancel_family([family], reason=reason) await self.bus.append_and_seq( Message.new( source, @@ -909,6 +922,7 @@ async def _handle_prune_branch(self, source: str, intent: Intent) -> None: { "kind": "prune_branch", "family": family, + "scope": scope, "cancelled_task_ids": cancelled, "reason": intent.payload.get("reason"), }, diff --git a/src/hyperloom/orchestrator/policy/gate.py b/src/hyperloom/orchestrator/policy/gate.py index c6236ca4e7..6e3d2c247b 100644 --- a/src/hyperloom/orchestrator/policy/gate.py +++ b/src/hyperloom/orchestrator/policy/gate.py @@ -366,6 +366,17 @@ def _whole_machine_pool_size() -> int: KILL_TASK_SOURCE_ALLOWLIST: frozenset[str] = frozenset({"robustness", "orchestration"}) KILL_TASK_ALLOWED_SCOPES: frozenset[str] = frozenset({"task"}) +# prune_branch scopes. ``family`` retires the action for the rest of the run; +# ``queued`` only drains the backlog and leaves the family usable. +PRUNE_BRANCH_SCOPE_FAMILY: str = "family" +PRUNE_BRANCH_SCOPE_QUEUED: str = "queued" +PRUNE_BRANCH_ALLOWED_SCOPES: frozenset[str] = frozenset( + { + PRUNE_BRANCH_SCOPE_FAMILY, + PRUNE_BRANCH_SCOPE_QUEUED, + } +) + # Ceiling on a single extend_lease step; repeated extensions are allowed. EXTEND_LEASE_MAX_SEC: int = 3600 @@ -2139,6 +2150,18 @@ def _validate_robustness_only(self, role: "AgentRole", intent_type: IntentType, family = str(payload.get("family", "")).strip() if not family: raise PolicyDenied("prune_branch missing family", rule="payload") + scope = str(payload.get("scope") or PRUNE_BRANCH_SCOPE_FAMILY).strip() + if scope not in PRUNE_BRANCH_ALLOWED_SCOPES: + raise PolicyDenied( + f"prune_branch scope={scope!r} not allowed " + f"(allowed: {sorted(PRUNE_BRANCH_ALLOWED_SCOPES)!r})", + rule="prune_scope", + hint=( + f"{PRUNE_BRANCH_SCOPE_FAMILY!r} retires the action for " + f"the rest of the run; {PRUNE_BRANCH_SCOPE_QUEUED!r} " + f"only cancels the queued backlog." + ), + ) # --------------------------------------------------------------------------- @@ -2252,6 +2275,9 @@ def to_policy_denial_summary(state, *, top_k: int = 6) -> str: "KILL_TASK_ALLOWED_SCOPES", "KILL_TASK_SOURCE_ALLOWLIST", "PATH_LIKE_FIELDS", + "PRUNE_BRANCH_ALLOWED_SCOPES", + "PRUNE_BRANCH_SCOPE_FAMILY", + "PRUNE_BRANCH_SCOPE_QUEUED", "PolicyDenied", "PolicyGate", "REQUEST_ROUTING", diff --git a/src/hyperloom/orchestrator/prompts/orchestration.md b/src/hyperloom/orchestrator/prompts/orchestration.md index cfca280abe..7977e71440 100644 --- a/src/hyperloom/orchestrator/prompts/orchestration.md +++ b/src/hyperloom/orchestrator/prompts/orchestration.md @@ -105,6 +105,18 @@ queued behind the lane or GPUs it holds. Three moves: lane rows, in bounded steps. For live work near expiry that the TTL watchdog would otherwise fail out. +A fourth move covers the queue rather than a single task: + +- `prune_branch{family, reason, scope='queued'}` — cancels every *queued* + task of one family and leaves the family usable. Use it when a backlog + outlived its purpose: several turns queued the same measurement before the + first one returned, and the answer is now in hand. The default + `scope='family'` instead retires the action for the rest of the run, so + reach for it only when the family itself is a dead end. Queued baselines + are drained automatically once `baseline_tput > 0` (a `baseline_drain` + observation reports what was cancelled); this is the manual equivalent for + any family. + Doing nothing is a legitimate choice; doing nothing because nothing prompted you is not. diff --git a/src/hyperloom/orchestrator/roles/mcp_emit_intent.py b/src/hyperloom/orchestrator/roles/mcp_emit_intent.py index 71f81ee893..f5c14ed852 100644 --- a/src/hyperloom/orchestrator/roles/mcp_emit_intent.py +++ b/src/hyperloom/orchestrator/roles/mcp_emit_intent.py @@ -49,7 +49,8 @@ "approve|reject|redirect|advise|needs_review}, " "kill_task:{task_id,reason}, " "extend_lease:{task_id,extra_sec,reason}, " - "prune_branch:{family,reason}, escalate_strategy_change:" + "prune_branch:{family,reason,scope ∈ family|queued}, " + "escalate_strategy_change:" "{reason,next_action_hint}, update_state:{changes}, " "alert:{severity,summary}." ), From 7dae704f045191558fd95ac694e05c0a29906689 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Mon, 3 Aug 2026 06:18:18 +0000 Subject: [PATCH 07/22] fix(baseline): report a measured zero accuracy as below-floor, not unavailable The sibling-accuracy salvage filtered on `accuracy > 0.0`, so a baseline that genuinely scored gsm8k=0.0 was indistinguishable from one whose eval never ran. The cold-start guard evaluates only in warmup_round, so the deciding measure_round always salvages: a real zero was dropped, `acc` stayed None, and `classify_accuracy_failure` stamped `accuracy_unavailable` with evidence `accuracy=None ... source=None`. The enablement specialist was handed "the eval produced no number" while a full gsm8k run of degenerate output sat unreferenced, and the "salvaged but below the floor" branch was unreachable for exactly zero. Salvage now returns any finite score and the callers decide usability through the existing `accuracy_meets_floor`, which already means "finite, strictly positive and >= floor". `_request_eval_rooted_baseline_stop` still stops the run on a zero, now with the score and its source on record. `_finite_score` replaces a hand-rolled float() guard that would have admitted NaN/inf, and the write-back block duplicated across both call sites moves into `_apply_salvaged_accuracy`. Co-authored-by: Cursor --- .../actions/executors/baseline.py | 110 +++++++++++------- 1 file changed, 70 insertions(+), 40 deletions(-) diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index 93aa3a05c4..ecaade8cb3 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -1554,7 +1554,7 @@ def _request_eval_rooted_baseline_stop( result (dict): The failed baseline result dict, mutated in place when a sibling accuracy is salvaged. """ - from ._accuracy_gate import request_baseline_accuracy_stop + from ._accuracy_gate import accuracy_meets_floor, request_baseline_accuracy_stop params = ctx.task.params or {} framework = str(params.get("framework") or "").strip() or os.environ.get("FRAMEWORK", "").strip() or None @@ -1562,26 +1562,26 @@ def _request_eval_rooted_baseline_stop( shared_state = extra.get("shared_state") or self.shared_state salvaged = self._salvage_sibling_baseline_accuracy(result, framework) if salvaged is not None: - acc_val = float(salvaged["accuracy"]) - result["accuracy"] = acc_val - result["accuracy_task"] = salvaged.get("task", "gsm8k") - result["accuracy_metric"] = salvaged.get("metric", "") - result["accuracy_source"] = salvaged.get("source_file", "") - result.setdefault("nonfatal_warnings", []) - result["nonfatal_warnings"].append("baseline_accuracy_salvaged_from_sibling_attempt") - if shared_state is not None: - try: - shared_state.baseline_accuracy = acc_val - except Exception: # noqa: BLE001 — salvage must never break baseline - log.debug("baseline_executor: salvage could not set shared_state", exc_info=True) + acc_val = self._apply_salvaged_accuracy(result, salvaged, shared_state) + # ``accuracy_meets_floor`` already means "finite, strictly positive + # and >= floor", so floor 0.0 is the legacy "any usable accuracy". + if accuracy_meets_floor(acc_val, 0.0): + log.warning( + "baseline_executor: eval-rooted baseline failure, but salvaged " + "a valid baseline accuracy=%.4f from a sibling attempt (%s); " + "not stopping the run", + acc_val, + salvaged.get("source_file", ""), + ) + return + # A measured zero is a broken baseline, not a usable reference: it + # must still reach the stop below, now with the score on record. log.warning( - "baseline_executor: eval-rooted baseline failure, but salvaged " - "a valid baseline accuracy=%.4f from a sibling attempt (%s); " - "not stopping the run", + "baseline_executor: eval-rooted baseline failure and the sibling " + "attempt measured accuracy=%.4f (%s); stopping the run", acc_val, salvaged.get("source_file", ""), ) - return request_baseline_accuracy_stop( shared_state, context=f"baseline:{framework or 'unknown'}:eval_aborted", @@ -1677,29 +1677,21 @@ def _maybe_stop_on_missing_baseline_accuracy( # should reach enablement. salvaged = self._salvage_sibling_baseline_accuracy(result, framework) if salvaged is not None: - acc_val = float(salvaged["accuracy"]) - result["accuracy"] = acc_val - result["accuracy_task"] = salvaged.get("task", "gsm8k") - result["accuracy_metric"] = salvaged.get("metric", "") - result["accuracy_source"] = salvaged.get("source_file", "") - result.setdefault("nonfatal_warnings", []) - result["nonfatal_warnings"].append("baseline_accuracy_salvaged_from_sibling_attempt") - if shared_state is not None: - try: - shared_state.baseline_accuracy = acc_val - except Exception: # noqa: BLE001 — salvage must never break baseline - log.debug("baseline_executor: salvage could not set shared_state", exc_info=True) + acc_val = self._apply_salvaged_accuracy(result, salvaged, shared_state) log.warning( "baseline_executor: this attempt's RESULT_DIR had no accuracy, " - "but salvaged a valid baseline accuracy=%.4f from a sibling " - "attempt (%s); not stopping the run", + "but salvaged a measured baseline accuracy=%.4f from a sibling " + "attempt (%s)", acc_val, salvaged.get("source_file", ""), ) - if not eval_enablement or accuracy_meets_floor(acc_val, floor): + # Floor 0.0 reproduces the non-enablement "any positive accuracy is + # usable" rule; ``accuracy_meets_floor`` rejects zero either way. + if accuracy_meets_floor(acc_val, floor if eval_enablement else 0.0): return - # Salvaged, but still under the floor: that is a real quality - # signal, so fall through to enablement with the observed value. + # Salvaged, but unusable (zero or still under the floor): that is a + # real quality signal, so fall through with the observed value + # rather than reporting it as a missing measurement. acc = acc_val # No opt-out: a genuine baseline exists to establish the accuracy @@ -1732,12 +1724,51 @@ def _maybe_stop_on_missing_baseline_accuracy( context=f"baseline:{framework or 'unknown'}", ) + def _apply_salvaged_accuracy( + self, + result: dict[str, Any], + salvaged: dict[str, Any], + shared_state: Any, + ) -> float: + """Record a salvaged sibling accuracy on the result and SharedState. + + Records the score whatever its value; deciding whether it is *usable* + belongs to the caller, which knows the applicable floor. + + Args: + result: The baseline result dict, mutated in place. + salvaged: The parsed eval dict from + :meth:`_salvage_sibling_baseline_accuracy`. + shared_state: The live SharedState, or ``None``. + + Returns: + float: The salvaged accuracy. + """ + acc_val = float(salvaged["accuracy"]) + result["accuracy"] = acc_val + result["accuracy_task"] = salvaged.get("task", "gsm8k") + result["accuracy_metric"] = salvaged.get("metric", "") + result["accuracy_source"] = salvaged.get("source_file", "") + result.setdefault("nonfatal_warnings", []) + result["nonfatal_warnings"].append("baseline_accuracy_salvaged_from_sibling_attempt") + if shared_state is not None: + try: + shared_state.baseline_accuracy = acc_val + except Exception: # noqa: BLE001 — salvage must never break baseline + log.debug("baseline_executor: salvage could not set shared_state", exc_info=True) + return acc_val + def _salvage_sibling_baseline_accuracy( self, result: dict[str, Any], framework: str | None, ) -> dict[str, Any] | None: - """Return a positive accuracy from a sibling baseline attempt, if any. + """Return a measured accuracy from a sibling baseline attempt, if any. + + A measured ``0.0`` is returned like any other score: it is evidence, + not an absent measurement. The caller decides whether the value is + usable; filtering zeros here would make a real quality failure + indistinguishable from "the eval never ran". Scans the shared ``runs/baseline`` root (the parent of this attempt's ``output_dir``) so eval output written by any sibling attempt is seen. @@ -1759,16 +1790,15 @@ def _salvage_sibling_baseline_accuracy( if not runs_root.exists(): return None try: - from ._accuracy_gate import parse_eval_results + from ._accuracy_gate import _finite_score, parse_eval_results eval_data = parse_eval_results(runs_root, framework=framework) except Exception: # noqa: BLE001 — salvage must never break the stop path log.debug("baseline_executor: sibling-accuracy salvage scan failed", exc_info=True) return None - acc = eval_data.get("accuracy") - if acc is not None and float(acc) > 0.0: - return eval_data - return None + if _finite_score(eval_data.get("accuracy")) is None: + return None + return eval_data async def _run_once( self, From b8bcfabbf4bfac530393dfaf4f5d6dfb620158a2 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Mon, 3 Aug 2026 06:18:28 +0000 Subject: [PATCH 08/22] feat(reference-script): lift the literal default out of ${FOO:-value} exports `export FOO=${FOO:-1}` is the idiomatic overridable default in InferenceX recipes, but `_extract_envs` skipped every value containing `$`, so those settings never reached the lifted recipe even though the script applies them whenever the caller does not override. The whitelisted default is now resolved to its literal, and quote stripping runs before the check so the quoted form is covered too. Only the self-referential form counts: `export FOO=${BAR:-1}` depends on an unrelated variable, so its default is not FOO's effective value here and that line is still skipped, as is any default that itself contains a `$`. Co-authored-by: Cursor --- .../inference_optimizer/reference_script.py | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/src/hyperloom/inference_optimizer/reference_script.py b/src/hyperloom/inference_optimizer/reference_script.py index fefd296bb7..dd24aecb04 100644 --- a/src/hyperloom/inference_optimizer/reference_script.py +++ b/src/hyperloom/inference_optimizer/reference_script.py @@ -204,7 +204,12 @@ def parse_reference_script(source: str, *, framework: str) -> ReferenceRecipe: def _extract_envs(text: str) -> dict[str, str]: - """Pull whitelisted ``export KEY=VALUE`` lines whose value has no ``$``.""" + """Pull whitelisted ``export KEY=VALUE`` lines that resolve to a literal. + + ``export FOO=${FOO:-1}`` is the idiomatic overridable-default form in every + InferenceX recipe, so its literal default is lifted too; any other ``$`` + reference is unresolvable here and the line is skipped. + """ envs: dict[str, str] = {} pat = re.compile(r"^\s*export\s+([A-Za-z_][A-Za-z0-9_]*)=(\S+)\s*$") for line in text.splitlines(): @@ -214,15 +219,35 @@ def _extract_envs(text: str) -> dict[str, str]: key, val = m.group(1), m.group(2) if key not in _ENV_WHITELIST: continue - if _has_var(val): - continue # strip surrounding quotes if present if len(val) >= 2 and val[0] == val[-1] and val[0] in ("'", '"'): val = val[1:-1] + if _has_var(val): + resolved = _resolve_self_default(key, val) + if resolved is None: + continue + val = resolved envs[key] = val return envs +# ``${FOO:-1}`` / ``${FOO-1}``, capturing the name and the default. +_SELF_DEFAULT_RE = re.compile(r"^\$\{([A-Za-z_][A-Za-z0-9_]*):?-(.*)\}$") + + +def _resolve_self_default(key: str, val: str) -> str | None: + """Return the literal default of ``${key:-default}``, else ``None``. + + Only the *self*-referential form counts: ``export FOO=${BAR:-1}`` depends on + an unrelated variable, so its default is not FOO's effective value here. + """ + m = _SELF_DEFAULT_RE.match(val) + if not m or m.group(1) != key: + return None + default = m.group(2) + return None if _has_var(default) else default + + def _extract_server_args( tokens: list[str], framework: str, From a0d708e151294c5d53e694088daa5dedb640c8e8 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Mon, 3 Aug 2026 06:40:25 +0000 Subject: [PATCH 09/22] fix(policy): admit tracked enablement revalidation baselines Keep the baseline singleton for normal work while allowing the Coordinator's identified accuracy revalidation to reach its executor. Co-authored-by: Cursor --- .../tests/test_dispatched_task_policy.py | 26 +++++++++++++ .../orchestrator/loop/sub_agent_runner.py | 1 + src/hyperloom/orchestrator/policy/gate.py | 38 +++++++++---------- 3 files changed, 45 insertions(+), 20 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_dispatched_task_policy.py b/src/hyperloom/inference_optimizer/tests/test_dispatched_task_policy.py index cb03d176e1..bbecdcc768 100644 --- a/src/hyperloom/inference_optimizer/tests/test_dispatched_task_policy.py +++ b/src/hyperloom/inference_optimizer/tests/test_dispatched_task_policy.py @@ -224,6 +224,32 @@ def test_validate_dispatched_task_internal_profile_skips_delegate_body(tmp_path, gate.validate_dispatched_task("profile", {"reason": "watermark_refresh"}) +@pytest.mark.asyncio +async def test_dispatched_tracked_enablement_revalidation_bypasses_baseline_singleton(tmp_path, monkeypatch): + sub = _runner_with_policy(tmp_path, monkeypatch) + state = sub.shared_state + assert isinstance(state, SharedState) + state.baseline_tput = 1000.0 + executed = {"ran": False} + + async def _stub(_ctx) -> dict: + executed["ran"] = True + return {"status": "ok"} + + sub.register_executor("baseline", _stub) + task = await sub.tasks.create( + kind="baseline", + params={"reason": "enablement_eval_revalidation"}, + idempotency_key="enablement-revalidation", + ) + state.enablement_revalidation_task_id = task.task_id + + result = await sub.run_task(task) + + assert result.state == "succeeded" + assert executed["ran"] is True + + def test_validate_dispatched_task_skips_phase_incompatible(tmp_path, monkeypatch): gate, _sd = _gate(tmp_path, monkeypatch, strict_phase=True) state = gate.shared_state diff --git a/src/hyperloom/orchestrator/loop/sub_agent_runner.py b/src/hyperloom/orchestrator/loop/sub_agent_runner.py index 92f6256833..482190ee76 100644 --- a/src/hyperloom/orchestrator/loop/sub_agent_runner.py +++ b/src/hyperloom/orchestrator/loop/sub_agent_runner.py @@ -215,6 +215,7 @@ async def run_task( self.policy.validate_dispatched_task( task.kind, dict(task.params or {}), + task_id=task.task_id, ) except PolicyDenied as denied: await self._transition_resilient( diff --git a/src/hyperloom/orchestrator/policy/gate.py b/src/hyperloom/orchestrator/policy/gate.py index 6e3d2c247b..2cdf6ee742 100644 --- a/src/hyperloom/orchestrator/policy/gate.py +++ b/src/hyperloom/orchestrator/policy/gate.py @@ -690,6 +690,8 @@ def validate_dispatched_task( self, action_name: str, params: dict[str, Any] | None, + *, + task_id: str = "", ) -> None: """Re-validate a persisted queued task before executor dispatch. @@ -703,6 +705,8 @@ def validate_dispatched_task( Args: action_name: The task ``kind`` / delegate action name. params: Task params deserialized from the DB row. + task_id: Persisted task id, used to admit the tracked enablement + revalidation baseline. Raises: PolicyDenied: When the task would have been rejected had it @@ -727,7 +731,17 @@ def validate_dispatched_task( # conc_sweep against itself and surface as a spurious conc_sweep_failed. if kind in COORDINATOR_INTERNAL_ACTIONS: return - self._validate_delegate_body(role, payload, check_phase=False) + tracked_revalidation = ( + kind == BASELINE_ACTION_NAME + and bool(task_id) + and str(task_id) == str(getattr(self.shared_state, "enablement_revalidation_task_id", "") or "") + ) + self._validate_delegate_body( + role, + payload, + check_phase=False, + skip_baseline_singleton=tracked_revalidation, + ) def _closing_phase_denial( self, @@ -820,6 +834,7 @@ def _validate_delegate_body( payload: dict[str, Any], *, check_phase: bool, + skip_baseline_singleton: bool = False, ) -> None: """Shared delegate validation for intents and dispatched task rows. @@ -863,7 +878,7 @@ def _validate_delegate_body( # conc_sweep as Coordinator-managed (phase_incompatible) below. if action_name == SWEEP_ACTION_NAME: self._validate_sweep_singleton(payload, intent_kind="delegate") - if action_name == BASELINE_ACTION_NAME: + if action_name == BASELINE_ACTION_NAME and not skip_baseline_singleton: self._validate_baseline_singleton(payload, intent_kind="delegate") self._validate_gemm_tuning_action(action_name, intent_kind="delegate") # Refuse delegate for unknown action names when an ActionRegistry is wired (no registry → fall through). @@ -1431,24 +1446,7 @@ def _validate_baseline_singleton( *, intent_kind: str, ) -> None: - """Deny an LLM ``baseline`` once the session has an anchor. - - PRELUDE allows ``baseline`` so the run can reach ``baseline_tput > 0``, - and nothing retired it afterwards: a run could keep re-measuring a - reference it already had, at roughly twenty GPU-minutes a turn, while - the newest measurement silently redefined every later gain. Escape: - ``params.bypass_baseline_singleton=True``. - - Args: - payload (dict[str, Any]): the intent payload; - ``params.bypass_baseline_singleton`` opts out of the guard. - intent_kind (str): the channel the action arrived on, used in the - error hint. - - Raises: - PolicyDenied: when ``baseline_tput`` is already positive and no - bypass flag is set. - """ + """Deny a repeat baseline once the session has an anchor.""" params = payload.get("params") or {} if isinstance(params, dict) and params.get("bypass_baseline_singleton"): return From d8b1b208c97d810af8f5672d7bfb80656ed3f600 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Mon, 3 Aug 2026 07:17:20 +0000 Subject: [PATCH 10/22] feat(trace): preserve orchestration turn diagnostics Persist redacted Claude turn state and MCP setup so failed control-plane calls can be diagnosed after a session ends. Co-authored-by: Cursor --- .../session/session_paths.py | 12 + .../orchestrator/loop/coordinator.py | 120 +++++++++ src/hyperloom/orchestrator/roles/claude.py | 229 +++++++++++++++++- src/hyperloom/orchestrator/trace/__init__.py | 10 + .../orchestrator/trace/orchestration_trace.py | 219 +++++++++++++++++ 5 files changed, 583 insertions(+), 7 deletions(-) create mode 100644 src/hyperloom/orchestrator/trace/orchestration_trace.py diff --git a/src/hyperloom/inference_optimizer/session/session_paths.py b/src/hyperloom/inference_optimizer/session/session_paths.py index 7ed17d3740..437dfce606 100644 --- a/src/hyperloom/inference_optimizer/session/session_paths.py +++ b/src/hyperloom/inference_optimizer/session/session_paths.py @@ -387,6 +387,11 @@ def conversations_path(session_dir: Path) -> Path: return trace_dir(session_dir) / "conversations.jsonl" +def orchestration_turns_path(session_dir: Path) -> Path: + """``/reports/trace/orchestration_turns.jsonl``.""" + return trace_dir(session_dir) / "orchestration_turns.jsonl" + + def research_hints_md(session_dir: Path) -> Path: """``/research_hints.md`` — human-readable proven-prior hints collected by the research scout. @@ -457,6 +462,11 @@ def agent_prompt_snapshot(session_dir: Path, role: str) -> Path: return agent_dir(session_dir, role) / "system_prompt.snapshot.md" +def agent_mcp_setup_path(session_dir: Path, role: str) -> Path: + """Compute the per-agent MCP setup snapshot path.""" + return agent_dir(session_dir, role) / "mcp_setup.json" + + # External baseline comparison artefacts. Dedicated top-level subdir (not # runs/) because target_analysis is a prep-phase action. def target_analysis_dir(session_dir: Path) -> Path: @@ -758,6 +768,7 @@ def allocate_turn_workdir(session_dir: Path, subdir: str, turn_idx: int, *, keep __all__ = [ "allocate_turn_workdir", "agent_dir", + "agent_mcp_setup_path", "agent_prompt_snapshot", "breakdown_parts_dir", "competitor_target_json", @@ -780,6 +791,7 @@ def allocate_turn_workdir(session_dir: Path, subdir: str, turn_idx: int, *, keep "kernel_agent_runs_root", "llm_calls_path", "manifest_path", + "orchestration_turns_path", "patches_dir", "reports_dir", "research_hints_json", diff --git a/src/hyperloom/orchestrator/loop/coordinator.py b/src/hyperloom/orchestrator/loop/coordinator.py index fa376dfd55..d311b9ee44 100644 --- a/src/hyperloom/orchestrator/loop/coordinator.py +++ b/src/hyperloom/orchestrator/loop/coordinator.py @@ -12,6 +12,7 @@ import signal import time import traceback +import uuid from dataclasses import dataclass, field from pathlib import Path from typing import Any, Awaitable, Callable @@ -77,6 +78,11 @@ from .sub_agent_runner import SubAgentRunner from ..state.task_registry import TaskRegistry from ..trace.llm_trace import LLMCallRecord, append_llm_call +from ..trace.orchestration_trace import ( + OrchestrationTurnRecord, + append_orchestration_turn, + write_mcp_setup_once, +) from .coordinator_helpers import ( _infer_model_class_from_config, effective_closing_grace_sec, @@ -1742,6 +1748,16 @@ async def _reactor_pass(self, agent_name: str) -> None: max_turns=0, ) except BackendError as exc: + self._trace_orchestration_turn( + agent_name=agent_name, + backend=backend, + prompt=prompt, + system_prompt=sys_prompt, + tools=tools, + outcome="backend_error", + error=exc, + latency_ms=int((time.perf_counter() - _t0) * 1000), + ) await self._record_observation( "coordinator", "observation", @@ -1750,6 +1766,16 @@ async def _reactor_pass(self, agent_name: str) -> None: await self._track_backend_error_streak(agent_name, exc) return except NoIntentEmitted as exc: + self._trace_orchestration_turn( + agent_name=agent_name, + backend=backend, + prompt=prompt, + system_prompt=sys_prompt, + tools=tools, + outcome="no_intent", + error=exc, + latency_ms=int((time.perf_counter() - _t0) * 1000), + ) # No parseable intents; surface as observation so the next tick self-corrects. await self._record_observation( "coordinator", @@ -1758,6 +1784,16 @@ async def _reactor_pass(self, agent_name: str) -> None: ) return except Exception as exc: # noqa: BLE001 + self._trace_orchestration_turn( + agent_name=agent_name, + backend=backend, + prompt=prompt, + system_prompt=sys_prompt, + tools=tools, + outcome="exception", + error=exc, + latency_ms=int((time.perf_counter() - _t0) * 1000), + ) # Catch-all so one agent's bad turn never stops the loop. log.exception("reactor pass for %s raised", agent_name) await self._record_observation( @@ -1777,6 +1813,16 @@ async def _reactor_pass(self, agent_name: str) -> None: self._backend_error_alarm_armed[agent_name] = True # Record this reactor turn's token spend on the unified ledger. latency_ms = int((time.perf_counter() - _t0) * 1000) + self._trace_orchestration_turn( + agent_name=agent_name, + backend=backend, + prompt=prompt, + system_prompt=sys_prompt, + tools=tools, + outcome="succeeded", + result=result, + latency_ms=latency_ms, + ) self._trace_reactor_llm_call(agent_name, result, latency_ms=latency_ms) # Full-trace: persist the redacted prompt+response for this turn. self._record_reactor_conversation(agent_name, result) @@ -1801,6 +1847,80 @@ async def _reactor_pass(self, agent_name: str) -> None: for intent in result.intents: await self._handle_intent(agent_name, intent) + def _trace_orchestration_turn( + self, + *, + agent_name: str, + backend: Backend, + prompt: str, + system_prompt: str, + tools: list[str], + outcome: str, + result: BackendTurnResult | None = None, + error: BaseException | None = None, + latency_ms: int | None = None, + ) -> None: + """Append one orchestration diagnostic row.""" + if agent_name != "orchestration": + return + try: + getter = getattr(backend, "get_turn_diagnostic", None) + diagnostic = getter() if callable(getter) else {} + if not isinstance(diagnostic, dict): + diagnostic = {} + metadata = result.metadata if result is not None else {} + err_trace = ( + "".join(traceback.format_exception(type(error), error, error.__traceback__))[-4000:] + if error is not None + else None + ) + record = OrchestrationTurnRecord( + session_id=self.session_dir.name, + turn_id=uuid.uuid4().hex, + tick=int(self.shared_state.tick or 0), + phase=(self.shared_state.phase or "") or None, + outcome=outcome, + backend=str(diagnostic.get("backend") or type(backend).__name__), + model=diagnostic.get("model") or metadata.get("model") or getattr(backend, "model", None), + sdk_name=diagnostic.get("sdk_name"), + sdk_version=diagnostic.get("sdk_version"), + cli_version=diagnostic.get("cli_version"), + gateway_endpoint=diagnostic.get("gateway_endpoint"), + request_id=diagnostic.get("request_id"), + resume_requested=bool(diagnostic.get("resume_requested", False)), + previous_session_id_hash=diagnostic.get("previous_session_id_hash"), + session_id_hash=diagnostic.get("session_id_hash"), + new_session=diagnostic.get("new_session"), + max_turns=diagnostic.get("max_turns"), + timeout_sec=diagnostic.get("timeout_sec"), + reasoning_effort=diagnostic.get("reasoning_effort"), + thinking=diagnostic.get("thinking"), + prompt=str(diagnostic.get("prompt") or prompt), + system_prompt=str(diagnostic.get("system_prompt") or system_prompt), + allowed_tools=list(diagnostic.get("allowed_tools") or tools), + mcp_servers=list(diagnostic.get("mcp_servers") or []), + emit_intent_registered=bool(diagnostic.get("emit_intent_registered", False)), + messages=list(diagnostic.get("messages") or []), + result=str(diagnostic.get("result") or metadata.get("response") or ""), + raw_text=str(diagnostic.get("raw_text") or getattr(result, "raw_text", "") or ""), + tool_blocks=list(diagnostic.get("tool_blocks") or []), + parse_errors=list(diagnostic.get("parse_errors") or []), + usage=dict(diagnostic.get("usage") or {}), + stderr_tail=list(diagnostic.get("stderr_tail") or []), + sdk_boundary_error=diagnostic.get("sdk_boundary_error"), + error_type=type(error).__name__ if error is not None else None, + error_message=str(error) if error is not None else None, + traceback=err_trace, + ) + append_orchestration_turn(session_dir=self.session_dir, record=record) + setup_getter = getattr(backend, "get_mcp_setup_diagnostic", None) + if callable(setup_getter): + setup = setup_getter() + if isinstance(setup, dict): + write_mcp_setup_once(session_dir=self.session_dir, setup=setup) + except Exception: # noqa: BLE001 + log.debug("orchestration turn trace failed", exc_info=True) + def _trace_reactor_llm_call( self, agent_name: str, diff --git a/src/hyperloom/orchestrator/roles/claude.py b/src/hyperloom/orchestrator/roles/claude.py index 0392264b65..8eb9a6b68b 100644 --- a/src/hyperloom/orchestrator/roles/claude.py +++ b/src/hyperloom/orchestrator/roles/claude.py @@ -13,11 +13,14 @@ from __future__ import annotations import asyncio +import hashlib import importlib +import json import logging import os from dataclasses import dataclass, field from typing import Any, Callable +from urllib.parse import urlsplit from hyperloom.common.llm_config import claude_sdk_env_options from hyperloom.inference_optimizer.protocol.intent import ( @@ -43,6 +46,7 @@ from .mcp_emit_intent import ( EMIT_INTENT_TOOL_NAME, EMIT_INTENT_TOOL_QUALIFIED, + EMIT_INTENT_TOOL_INPUT_SCHEMA, MCP_SERVER_NAME, build_emit_intent_server, ) @@ -196,6 +200,10 @@ class ClaudeBackend: # Read-only context-pull MCP server config, set via # ``set_context_provider`` and merged into the SDK options. _context_server_config: Any | None = field(default=None, init=False) + _mcp_setup_error: str | None = field(default=None, init=False) + _active_turn_diagnostic: dict[str, Any] | None = field(default=None, init=False) + _last_turn_diagnostic: dict[str, Any] = field(default_factory=dict, init=False) + _active_stderr: list[str] = field(default_factory=list, init=False) def __post_init__(self) -> None: """Resolve the SDK and optionally register the ``emit_intent`` tool. @@ -252,6 +260,7 @@ def __post_init__(self) -> None: ) except Exception as exc: # noqa: BLE001 self.calls.append({"warn": f"emit_intent MCP setup failed: {exc!r}"}) + self._mcp_setup_error = f"{type(exc).__name__}: {exc}" cfg = None if cfg is not None: self.mcp_server_config = cfg @@ -284,6 +293,11 @@ async def run( The parsed :class:`BackendTurnResult` for the turn. """ full_prompt = self._compose_prompt(prompt) + self._begin_turn_diagnostic( + prompt=full_prompt, + system_prompt=system_prompt, + tools=tools or [], + ) max_turns_use = max_turns or self.max_turns_default # Claude Code counts the model's own text/tool messages as turns, so a # literal max_turns=1 trips ("Reached maximum number of turns (1)") @@ -294,11 +308,20 @@ async def run( # raw-completion floor to every mode, not just raw_completion. max_turns_use = max(max_turns_use, _RAW_COMPLETION_MIN_MAX_TURNS) resume_session = self._session_id if self.conversational else None - options = self._build_options( - tools=tools or [], + try: + options = self._build_options( + tools=tools or [], + max_turns=max_turns_use, + system_prompt=system_prompt, + resume_session_id=resume_session, + ) + except BaseException as exc: + self._finish_turn_diagnostic(outcome="backend_error", error=exc) + raise + self._update_turn_options( + options, max_turns=max_turns_use, - system_prompt=system_prompt, - resume_session_id=resume_session, + resume_session=resume_session, ) # Each attempt bounds the gap BETWEEN streamed SDK messages (silence @@ -361,9 +384,17 @@ def _note_retry(attempt: int, exc: BaseException, delay: float) -> None: ), } ) - raise BackendError( + error = BackendError( f"Claude backend timed out: stream idle for >{self.call_timeout_s:.0f}s (likely upstream proxy stall)" - ) from exc + ) + self._finish_turn_diagnostic(outcome="backend_error", error=error) + raise error from exc + except BaseException as exc: + self._finish_turn_diagnostic(outcome="backend_error", error=exc) + raise + if self._active_turn_diagnostic is not None: + self._active_turn_diagnostic["session_id_hash"] = self._session_hash(session_id) + self._active_turn_diagnostic["new_session"] = bool(session_id and not resume_session) # Capture the SDK session token for the next conversational resume; # only overwrite on a non-empty id. if self.conversational: @@ -383,6 +414,13 @@ def _note_retry(attempt: int, exc: BaseException, delay: float) -> None: cache_read = safe_int(usage.get("cache_read_input_tokens") if usage else None) input_tokens = safe_int(usage.get("input_tokens") if usage else None) output_tokens = safe_int(usage.get("output_tokens") if usage else None) + if self._active_turn_diagnostic is not None: + self._active_turn_diagnostic["usage"] = { + "cache_creation_input_tokens": cache_creation, + "cache_read_input_tokens": cache_read, + "input_tokens": input_tokens, + "output_tokens": output_tokens, + } self.calls.append( { "prompt_chars": len(full_prompt), @@ -396,10 +434,13 @@ def _note_retry(attempt: int, exc: BaseException, delay: float) -> None: } ) if not intents and not self.raw_completion and not allow_no_intent: - raise NoIntentEmitted( + error = NoIntentEmitted( f"claude reply contained no parseable emit_intent tool_use " f"blocks (raw_text_len={len(raw_text)}, tool_blocks={tool_block_count})" ) + self._finish_turn_diagnostic(outcome="no_intent", error=error) + raise error + self._finish_turn_diagnostic(outcome="succeeded") return BackendTurnResult( intents=intents, raw_text=raw_text, @@ -468,6 +509,138 @@ def reset_conversation(self) -> None: """ self._session_id = None + def get_turn_diagnostic(self) -> dict[str, Any]: + """Return the most recently completed turn diagnostic.""" + return dict(self._last_turn_diagnostic) + + def get_mcp_setup_diagnostic(self) -> dict[str, Any]: + """Return the current MCP setup snapshot.""" + schema = json.dumps(EMIT_INTENT_TOOL_INPUT_SCHEMA, sort_keys=True, separators=(",", ":")) + diag = self.get_turn_diagnostic() + return { + "backend": type(self).__name__, + "model": self.model, + "sdk_name": getattr(self.sdk_module, "__name__", None), + "sdk_version": getattr(self.sdk_module, "__version__", None), + "cli_version": os.environ.get("CLAUDE_CODE_VERSION") or None, + "gateway_endpoint": self._gateway_endpoint_identifier(), + "mcp_servers": diag.get("mcp_servers", []), + "emit_intent": { + "qualified_name": EMIT_INTENT_TOOL_QUALIFIED, + "registered": bool(self.mcp_server_config is not None and self.mcp_tool_name), + "schema_sha256": hashlib.sha256(schema.encode("utf-8")).hexdigest(), + "setup_error": self._mcp_setup_error, + }, + "allowed_tools": diag.get("allowed_tools", []), + } + + def _begin_turn_diagnostic( + self, + *, + prompt: str, + system_prompt: str | None, + tools: list[str], + ) -> None: + previous_session = self._session_id if self.conversational else None + self._active_stderr = [] + self._active_turn_diagnostic = { + "backend": type(self).__name__, + "model": self.model, + "sdk_name": getattr(self.sdk_module, "__name__", None), + "sdk_version": getattr(self.sdk_module, "__version__", None), + "cli_version": os.environ.get("CLAUDE_CODE_VERSION") or None, + "gateway_endpoint": self._gateway_endpoint_identifier(), + "resume_requested": bool(previous_session), + "previous_session_id_hash": self._session_hash(previous_session), + "session_id_hash": None, + "new_session": None, + "max_turns": None, + "timeout_sec": self.call_timeout_s, + "reasoning_effort": None, + "thinking": None, + "prompt": prompt, + "system_prompt": system_prompt or "", + "allowed_tools": list(tools), + "mcp_servers": [], + "emit_intent_registered": bool(self.mcp_server_config is not None and self.mcp_tool_name), + "messages": [], + "result": "", + "raw_text": "", + "tool_blocks": [], + "parse_errors": [], + "usage": {}, + "stderr_tail": [], + } + + def _update_turn_options(self, options: Any, *, max_turns: int, resume_session: str | None) -> None: + diag = self._active_turn_diagnostic + if diag is None: + return + kwargs = getattr(options, "kwargs", None) + if not isinstance(kwargs, dict): + kwargs = {} + allowed = kwargs.get("allowed_tools", getattr(options, "allowed_tools", diag["allowed_tools"])) + servers = kwargs.get("mcp_servers", getattr(options, "mcp_servers", {})) + if not kwargs: + if self.raw_completion: + allowed = [] + servers = {} + else: + allowed = [tool for tool in diag["allowed_tools"] if tool != EMIT_INTENT_TOOL_NAME] + if self.mcp_tool_name and self.mcp_tool_name not in allowed: + allowed.append(self.mcp_tool_name) + if self._context_server_config is not None: + allowed.extend(tool for tool in CONTEXT_TOOL_QUALIFIED_NAMES if tool not in allowed) + servers = {} + if self.mcp_server_config is not None: + servers[MCP_SERVER_NAME] = self.mcp_server_config + if self._context_server_config is not None: + servers[CONTEXT_MCP_SERVER_NAME] = self._context_server_config + diag["max_turns"] = max_turns + diag["resume_requested"] = bool(resume_session) + diag["allowed_tools"] = [str(tool) for tool in allowed or []] + diag["mcp_servers"] = sorted(str(name) for name in (servers or {})) + role_env = _EFFORT_ENV_ORCH if self.conversational else _EFFORT_ENV_KERNEL + default_effort = "medium" if self.conversational else "low" + diag["reasoning_effort"] = kwargs.get( + "effort", + getattr(options, "effort", (os.environ.get(role_env) or os.environ.get(_EFFORT_ENV) or default_effort).strip()), + ) + diag["thinking"] = kwargs.get( + "thinking", + getattr(options, "thinking", {"type": (os.environ.get(_THINKING_ENV) or "adaptive").strip().lower()}), + ) + + def _finish_turn_diagnostic(self, *, outcome: str, error: BaseException | None = None) -> None: + diag = self._active_turn_diagnostic + if diag is None: + return + diag["outcome"] = outcome + diag["stderr_tail"] = self._active_stderr[-50:] + if error is not None: + diag["error_type"] = type(error).__name__ + diag["error_message"] = str(error) + self._last_turn_diagnostic = diag + self._active_turn_diagnostic = None + + def _gateway_endpoint_identifier(self) -> str | None: + raw = ( + os.environ.get("ANTHROPIC_BASE_URL") + or os.environ.get("DEEPSEEK_BASE_URL") + or os.environ.get("OPENAI_BASE_URL") + or "" + ).strip() + if not raw: + return None + parts = urlsplit(raw) + return parts.netloc or "configured" + + @staticmethod + def _session_hash(session_id: str | None) -> str | None: + if not session_id: + return None + return hashlib.sha256(session_id.encode("utf-8")).hexdigest() + def _build_options( self, *, @@ -572,6 +745,7 @@ def _stderr_sink(self, line: str) -> None: text = line.strip() if text: self.calls.append({"stderr": text}) + self._active_stderr.append(text) async def _invoke_and_collect( self, prompt: str, options: Any, *, idle_timeout_s: float | None = None @@ -615,7 +789,9 @@ async def _invoke_and_collect( msg_session = getattr(message, "session_id", None) if isinstance(msg_session, str) and msg_session: session_id = msg_session + self._record_message_diagnostic(message) for block in self._iter_blocks(message): + self._record_tool_block_diagnostic(block) if self._is_tool_use_for_emit_intent(block): tool_block_count += 1 intent = self._parse_tool_use_block(block) @@ -646,6 +822,8 @@ async def _invoke_and_collect( err_str = str(exc) _non_fatal = "error result: success" in err_str or "maximum number of turns" in err_str if _non_fatal: + if self._active_turn_diagnostic is not None: + self._active_turn_diagnostic["sdk_boundary_error"] = err_str if intents: log.warning( "claude SDK raised '%s' but %d intents already collected; returning partial results", @@ -671,6 +849,9 @@ async def _invoke_and_collect( pass # Prefer the consolidated ResultMessage text; fall back to TextBlocks. raw_text = "".join(result_chunks) or "".join(text_chunks) + if self._active_turn_diagnostic is not None: + self._active_turn_diagnostic["result"] = "".join(result_chunks) + self._active_turn_diagnostic["raw_text"] = raw_text return intents, raw_text, tool_block_count, last_usage, session_id @staticmethod @@ -727,9 +908,43 @@ def _parse_tool_use_block(self, block: Any) -> Intent | None: validated = validate_envelope(envelope) except IntentValidationError as exc: log.info("claude tool_use validation failed: %s", exc) + if self._active_turn_diagnostic is not None: + self._active_turn_diagnostic["parse_errors"].append(str(exc)) return None return validated[0] if validated else None + def _record_message_diagnostic(self, message: Any) -> None: + diag = self._active_turn_diagnostic + if diag is None: + return + summary: dict[str, Any] = {"type": type(message).__name__} + for name in ("is_error", "subtype", "request_id"): + value = getattr(message, name, None) + if value is not None: + summary[name] = value + if name == "request_id" and not diag.get("request_id"): + diag["request_id"] = str(value) + result = getattr(message, "result", None) + if isinstance(result, str): + summary["result"] = result + diag["messages"].append(summary) + + def _record_tool_block_diagnostic(self, block: Any) -> None: + diag = self._active_turn_diagnostic + if diag is None: + return + summary: dict[str, Any] = {"type": type(block).__name__} + name = getattr(block, "name", None) + if isinstance(name, str) and name: + summary["name"] = name + raw_input = getattr(block, "input", None) + if isinstance(raw_input, dict): + summary["input_keys"] = sorted(str(key) for key in raw_input) + intent_type = raw_input.get("intent_type") + if isinstance(intent_type, str): + summary["intent_type"] = intent_type + diag["tool_blocks"].append(summary) + @staticmethod def _extract_text(block: Any) -> str: """Extract plain text from a content block across block shapes. diff --git a/src/hyperloom/orchestrator/trace/__init__.py b/src/hyperloom/orchestrator/trace/__init__.py index b65a21fb60..ad076d651d 100644 --- a/src/hyperloom/orchestrator/trace/__init__.py +++ b/src/hyperloom/orchestrator/trace/__init__.py @@ -29,6 +29,12 @@ LLMTraceRowError, append_llm_call, ) +from .orchestration_trace import ( + OrchestrationTraceRowError, + OrchestrationTurnRecord, + append_orchestration_turn, + write_mcp_setup_once, +) from .langfuse_emitter import flush_session, get_emitter from .parse_usage import ( normalize_usage, @@ -41,12 +47,16 @@ "ConversationRowError", "LLMCallRecord", "LLMTraceRowError", + "OrchestrationTraceRowError", + "OrchestrationTurnRecord", "append_conversation", "append_llm_call", + "append_orchestration_turn", "flush_session", "get_emitter", "langfuse_live_enabled", "normalize_usage", "parse_claude_stream_json_usage", "redact_secrets", + "write_mcp_setup_once", ] diff --git a/src/hyperloom/orchestrator/trace/orchestration_trace.py b/src/hyperloom/orchestrator/trace/orchestration_trace.py new file mode 100644 index 0000000000..8970d61654 --- /dev/null +++ b/src/hyperloom/orchestrator/trace/orchestration_trace.py @@ -0,0 +1,219 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Durable diagnostics for orchestration turns.""" + +from __future__ import annotations + +import hashlib +import logging +from dataclasses import dataclass, fields +from pathlib import Path +from typing import Any + +from hyperloom.common.io import append_jsonl, atomic_write_json +from hyperloom.common.timeutil import now_iso +from hyperloom.inference_optimizer.session.session_paths import ( + agent_mcp_setup_path, + orchestration_turns_path, +) + +from .conversation_trace import redact_secrets + +log = logging.getLogger(__name__) + +_ROW_FIELDS: frozenset[str] = frozenset( + { + "session_id", + "turn_id", + "ts", + "tick", + "phase", + "outcome", + "backend", + "model", + "sdk_name", + "sdk_version", + "cli_version", + "gateway_endpoint", + "request_id", + "resume_requested", + "previous_session_id_hash", + "session_id_hash", + "new_session", + "max_turns", + "timeout_sec", + "reasoning_effort", + "thinking", + "prompt_sha256", + "prompt_chars", + "system_prompt_sha256", + "system_prompt_chars", + "allowed_tools", + "mcp_servers", + "emit_intent_registered", + "messages", + "result", + "raw_text", + "tool_blocks", + "parse_errors", + "usage", + "stderr_tail", + "sdk_boundary_error", + "error_type", + "error_message", + "traceback", + } +) + + +class OrchestrationTraceRowError(ValueError): + """Raised when an orchestration trace row violates its schema.""" + + +def _sha256(value: str | None) -> str: + return hashlib.sha256((value or "").encode("utf-8")).hexdigest() + + +def _safe_value(value: Any) -> Any: + if isinstance(value, str): + return redact_secrets(value) + if isinstance(value, dict): + return {str(key): _safe_value(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_safe_value(item) for item in value] + if value is None or isinstance(value, (bool, int, float)): + return value + return redact_secrets(str(value)) + + +@dataclass +class OrchestrationTurnRecord: + """One orchestration backend invocation.""" + + session_id: str + turn_id: str + tick: int | None + phase: str | None + outcome: str + backend: str + model: str | None + sdk_name: str | None + sdk_version: str | None + cli_version: str | None + gateway_endpoint: str | None + request_id: str | None + resume_requested: bool + previous_session_id_hash: str | None + session_id_hash: str | None + new_session: bool | None + max_turns: int | None + timeout_sec: float | None + reasoning_effort: str | None + thinking: Any + prompt: str + system_prompt: str + allowed_tools: list[str] + mcp_servers: list[str] + emit_intent_registered: bool + messages: list[dict[str, Any]] + result: str + raw_text: str + tool_blocks: list[dict[str, Any]] + parse_errors: list[str] + usage: dict[str, Any] + stderr_tail: list[str] + sdk_boundary_error: str | None = None + error_type: str | None = None + error_message: str | None = None + traceback: str | None = None + + def to_row(self) -> dict[str, Any]: + """Serialize a redacted append-only row.""" + return { + "session_id": str(self.session_id), + "turn_id": str(self.turn_id), + "ts": now_iso(), + "tick": self.tick, + "phase": self.phase, + "outcome": str(self.outcome), + "backend": str(self.backend), + "model": self.model, + "sdk_name": self.sdk_name, + "sdk_version": self.sdk_version, + "cli_version": self.cli_version, + "gateway_endpoint": self.gateway_endpoint, + "request_id": self.request_id, + "resume_requested": bool(self.resume_requested), + "previous_session_id_hash": self.previous_session_id_hash, + "session_id_hash": self.session_id_hash, + "new_session": self.new_session, + "max_turns": self.max_turns, + "timeout_sec": self.timeout_sec, + "reasoning_effort": self.reasoning_effort, + "thinking": _safe_value(self.thinking), + "prompt_sha256": _sha256(self.prompt), + "prompt_chars": len(self.prompt), + "system_prompt_sha256": _sha256(self.system_prompt), + "system_prompt_chars": len(self.system_prompt), + "allowed_tools": _safe_value(self.allowed_tools), + "mcp_servers": _safe_value(self.mcp_servers), + "emit_intent_registered": bool(self.emit_intent_registered), + "messages": _safe_value(self.messages), + "result": _safe_value(self.result), + "raw_text": _safe_value(self.raw_text), + "tool_blocks": _safe_value(self.tool_blocks), + "parse_errors": _safe_value(self.parse_errors), + "usage": _safe_value(self.usage), + "stderr_tail": _safe_value(self.stderr_tail), + "sdk_boundary_error": _safe_value(self.sdk_boundary_error), + "error_type": self.error_type, + "error_message": _safe_value(self.error_message), + "traceback": _safe_value(self.traceback), + } + + +def append_orchestration_turn(*, session_dir: Path, record: OrchestrationTurnRecord) -> None: + """Append an orchestration turn without disrupting the coordinator.""" + row = record.to_row() + if set(row) != _ROW_FIELDS: + raise OrchestrationTraceRowError("orchestration_turns row violates closed schema") + if not row["session_id"].strip(): + raise OrchestrationTraceRowError("orchestration_turns row requires session_id") + try: + append_jsonl(orchestration_turns_path(session_dir), row, make_parents=True, ensure_ascii=False, sort_keys=True) + except OSError as exc: + log.warning("orchestration_trace: append failed for session_id=%s: %r", record.session_id, exc) + + +def write_mcp_setup_once(*, session_dir: Path, setup: dict[str, Any]) -> None: + """Persist the orchestration MCP setup once per session.""" + path = agent_mcp_setup_path(session_dir, "orchestration") + if path.exists(): + return + try: + atomic_write_json( + path, + {"ts": now_iso(), "schema_version": 1, **_safe_value(setup)}, + ensure_ascii=False, + trailing_newline=True, + mode=0o600, + ) + except OSError as exc: + log.warning("orchestration_trace: MCP setup write failed for %s: %r", session_dir.name, exc) + + +_DATACLASS_FIELDS = frozenset(field.name for field in fields(OrchestrationTurnRecord)) +assert ( + (_DATACLASS_FIELDS - {"prompt", "system_prompt"}) + | {"ts", "prompt_sha256", "prompt_chars", "system_prompt_sha256", "system_prompt_chars"} + == _ROW_FIELDS +) + + +__all__ = [ + "OrchestrationTraceRowError", + "OrchestrationTurnRecord", + "append_orchestration_turn", + "write_mcp_setup_once", +] From 23cffe6b423a53a422f96de8f271af5c88e1ad20 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Mon, 3 Aug 2026 07:17:49 +0000 Subject: [PATCH 11/22] test(trace): cover orchestration diagnostics Verify no-intent details, durable turn records, MCP snapshots, and the new session paths. Co-authored-by: Cursor --- .../test_claude_backend_branches_unit.py | 4 ++ .../test_coordinator_async_batch2_unit.py | 52 +++++++++++++++++++ .../tests/test_session_paths_unit.py | 2 + 3 files changed, 58 insertions(+) diff --git a/src/hyperloom/inference_optimizer/tests/test_claude_backend_branches_unit.py b/src/hyperloom/inference_optimizer/tests/test_claude_backend_branches_unit.py index 085f4d4a55..e94ef3f661 100644 --- a/src/hyperloom/inference_optimizer/tests/test_claude_backend_branches_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_claude_backend_branches_unit.py @@ -250,6 +250,10 @@ async def test_run_no_intent_raises(): b.sdk_query_factory = _query([msg]) with pytest.raises(NoIntentEmitted): await b.run("hi") + diag = b.get_turn_diagnostic() + assert diag["outcome"] == "no_intent" + assert diag["raw_text"] == "hi" + assert diag["messages"] == [{"type": "_Msg", "result": "hi"}] # ---- _invoke_and_collect: error-result-success tolerance ------------------ diff --git a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py index 209b45a1b8..32bca877c2 100644 --- a/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py @@ -7,6 +7,7 @@ from __future__ import annotations +import json from pathlib import Path from types import SimpleNamespace @@ -69,6 +70,57 @@ def test_context_inbox_reader_empty(coord: Coordinator) -> None: assert out == "(no inbox events)" +def test_trace_orchestration_turn_persists_diagnostics(coord: Coordinator) -> None: + backend = SimpleNamespace( + model="claude-test", + get_turn_diagnostic=lambda: { + "backend": "ClaudeBackend", + "model": "claude-test", + "sdk_name": "claude_agent_sdk", + "sdk_version": "1.2.3", + "resume_requested": True, + "previous_session_id_hash": "old", + "session_id_hash": "new", + "new_session": False, + "max_turns": 12, + "timeout_sec": 300.0, + "prompt": "prompt", + "system_prompt": "system", + "allowed_tools": ["mcp__inference_optimizer__emit_intent"], + "mcp_servers": ["inference_optimizer"], + "emit_intent_registered": True, + "messages": [{"type": "ResultMessage", "is_error": False, "result": "done"}], + "result": "done", + "raw_text": "done", + "tool_blocks": [], + "parse_errors": [], + "usage": {"input_tokens": 3}, + "stderr_tail": [], + }, + get_mcp_setup_diagnostic=lambda: { + "sdk_name": "claude_agent_sdk", + "emit_intent": {"registered": True}, + }, + ) + + coord._trace_orchestration_turn( + agent_name="orchestration", + backend=backend, + prompt="prompt", + system_prompt="system", + tools=["emit_intent"], + outcome="no_intent", + error=RuntimeError("missing intent"), + ) + + row = json.loads((coord.session_dir / "reports" / "trace" / "orchestration_turns.jsonl").read_text()) + setup = json.loads((coord.session_dir / "agents" / "orchestration" / "mcp_setup.json").read_text()) + assert row["outcome"] == "no_intent" + assert row["resume_requested"] is True + assert row["error_type"] == "RuntimeError" + assert setup["emit_intent"]["registered"] is True + + @pytest.mark.asyncio async def test_context_inbox_reader_with_events(coord: Coordinator) -> None: await coord.bus.append_and_seq(Message.new("kernel_agent", "orchestration", "heartbeat", {"body_md": "hi"})) diff --git a/src/hyperloom/inference_optimizer/tests/test_session_paths_unit.py b/src/hyperloom/inference_optimizer/tests/test_session_paths_unit.py index ed08ed47b6..7e8a47ab57 100644 --- a/src/hyperloom/inference_optimizer/tests/test_session_paths_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_session_paths_unit.py @@ -89,6 +89,7 @@ def test_trace_paths(): assert sp.llm_calls_path(SD).name == "llm_calls.jsonl" assert sp.decision_trace_path(SD).name == "decision_trace.jsonl" assert sp.conversations_path(SD).name == "conversations.jsonl" + assert sp.orchestration_turns_path(SD).name == "orchestration_turns.jsonl" assert sp.proposal_task_map_path(SD).name == "proposal_task_map.jsonl" @@ -101,6 +102,7 @@ def test_research_and_competitor_paths(): def test_agent_paths(): assert sp.agent_dir(SD, "critic") == SD / "agents" / "critic" assert sp.agent_prompt_snapshot(SD, "critic").name == "system_prompt.snapshot.md" + assert sp.agent_mcp_setup_path(SD, "orchestration").name == "mcp_setup.json" def test_target_analysis_paths(): From d13d51df7c993792123bde280ced4c6969513f8c Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Mon, 3 Aug 2026 07:52:01 +0000 Subject: [PATCH 12/22] fix(orchestrator): preserve valid revalidation anchors Keep full-stack revalidation and framework patch decisions aligned with live state while preventing agent-controlled baseline bypasses and unnecessary backend diagnostics. Co-authored-by: Cursor --- .../inference_optimizer/cli/backends.py | 1 + .../inference_optimizer/reference_script.py | 7 +--- .../test_claude_backend_branches_unit.py | 10 +++++- .../tests/test_dispatched_task_policy.py | 25 +++++++++++++ .../tests/test_explore_executor.py | 26 ++++++++------ .../tests/test_framework_agent_executor.py | 36 +++++++++++++++++++ .../tests/test_reference_script.py | 14 ++++++++ .../tests/test_sweep_phase_auto.py | 26 +++++++------- .../actions/executors/baseline.py | 2 +- .../orchestrator/actions/executors/explore.py | 8 ++--- .../actions/executors/framework_agent.py | 8 ++--- .../actions/executors/integrate_patch.py | 5 +-- .../orchestrator/loop/coordinator.py | 5 --- src/hyperloom/orchestrator/loop/writeback.py | 22 ++---------- src/hyperloom/orchestrator/policy/gate.py | 26 ++++---------- src/hyperloom/orchestrator/roles/claude.py | 8 ++++- .../orchestrator/roles/mcp_emit_intent.py | 2 +- 17 files changed, 140 insertions(+), 91 deletions(-) diff --git a/src/hyperloom/inference_optimizer/cli/backends.py b/src/hyperloom/inference_optimizer/cli/backends.py index 4a98e2882f..a6ffd1749a 100644 --- a/src/hyperloom/inference_optimizer/cli/backends.py +++ b/src/hyperloom/inference_optimizer/cli/backends.py @@ -254,6 +254,7 @@ def _build_backends( model=claude_model, max_turns_default=4, conversational=True, + capture_turn_diagnostics=True, ) backends: dict[str, Any] = { diff --git a/src/hyperloom/inference_optimizer/reference_script.py b/src/hyperloom/inference_optimizer/reference_script.py index dd24aecb04..a32ee3075f 100644 --- a/src/hyperloom/inference_optimizer/reference_script.py +++ b/src/hyperloom/inference_optimizer/reference_script.py @@ -204,12 +204,7 @@ def parse_reference_script(source: str, *, framework: str) -> ReferenceRecipe: def _extract_envs(text: str) -> dict[str, str]: - """Pull whitelisted ``export KEY=VALUE`` lines that resolve to a literal. - - ``export FOO=${FOO:-1}`` is the idiomatic overridable-default form in every - InferenceX recipe, so its literal default is lifted too; any other ``$`` - reference is unresolvable here and the line is skipped. - """ + """Pull whitelisted literal exports, including self-referential defaults.""" envs: dict[str, str] = {} pat = re.compile(r"^\s*export\s+([A-Za-z_][A-Za-z0-9_]*)=(\S+)\s*$") for line in text.splitlines(): diff --git a/src/hyperloom/inference_optimizer/tests/test_claude_backend_branches_unit.py b/src/hyperloom/inference_optimizer/tests/test_claude_backend_branches_unit.py index e94ef3f661..b067091976 100644 --- a/src/hyperloom/inference_optimizer/tests/test_claude_backend_branches_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_claude_backend_branches_unit.py @@ -246,7 +246,7 @@ async def test_run_conversational_session_capture(monkeypatch): # ---- run(): no-intent raises ---------------------------------------------- async def test_run_no_intent_raises(): msg = _Msg(content=[TextBlock("just text")], result="hi") - b = _backend() + b = _backend(capture_turn_diagnostics=True) b.sdk_query_factory = _query([msg]) with pytest.raises(NoIntentEmitted): await b.run("hi") @@ -256,6 +256,14 @@ async def test_run_no_intent_raises(): assert diag["messages"] == [{"type": "_Msg", "result": "hi"}] +async def test_run_skips_diagnostics_when_not_requested(): + msg = _Msg(content=[TextBlock("just text")], result="hi") + b = _backend() + b.sdk_query_factory = _query([msg]) + await b.run("hi", allow_no_intent=True) + assert b.get_turn_diagnostic() == {} + + # ---- _invoke_and_collect: error-result-success tolerance ------------------ async def test_invoke_error_result_success_with_intents(): msg = _Msg(content=[_emit_tool_block()]) diff --git a/src/hyperloom/inference_optimizer/tests/test_dispatched_task_policy.py b/src/hyperloom/inference_optimizer/tests/test_dispatched_task_policy.py index bbecdcc768..bea0a37e67 100644 --- a/src/hyperloom/inference_optimizer/tests/test_dispatched_task_policy.py +++ b/src/hyperloom/inference_optimizer/tests/test_dispatched_task_policy.py @@ -250,6 +250,31 @@ async def _stub(_ctx) -> dict: assert executed["ran"] is True +@pytest.mark.asyncio +async def test_dispatched_baseline_singleton_bypass_is_denied(tmp_path, monkeypatch): + sub = _runner_with_policy(tmp_path, monkeypatch) + state = sub.shared_state + assert isinstance(state, SharedState) + state.baseline_tput = 1000.0 + executed = {"ran": False} + + async def _stub(_ctx) -> dict: + executed["ran"] = True + return {"status": "ok"} + + sub.register_executor("baseline", _stub) + task = await sub.tasks.create( + kind="baseline", + params={"bypass_baseline_singleton": True}, + idempotency_key="rejected-rebaseline", + ) + + result = await sub.run_task(task) + + assert result.state == "failed" + assert executed["ran"] is False + + def test_validate_dispatched_task_skips_phase_incompatible(tmp_path, monkeypatch): gate, _sd = _gate(tmp_path, monkeypatch, strict_phase=True) state = gate.shared_state diff --git a/src/hyperloom/inference_optimizer/tests/test_explore_executor.py b/src/hyperloom/inference_optimizer/tests/test_explore_executor.py index b594fdd58b..353c5a5e3a 100644 --- a/src/hyperloom/inference_optimizer/tests/test_explore_executor.py +++ b/src/hyperloom/inference_optimizer/tests/test_explore_executor.py @@ -740,17 +740,22 @@ def _fake_run(cmd, *args, **kwargs): @pytest.mark.asyncio +@pytest.mark.parametrize( + ("source", "expected_base_tput", "expected_outcome", "has_winner"), + [ + (None, 2358.80, "REVERT", False), + ("resume_stack_revalidate", 2192.52, "KEEP", True), + ], +) async def test_explore_executor_supersedes_stale_params_base_tput( sub_agent_runner, tmp_path, + source, + expected_base_tput, + expected_outcome, + has_winner, ): - """A queued task's stale ``base_tput`` is superseded by the live anchor. - - Reproduces MiniMax-M3-MXFP8 session 96879: the task was queued against the - bare baseline (2192.5) while a warm replay had already lifted current_best - to 2358.8, so a 2355.5 variant read as +7.4% and was KEEP'd even though it - regressed the recipe by 0.14%. - """ + """Use the live anchor except when revalidating the complete stack.""" sub, tr, _ = sub_agent_runner state = SharedState() state.baseline_tput = 2195.86 @@ -789,6 +794,7 @@ def _fake_run(cmd, *args, **kwargs): "base_tput": 2192.52, "grid": grid, "variant_timeout_sec": 10, + **({"source": source} if source else {}), }, idempotency_key="ex-stale-anchor", ) @@ -801,11 +807,11 @@ def _fake_run(cmd, *args, **kwargs): out = res.result assert out["status"] == "succeeded" - assert out["winners"] == [] + assert bool(out["winners"]) is has_winner fp = canonical_fingerprint("--fused-flag", {}) tested = out["explore_search_update"]["tested"][fp] - assert tested["base_tput"] == 2358.80 - assert tested["outcome"] == "REVERT" + assert tested["base_tput"] == expected_base_tput + assert tested["outcome"] == expected_outcome @pytest.mark.asyncio diff --git a/src/hyperloom/inference_optimizer/tests/test_framework_agent_executor.py b/src/hyperloom/inference_optimizer/tests/test_framework_agent_executor.py index bb7ff61474..b0427c12c5 100644 --- a/src/hyperloom/inference_optimizer/tests/test_framework_agent_executor.py +++ b/src/hyperloom/inference_optimizer/tests/test_framework_agent_executor.py @@ -26,6 +26,7 @@ VariantResult, ) from hyperloom.orchestrator.loop.sub_agent_runner import RunnerContext +from hyperloom.orchestrator.state.shared_state import SharedState from hyperloom.orchestrator.state.task_registry import Task @@ -359,6 +360,41 @@ async def fake_bench(self, *, params, output_root, slug): # noqa: ARG001 assert (repo / "src.py").read_text().endswith("return 2\n") +@pytest.mark.asyncio +async def test_executor_reverts_when_live_anchor_exceeds_queued_baseline(tmp_path: Path): + session_dir = tmp_path / "session" + session_dir.mkdir() + repo = tmp_path / "framework" + init_git_repo(repo) + patch_path = tmp_path / "p.patch" + patch_path.write_text(_VALID_PATCH, encoding="utf-8") + state = SharedState() + state.baseline_tput = 1000.0 + state.current_best = {"tput": 1150.0} + + async def fake_bench(self, *, params, output_root, slug): # noqa: ARG001 + return {"status": "succeeded", "output_throughput": 1100.0}, {"accuracy_pass": None} + + ctx = _make_ctx( + "t-fp-stale-anchor", + { + "candidate": _make_candidate(), + "patches": [str(patch_path)], + "framework_source_root": str(repo), + "base_tput": 1000.0, + "keep_threshold_pct": 1.0, + }, + ) + ctx.extra["shared_state"] = state + executor = FrameworkAgentExecutor(session_dir=session_dir) + with patch.object(FrameworkAgentExecutor, "_bench_candidate", new=fake_bench): + result = await executor(ctx) + + assert result["status"] == "reverted" + assert result["base_tput"] == 1150.0 + assert (repo / "src.py").read_text().endswith("return 1\n") + + @pytest.mark.asyncio async def test_executor_keep_writes_kb_lessons(tmp_path: Path, monkeypatch): """A KEEP appends an 'integrated' record to lessons.jsonl for dedup.""" diff --git a/src/hyperloom/inference_optimizer/tests/test_reference_script.py b/src/hyperloom/inference_optimizer/tests/test_reference_script.py index f7aad010c0..7913a2f5f4 100644 --- a/src/hyperloom/inference_optimizer/tests/test_reference_script.py +++ b/src/hyperloom/inference_optimizer/tests/test_reference_script.py @@ -93,6 +93,20 @@ def test_parse_env_whitelist(tmp_path): assert "SOME_SECRET" not in r.envs +def test_parse_env_self_defaults(tmp_path): + text = """\ +export VLLM_USE_BREAKABLE_CUDAGRAPH=${VLLM_USE_BREAKABLE_CUDAGRAPH:-0} +export NCCL_DMABUF_ENABLE="${NCCL_DMABUF_ENABLE-1}" +export VLLM_ROCM_USE_AITER=${OTHER:-1} +export VLLM_USE_RUST_FRONTEND=${VLLM_USE_RUST_FRONTEND:-$DEFAULT} +""" + r = parse_reference_script(_write(tmp_path, text), framework="vllm") + assert r.envs["VLLM_USE_BREAKABLE_CUDAGRAPH"] == "0" + assert r.envs["NCCL_DMABUF_ENABLE"] == "1" + assert "VLLM_ROCM_USE_AITER" not in r.envs + assert "VLLM_USE_RUST_FRONTEND" not in r.envs + + def test_parse_continuation_parity(tmp_path): """A \\-continuation recipe parses identically to its single-line form.""" multi = _write(tmp_path, _M3_RECIPE, "multi.sh") diff --git a/src/hyperloom/inference_optimizer/tests/test_sweep_phase_auto.py b/src/hyperloom/inference_optimizer/tests/test_sweep_phase_auto.py index e513edb43c..1d03830d3d 100644 --- a/src/hyperloom/inference_optimizer/tests/test_sweep_phase_auto.py +++ b/src/hyperloom/inference_optimizer/tests/test_sweep_phase_auto.py @@ -1227,8 +1227,7 @@ def __init__(self, baseline_tput: float = 0.0): self.baseline_tput = baseline_tput -@pytest.mark.parametrize("intent_kind", ["delegate", "propose_action"]) -def test_baseline_singleton_denies_once_anchor_is_established(intent_kind): +def test_baseline_singleton_denies_once_anchor_is_established(): """Both channels refuse a repeat baseline after baseline_tput turns positive.""" from hyperloom.orchestrator.policy.gate import PolicyDenied @@ -1237,10 +1236,9 @@ def test_baseline_singleton_denies_once_anchor_is_established(intent_kind): with pytest.raises(PolicyDenied) as excinfo: gate._validate_baseline_singleton( payload={"action_name": "baseline", "params": {}}, - intent_kind=intent_kind, ) assert excinfo.value.rule == "baseline_phase_singleton" - assert "bypass_baseline_singleton" in (excinfo.value.hint or "") + assert "PRELUDE is done with baseline" in (excinfo.value.hint or "") def test_baseline_singleton_inert_before_the_anchor_exists(): @@ -1248,7 +1246,6 @@ def test_baseline_singleton_inert_before_the_anchor_exists(): gate = _make_policy_gate(shared_state=_BaselineSingletonState(0.0)) gate._validate_baseline_singleton( payload={"action_name": "baseline", "params": {}}, - intent_kind="propose_action", ) @@ -1256,19 +1253,20 @@ def test_baseline_singleton_inert_when_shared_state_is_none(): gate = _make_policy_gate(shared_state=None) gate._validate_baseline_singleton( payload={"action_name": "baseline"}, - intent_kind="delegate", ) -def test_baseline_singleton_bypass_flag_lets_operator_force_a_fresh_anchor(): +def test_baseline_singleton_bypass_flag_is_rejected(): + from hyperloom.orchestrator.policy.gate import PolicyDenied + gate = _make_policy_gate(shared_state=_BaselineSingletonState(2195.86)) - gate._validate_baseline_singleton( - payload={ - "action_name": "baseline", - "params": {"bypass_baseline_singleton": True}, - }, - intent_kind="delegate", - ) + with pytest.raises(PolicyDenied): + gate._validate_baseline_singleton( + payload={ + "action_name": "baseline", + "params": {"bypass_baseline_singleton": True}, + }, + ) # 6b. End-to-end through full validate_intent (delegate / propose_action) diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index ecaade8cb3..d2a4e1c6db 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -1780,7 +1780,7 @@ def _salvage_sibling_baseline_accuracy( framework: Framework name threaded into the eval parser. Returns: - The parsed eval dict when a positive accuracy is found, else + The parsed eval dict when a finite accuracy is found, else ``None``. """ out = result.get("output_dir") diff --git a/src/hyperloom/orchestrator/actions/executors/explore.py b/src/hyperloom/orchestrator/actions/executors/explore.py index 73a9da9873..c82f4a7775 100644 --- a/src/hyperloom/orchestrator/actions/executors/explore.py +++ b/src/hyperloom/orchestrator/actions/executors/explore.py @@ -599,13 +599,11 @@ async def __call__(self, ctx) -> dict[str, Any]: base_unset_envs = to_str_list(params.get("base_unset_envs")) base_args_mode = str(params.get("base_args_mode") or "append").strip().lower() base_tput = float(params.get("base_tput") or 0.0) - # ``base_tput`` is snapshotted when the task is queued, so it goes stale - # whenever current_best advances before the task runs. Grading against - # the stale value lets a regression read as a win, so always take the - # live anchor when it is higher (this also covers an absent param). + # Grade candidates against the live anchor; revalidation uses baseline. ss = extra.get("shared_state") or extra.get("state") live_anchor = resolve_grading_anchor_tput(ss) - if live_anchor > base_tput: + revalidating_stack = params.get("source") == "resume_stack_revalidate" + if not revalidating_stack and live_anchor > base_tput: if base_tput > 0: log.warning( "explore: anchor drift, params base_tput=%.1f but live anchor is %.1f; grading against live", diff --git a/src/hyperloom/orchestrator/actions/executors/framework_agent.py b/src/hyperloom/orchestrator/actions/executors/framework_agent.py index 1fdabd3538..21c59f9d8c 100644 --- a/src/hyperloom/orchestrator/actions/executors/framework_agent.py +++ b/src/hyperloom/orchestrator/actions/executors/framework_agent.py @@ -53,6 +53,7 @@ OUTCOME_REVERTED_SMOKE_FAIL, write_framework_record, ) +from ...state.shared_state import resolve_grading_anchor_tput log = logging.getLogger(__name__) @@ -844,10 +845,9 @@ async def __call__(self, ctx) -> dict[str, Any]: # KEEP / REVERT decision. base_tput = float(params.get("base_tput") or 0.0) - if base_tput <= 0: - ss = extra.get("shared_state") or extra.get("state") - if ss is not None: - base_tput = float(getattr(ss, "baseline_tput", 0.0) or 0.0) + live_anchor = resolve_grading_anchor_tput(extra.get("shared_state") or extra.get("state")) + if live_anchor > base_tput: + base_tput = live_anchor keep_threshold_pct = float( params.get("keep_threshold_pct", self.keep_threshold_pct), ) diff --git a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py index c4079eef3f..c17e59aab8 100644 --- a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py @@ -54,7 +54,6 @@ sanitize_result_dir, sanitize_script_name, ) -from ._grid_server_args import merge_server_args, split_config_changes from ._stack_rebench import DEFAULT_STACK_STABLE_PCT, measure_stack_rebench from ._workload_envs import ( FrameworkScriptMismatchError, @@ -1420,7 +1419,6 @@ async def __call__(self, ctx) -> dict[str, Any]: # _stage_resolve populates these onto ctx for stage communication. specialist_task_id: str = ctx._ip_specialist_task_id # type: ignore[attr-defined] shared_state = ctx._ip_shared_state # type: ignore[attr-defined] - specialist_workspace: Path = ctx._ip_specialist_workspace # type: ignore[attr-defined] done_payload: dict[str, Any] = ctx._ip_done_payload # type: ignore[attr-defined] # Provision an attempt-scoped runtime AFTER the Critic gate (in @@ -2657,8 +2655,7 @@ async def _gate_perf( ) -> dict[str, Any]: """Throughput KEEP / REVERT decision with optional stack rebench.""" base_tput = float(params.get("base_tput") or 0.0) - # Same stale-snapshot hazard as explore: prefer the live anchor whenever - # it is higher than whatever the task was queued with. + # Grade against the current live anchor, not a stale task snapshot. live_anchor = resolve_grading_anchor_tput(shared_state) if live_anchor > base_tput: if base_tput > 0: diff --git a/src/hyperloom/orchestrator/loop/coordinator.py b/src/hyperloom/orchestrator/loop/coordinator.py index d311b9ee44..974597c506 100644 --- a/src/hyperloom/orchestrator/loop/coordinator.py +++ b/src/hyperloom/orchestrator/loop/coordinator.py @@ -1756,7 +1756,6 @@ async def _reactor_pass(self, agent_name: str) -> None: tools=tools, outcome="backend_error", error=exc, - latency_ms=int((time.perf_counter() - _t0) * 1000), ) await self._record_observation( "coordinator", @@ -1774,7 +1773,6 @@ async def _reactor_pass(self, agent_name: str) -> None: tools=tools, outcome="no_intent", error=exc, - latency_ms=int((time.perf_counter() - _t0) * 1000), ) # No parseable intents; surface as observation so the next tick self-corrects. await self._record_observation( @@ -1792,7 +1790,6 @@ async def _reactor_pass(self, agent_name: str) -> None: tools=tools, outcome="exception", error=exc, - latency_ms=int((time.perf_counter() - _t0) * 1000), ) # Catch-all so one agent's bad turn never stops the loop. log.exception("reactor pass for %s raised", agent_name) @@ -1821,7 +1818,6 @@ async def _reactor_pass(self, agent_name: str) -> None: tools=tools, outcome="succeeded", result=result, - latency_ms=latency_ms, ) self._trace_reactor_llm_call(agent_name, result, latency_ms=latency_ms) # Full-trace: persist the redacted prompt+response for this turn. @@ -1858,7 +1854,6 @@ def _trace_orchestration_turn( outcome: str, result: BackendTurnResult | None = None, error: BaseException | None = None, - latency_ms: int | None = None, ) -> None: """Append one orchestration diagnostic row.""" if agent_name != "orchestration": diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index b7f4796778..4e0948c2fc 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -2030,13 +2030,7 @@ def _lift_to_current_best( *, gap_canonical_id: str = "", ) -> bool: - """Update SharedState.current_best + recompute cumulative_gain; gap_canonical_id (when known) is stamped onto the stack entry so provenance resolves by gap id not name. - - ``current_best`` never moves down: a winner that does not beat the - anchor it was composed on top of is refused here even if its own - executor called it a KEEP, so a stale task-level ``base_tput`` cannot - regress the recipe. Revalidation flows confirm the existing stack and - never reach this method. + """Lift a winner only when it improves the current throughput anchor. Args: task_kind: The action kind that produced the winner (stamped on the @@ -2558,13 +2552,7 @@ async def _promote_baseline( outcome.audit_extras = audit_extras async def _drain_queued_baselines(self, *, reason: str) -> list[str]: - """Cancel queued baselines that the established anchor has made redundant. - - Baseline is LLM-proposable and nothing stopped a run from queueing more - of them while the first was still measuring, so a succeeded baseline can - leave a backlog that re-measures a number the session already has. The - enablement revalidation baseline is spared: it re-anchors a stack the - specialist changed, so it is not redundant work. + """Cancel redundant queued baselines, preserving enablement revalidation. Args: reason: Stamped onto the cancellation history and the observation. @@ -2603,11 +2591,7 @@ async def _drain_queued_baselines(self, *, reason: str) -> list[str]: return cancelled async def _enablement_revalidation_task_ids(self) -> set[str]: - """Queued baseline task ids that re-anchor an enablement-changed stack. - - Matches the identity ``_promote_baseline`` uses: the tracked task id, or - a params reason recorded before the id was persisted. - """ + """Return queued enablement-revalidation baseline task IDs.""" spared: set[str] = set() tracked = str(getattr(self.shared_state, "enablement_revalidation_task_id", "") or "").strip() if tracked: diff --git a/src/hyperloom/orchestrator/policy/gate.py b/src/hyperloom/orchestrator/policy/gate.py index 2cdf6ee742..c94217939c 100644 --- a/src/hyperloom/orchestrator/policy/gate.py +++ b/src/hyperloom/orchestrator/policy/gate.py @@ -34,7 +34,6 @@ ) from ..specialists.domains import ( KNOWLEDGE_DOMAIN_TAG_SET, - SPECIALIST_DOMAIN_KEYS, SPECIALIST_MAX_TURNS_HARD_CAP, domain_for_tag, get_domain, @@ -731,7 +730,7 @@ def validate_dispatched_task( # conc_sweep against itself and surface as a spurious conc_sweep_failed. if kind in COORDINATOR_INTERNAL_ACTIONS: return - tracked_revalidation = ( + skip_baseline_singleton = ( kind == BASELINE_ACTION_NAME and bool(task_id) and str(task_id) == str(getattr(self.shared_state, "enablement_revalidation_task_id", "") or "") @@ -740,7 +739,7 @@ def validate_dispatched_task( role, payload, check_phase=False, - skip_baseline_singleton=tracked_revalidation, + skip_baseline_singleton=skip_baseline_singleton, ) def _closing_phase_denial( @@ -879,7 +878,7 @@ def _validate_delegate_body( if action_name == SWEEP_ACTION_NAME: self._validate_sweep_singleton(payload, intent_kind="delegate") if action_name == BASELINE_ACTION_NAME and not skip_baseline_singleton: - self._validate_baseline_singleton(payload, intent_kind="delegate") + self._validate_baseline_singleton(payload) self._validate_gemm_tuning_action(action_name, intent_kind="delegate") # Refuse delegate for unknown action names when an ActionRegistry is wired (no registry → fall through). if self.action_registry is not None and self.action_registry.get(action_name) is None: @@ -977,10 +976,7 @@ def _validate_propose_action(self, role: "AgentRole", payload: dict[str, Any]) - intent_kind="propose_action", ) if action_name == BASELINE_ACTION_NAME: - self._validate_baseline_singleton( - payload, - intent_kind="propose_action", - ) + self._validate_baseline_singleton(payload) # Per-action source allowlist (e.g. ``recover`` is robustness-only); mirrors the delegate-path guard. allowed_sources = DELEGATE_ACTION_SOURCE_ALLOWLIST.get(action_name) if allowed_sources is not None and role.name not in allowed_sources: @@ -1443,13 +1439,8 @@ def _validate_sweep_singleton( def _validate_baseline_singleton( self, payload: dict[str, Any], - *, - intent_kind: str, ) -> None: """Deny a repeat baseline once the session has an anchor.""" - params = payload.get("params") or {} - if isinstance(params, dict) and params.get("bypass_baseline_singleton"): - return ss = getattr(self, "shared_state", None) if ss is None: return @@ -1464,13 +1455,8 @@ def _validate_baseline_singleton( ), rule="baseline_phase_singleton", hint=( - "PRELUDE is done with baseline; let the phase advance. " - "A measurement you distrust is a reason to re-measure the " - "candidate, not the reference. " - f"If you genuinely need a fresh anchor, set " - f"params.bypass_baseline_singleton=True on the " - f"{intent_kind} payload (the override is recorded " - f"on the audit trail)." + "PRELUDE is done with baseline; let the phase advance and " + "re-measure the candidate rather than the reference." ), ) diff --git a/src/hyperloom/orchestrator/roles/claude.py b/src/hyperloom/orchestrator/roles/claude.py index 8eb9a6b68b..335d617b00 100644 --- a/src/hyperloom/orchestrator/roles/claude.py +++ b/src/hyperloom/orchestrator/roles/claude.py @@ -157,6 +157,8 @@ class ClaudeBackend: max_turns_default: agent-loop budget when caller doesn't override. enable_mcp_emit_intent: if True (default), registers the in-process MCP ``emit_intent`` tool. + capture_turn_diagnostics: Capture full turn diagnostics for durable + orchestration tracing. """ model: str | None = None @@ -167,6 +169,7 @@ class ClaudeBackend: # feeding only a per-tick delta. kernel / critic / robustness stay stateless. conversational: bool = False enable_mcp_emit_intent: bool = True + capture_turn_diagnostics: bool = False # Raw single-shot completion mode: skips the emit_intent server + suffix, # disallows all tools, and returns ``raw_text`` without an emitted intent. raw_completion: bool = False @@ -541,6 +544,8 @@ def _begin_turn_diagnostic( system_prompt: str | None, tools: list[str], ) -> None: + if not self.capture_turn_diagnostics: + return previous_session = self._session_id if self.conversational else None self._active_stderr = [] self._active_turn_diagnostic = { @@ -745,7 +750,8 @@ def _stderr_sink(self, line: str) -> None: text = line.strip() if text: self.calls.append({"stderr": text}) - self._active_stderr.append(text) + if self._active_turn_diagnostic is not None: + self._active_stderr.append(text) async def _invoke_and_collect( self, prompt: str, options: Any, *, idle_timeout_s: float | None = None diff --git a/src/hyperloom/orchestrator/roles/mcp_emit_intent.py b/src/hyperloom/orchestrator/roles/mcp_emit_intent.py index f5c14ed852..fd9ceee3f6 100644 --- a/src/hyperloom/orchestrator/roles/mcp_emit_intent.py +++ b/src/hyperloom/orchestrator/roles/mcp_emit_intent.py @@ -49,7 +49,7 @@ "approve|reject|redirect|advise|needs_review}, " "kill_task:{task_id,reason}, " "extend_lease:{task_id,extra_sec,reason}, " - "prune_branch:{family,reason,scope ∈ family|queued}, " + "prune_branch:{family, optional reason, scope ∈ family|queued}, " "escalate_strategy_change:" "{reason,next_action_hint}, update_state:{changes}, " "alert:{severity,summary}." From 336803652343c9c00e3a6d3fd3d5b3552407d455 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Mon, 3 Aug 2026 09:01:22 +0000 Subject: [PATCH 13/22] feat(eval): add generation-pathology kind and probe sidecar readers Adds EVAL_KIND_GENERATION_PATHOLOGY to the existing eval-failure taxonomy, plus read_eval_probe / eval_probe_summary for the sidecar the lm-eval probe writes when it cuts a runaway eval short. The probe knobs join the eval-contract keys so a baseline and a variant evaluated under different settings no longer fingerprint as comparable. No caller yet; the probe that produces the sidecar lands separately. --- .../actions/executors/_accuracy_gate.py | 66 ++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py b/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py index ecebc4ee09..b2e4bac4e3 100644 --- a/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py +++ b/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py @@ -68,6 +68,14 @@ EVAL_KIND_RUNTIME_FAILURE = "eval_runtime_failure" EVAL_KIND_ACCURACY_UNAVAILABLE = "accuracy_unavailable" EVAL_KIND_ACCURACY_BELOW_FLOOR = "accuracy_below_floor" +# The model never emitted EOS, so the eval was cut short and scored ~0. +# Distinct from ``accuracy_below_floor``: a broken generation loop, not a model +# that answered and got them wrong. +EVAL_KIND_GENERATION_PATHOLOGY = "eval_generation_pathology" + +# Sidecar the probe writes into ``$RESULT_DIR`` when it trips. Deliberately not +# ``results*.json``: :func:`parse_eval_results` globs that name for the score. +EVAL_PROBE_FILENAME = "hyperloom_eval_probe.json" # stop_reason recorded when the baseline could not produce an accuracy result # even though the accuracy test was expected to run. A broken baseline accuracy @@ -201,11 +209,15 @@ def _extract_eval_contract_fields(config_path: str | Path | None) -> dict[str, s bench = data.get("benchmark") or {} envs: dict = bench.get("envs") or {} - # Eval-contract keys in benchmark.envs; all others are excluded. + # Eval-contract keys in benchmark.envs; all others are excluded. The probe + # knobs belong here: they change how early an eval is cut short. _EVAL_CONTRACT_ENV_KEYS = ( "RUN_EVAL", "MAGPIE_EVAL_TASKS", "MAGPIE_EVAL_LIMIT", + "HYPERLOOM_EVAL_PROBE", + "HYPERLOOM_EVAL_PROBE_MIN_SAMPLES", + "HYPERLOOM_EVAL_PROBE_LENGTH_RATIO", ) # Workload-shape keys that define what is being measured. _WORKLOAD_SHAPE_ENV_KEYS = ( @@ -555,6 +567,54 @@ def parse_eval_results( return {"accuracy": None, "error": f"no recognized metric in {latest}"} +def read_eval_probe(workspace: Path | str) -> dict[str, Any] | None: + """Read the generation-pathology probe sidecar, when the probe tripped. + + The probe only writes :data:`EVAL_PROBE_FILENAME` when it cuts an eval + short, so ``None`` means the ordinary thing happened: the model terminated + its answers, or the InferenceX patch never applied. Searched recursively + because the baseline double-run evaluates in the warmup round, whose + ``$RESULT_DIR`` nests under the task workspace. + + Args: + workspace (Path | str): Benchmark workspace to search recursively. + + Returns: + dict[str, Any] | None: The probe record stamped with ``kind`` and + ``source_file``, or ``None`` when no readable sidecar exists. + """ + matches = sorted(Path(workspace).rglob(EVAL_PROBE_FILENAME)) + if not matches: + return None + latest = matches[-1] + try: + record = json.loads(latest.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return None + record["kind"] = EVAL_KIND_GENERATION_PATHOLOGY + record["source_file"] = str(latest) + return record + + +def eval_probe_summary(probe: dict[str, Any] | None) -> str: + """Render a one-line summary of a tripped probe for a log / journal reason. + + Args: + probe (dict[str, Any] | None): A :func:`read_eval_probe` record. + + Returns: + str: The summary, or ``""`` when ``probe`` is empty. + """ + if not probe: + return "" + return ( + f"{EVAL_KIND_GENERATION_PATHOLOGY}: {probe.get('finish_reason_length', 0)}/" + f"{probe.get('observed_samples', 0)} sampled responses hit the max_tokens cap " + f"(up to {probe.get('max_completion_tokens_seen', 0)} tokens); the model never " + "emitted EOS, so the eval was cut short and scored ~0" + ) + + def accuracy_passed( baseline_accuracy: float, new_accuracy: float, @@ -598,7 +658,9 @@ def accuracy_passed( "ENABLEMENT_MODE_OFF", "EVAL_KIND_ACCURACY_BELOW_FLOOR", "EVAL_KIND_ACCURACY_UNAVAILABLE", + "EVAL_KIND_GENERATION_PATHOLOGY", "EVAL_KIND_RUNTIME_FAILURE", + "EVAL_PROBE_FILENAME", "_extract_eval_contract_fields", "accuracy_keep_block", "accuracy_meets_floor", @@ -606,9 +668,11 @@ def accuracy_passed( "classify_accuracy_failure", "eval_contract_fingerprint", "eval_enablement_allowed", + "eval_probe_summary", "is_high_accuracy_risk", "launch_enablement_allowed", "parse_eval_results", + "read_eval_probe", "request_baseline_accuracy_stop", "resolve_enablement_mode", "require_framework_accuracy_default", From f24811b597e95c234aeb490d1d07ab44f9a6b23d Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Mon, 3 Aug 2026 09:01:22 +0000 Subject: [PATCH 14/22] feat(breakdown): carry writeback audit extras into baseline attempt history record_action_attempt already persists an arbitrary extras dict per attempt, but collect_baseline dropped it, so anything the writeback audit recorded was invisible in session_breakdown.json. Pass it through and declare it on BaselineAttemptSummary. Additive only: schema.py states schema_version bumps on breaking changes, so SCHEMA_VERSION is unchanged. --- .../inference_optimizer/breakdown/collectors/sessions.py | 3 +++ src/hyperloom/inference_optimizer/breakdown/schema.py | 4 ++++ src/hyperloom/orchestrator/loop/writeback.py | 3 +++ 3 files changed, 10 insertions(+) diff --git a/src/hyperloom/inference_optimizer/breakdown/collectors/sessions.py b/src/hyperloom/inference_optimizer/breakdown/collectors/sessions.py index adef1921b5..023587eb86 100644 --- a/src/hyperloom/inference_optimizer/breakdown/collectors/sessions.py +++ b/src/hyperloom/inference_optimizer/breakdown/collectors/sessions.py @@ -838,6 +838,7 @@ def collect_baseline( "error_excerpt": a.get("error_excerpt"), "stderr_tail": a.get("stderr_tail"), "stderr_log_path": a.get("stderr_log_path"), + "extras": a.get("extras") or {}, } ) @@ -994,6 +995,8 @@ def _reconstruct_baseline_attempts( "error_excerpt": None, "stderr_tail": None, "stderr_log_path": None, + # Reconstruction reads the on-disk reports; audit extras are state-only. + "extras": {}, } ) return out diff --git a/src/hyperloom/inference_optimizer/breakdown/schema.py b/src/hyperloom/inference_optimizer/breakdown/schema.py index 289ca71eba..cb1f46d191 100644 --- a/src/hyperloom/inference_optimizer/breakdown/schema.py +++ b/src/hyperloom/inference_optimizer/breakdown/schema.py @@ -239,6 +239,9 @@ class BaselineAttemptSummary(TypedDict, total=False): key_metric (float | None): Headline metric value, or None if absent. workspace (str | None): Benchmark workspace path, or None. error_class (str | None): Error classification on failure, or None. + extras (dict[str, Any]): Attempt-specific fields from the writeback + audit: ``fingerprint``, ``anchor_kept_tput``, and ``eval_probe`` + (why an accuracy of ~0 was a runaway generation, not wrong answers). """ ts: str @@ -252,6 +255,7 @@ class BaselineAttemptSummary(TypedDict, total=False): error_excerpt: str | None stderr_tail: str | None stderr_log_path: str | None + extras: dict[str, Any] class BenchmarkInvocation(TypedDict, total=False): diff --git a/src/hyperloom/orchestrator/loop/writeback.py b/src/hyperloom/orchestrator/loop/writeback.py index 4e0948c2fc..69907ea268 100644 --- a/src/hyperloom/orchestrator/loop/writeback.py +++ b/src/hyperloom/orchestrator/loop/writeback.py @@ -2502,6 +2502,9 @@ async def _promote_baseline( } if not anchor_accepted and isinstance(tput, (int, float)) and tput > 0: audit_extras["anchor_kept_tput"] = prior_anchor + # Present only when the probe cut a runaway eval short; explains a ~0 accuracy. + if result.get("eval_probe"): + audit_extras["eval_probe"] = result["eval_probe"] # seed the gaps[] ledger from baseline (best-effort). await self._refresh_gaps(reason="baseline_done") if self.shared_state.baseline_tput > 0: From 753d14e96041eba5a2cd5e3789fb7fef1af73748 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Mon, 3 Aug 2026 09:02:02 +0000 Subject: [PATCH 15/22] feat(eval): cut short an accuracy eval whose model never emits EOS InferenceX runs lm-eval with max_tokens=min(16384, ctx-4096), so a model that never emits EOS burns that budget on all 1319 GSM8K docs -- ~21.6M decode tokens against ~0.26M for a healthy model. Nothing bounded it: the soft deadline is retired at eval start by design, the stall watchdog needs total log silence, and --max-hours is only checked between coordinator ticks. The 7800s baseline timeout was the only backstop, and blowing it also discards the throughput benchmark that had already finished. Inject a probe into the sitecustomize.py InferenceX already writes in _patch_lm_eval. It watches finish_reason and, once the sample is decisive, answers the remaining generate requests with an empty string so lm-eval still writes a results*.json scoring ~0 -- the same verdict a full run would reach, minutes instead of hours. Downstream is unchanged: baseline stops on the existing baseline_accuracy_failed, a variant REVERTs through accuracy_keep_block. No new stop_reason. The short-circuit hooks amodel_call rather than _create_payload because get_batched_requests builds every payload before awaiting the inner semaphore, so lowering max_tokens after the fact is a no-op. An equally sized outer gate parks the tasks instead; it is loop-keyed because asyncio.run() builds a fresh loop per batch. Tunable via HYPERLOOM_EVAL_PROBE{,_MIN_SAMPLES,_LENGTH_RATIO}. --- .../tests/test_eval_probe.py | 324 ++++++++++++++++++ .../tests/test_inferencex_patcher.py | 160 +++++++++ .../actions/executors/_grid_runner.py | 16 +- .../actions/executors/_inferencex_patcher.py | 202 ++++++++++- .../actions/executors/baseline.py | 16 +- .../actions/executors/integrate_patch.py | 15 +- 6 files changed, 719 insertions(+), 14 deletions(-) create mode 100644 src/hyperloom/inference_optimizer/tests/test_eval_probe.py diff --git a/src/hyperloom/inference_optimizer/tests/test_eval_probe.py b/src/hyperloom/inference_optimizer/tests/test_eval_probe.py new file mode 100644 index 0000000000..e9bd54da21 --- /dev/null +++ b/src/hyperloom/inference_optimizer/tests/test_eval_probe.py @@ -0,0 +1,324 @@ +# SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: MIT + +"""Tests for the lm-eval generation-pathology probe. + +The probe body ships as a source string in ``_inferencex_patcher`` (it is +injected into the lm-eval subprocess's ``sitecustomize.py``), so no linter or +import ever type-checks it. These tests exec it against stub lm-eval modules — +hermetic, so they pin the contract whether or not lm-eval is installed. + +What the probe must guarantee: + +* a model whose answers terminate is never short-circuited; +* once a decisive share of responses hit the ``max_tokens`` cap, the remaining + generate requests are answered with an empty string so lm-eval still writes a + ``results*.json`` scoring ~0 instead of running for hours; +* loglikelihood requests are never short-circuited (they have no EOS to emit); +* a bug in any of the above degrades to "eval runs as before", never a crash. +""" + +from __future__ import annotations + +import asyncio +import json +import sys +import types +from pathlib import Path +from typing import Any + +import pytest + +from hyperloom.orchestrator.actions.executors._accuracy_gate import ( + EVAL_KIND_GENERATION_PATHOLOGY, + EVAL_PROBE_FILENAME, + eval_probe_summary, + read_eval_probe, +) +from hyperloom.orchestrator.actions.executors._inferencex_patcher import _EVAL_PROBE_PY + + +class _StubTemplateAPI: + """Stands in for ``lm_eval.models.api_models.TemplateAPI``.""" + + _concurrent = 4 + + def __init__(self) -> None: + self.inner_calls = 0 + self.cached: list[tuple[str, Any, str]] = [] + self.cache_hook = types.SimpleNamespace( + add_partial=lambda method, key, res: self.cached.append((method, key, res)) + ) + + async def amodel_call(self, session, sem, messages, **kwargs): + self.inner_calls += 1 + return ["real answer"] * len(messages) + + +class _StubLocalChatCompletion: + """Stands in for ``lm_eval.models.openai_completions.LocalChatCompletion``.""" + + @staticmethod + def parse_generations(outputs, **kwargs): + return ["upstream"] + + +def _install_stub_lm_eval(monkeypatch: pytest.MonkeyPatch) -> tuple[Any, Any]: + """Put stub ``lm_eval`` modules on ``sys.modules`` for the probe to patch. + + Returns: + The stub ``api_models`` and ``openai_completions`` modules. + """ + pkg = types.ModuleType("lm_eval") + models = types.ModuleType("lm_eval.models") + api_models = types.ModuleType("lm_eval.models.api_models") + openai_completions = types.ModuleType("lm_eval.models.openai_completions") + api_models.TemplateAPI = _StubTemplateAPI + openai_completions.LocalChatCompletion = _StubLocalChatCompletion + pkg.models = models + models.api_models = api_models + models.openai_completions = openai_completions + for name, mod in ( + ("lm_eval", pkg), + ("lm_eval.models", models), + ("lm_eval.models.api_models", api_models), + ("lm_eval.models.openai_completions", openai_completions), + ): + monkeypatch.setitem(sys.modules, name, mod) + return api_models, openai_completions + + +@pytest.fixture +def probe(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): + """Install the probe over stub lm-eval modules with a low trip threshold.""" + monkeypatch.setenv("RESULT_DIR", str(tmp_path)) + monkeypatch.setenv("HYPERLOOM_EVAL_PROBE_MIN_SAMPLES", "4") + monkeypatch.setenv("HYPERLOOM_EVAL_PROBE_LENGTH_RATIO", "0.75") + monkeypatch.delenv("HYPERLOOM_EVAL_PROBE", raising=False) + api_models, openai_completions = _install_stub_lm_eval(monkeypatch) + # Restore the pristine staticmethod so tests never leak patches into + # each other via the shared stub classes. + monkeypatch.setattr( + _StubLocalChatCompletion, + "parse_generations", + staticmethod(_StubLocalChatCompletion.__dict__["parse_generations"].__func__), + ) + monkeypatch.setattr(_StubTemplateAPI, "amodel_call", _StubTemplateAPI.amodel_call) + exec(compile(_EVAL_PROBE_PY, "", "exec"), {"__name__": "sitecustomize"}) + return types.SimpleNamespace( + api_models=api_models, + openai_completions=openai_completions, + result_dir=tmp_path, + ) + + +def _response(finish_reason: str, completion_tokens: int = 16384) -> dict[str, Any]: + """One OpenAI-shaped chat-completion response.""" + return { + "choices": [{"index": 0, "finish_reason": finish_reason, "message": {"content": "x"}}], + "usage": {"completion_tokens": completion_tokens}, + } + + +def _feed(probe, finish_reason: str, count: int) -> None: + """Push ``count`` responses through the observation hook.""" + for _ in range(count): + probe.openai_completions.LocalChatCompletion.parse_generations(outputs=_response(finish_reason)) + + +async def _call(probe, obj: _StubTemplateAPI, *, generate: bool = True, cache_keys=None): + return await probe.api_models.TemplateAPI.amodel_call( + obj, None, None, ["msg"], generate=generate, cache_keys=cache_keys + ) + + +def test_probe_installs_over_upstream_patches(probe): + """The probe wraps rather than replaces, so InferenceX's own + parse_generations fix (appended just above it) stays in effect.""" + out = probe.openai_completions.LocalChatCompletion.parse_generations(outputs=_response("stop")) + assert out == ["upstream"] + + +def test_below_min_samples_does_not_short_circuit(probe): + """Three capped responses is not yet evidence; the eval must run on.""" + obj = _StubTemplateAPI() + _feed(probe, "length", 3) + assert asyncio.run(_call(probe, obj)) == ["real answer"] + assert obj.inner_calls == 1 + + +def test_trips_and_short_circuits_once_decisive(probe): + obj = _StubTemplateAPI() + _feed(probe, "length", 4) + assert asyncio.run(_call(probe, obj)) == [""] + assert obj.inner_calls == 0, "a tripped probe must not reach the server at all" + + +def test_terminating_model_is_never_short_circuited(probe): + """The whole point: a model that emits EOS must be graded normally even + though some answers legitimately hit the cap.""" + obj = _StubTemplateAPI() + _feed(probe, "stop", 8) + _feed(probe, "length", 1) + assert asyncio.run(_call(probe, obj)) == ["real answer"] + assert not (probe.result_dir / EVAL_PROBE_FILENAME).exists() + + +def test_loglikelihood_requests_are_never_short_circuited(probe): + """Loglikelihood scoring emits no tokens, so the pathology cannot apply.""" + obj = _StubTemplateAPI() + _feed(probe, "length", 4) + assert asyncio.run(_call(probe, obj, generate=False)) == ["real answer"] + assert obj.inner_calls == 1 + + +def test_short_circuit_still_populates_the_harness_cache(probe): + """lm-eval reconciles answers against cache_keys; skipping the hook would + desync the run it is supposed to let finish cleanly.""" + obj = _StubTemplateAPI() + _feed(probe, "length", 4) + asyncio.run(_call(probe, obj, cache_keys=[("ctx", "kwargs")])) + assert obj.cached == [("generate_until", ("ctx", "kwargs"), "")] + + +def test_gate_survives_a_fresh_event_loop(probe): + """lm-eval calls asyncio.run() once per batch. A semaphore binds to the + first loop that awaits it, so a non-loop-keyed gate would raise here.""" + obj = _StubTemplateAPI() + asyncio.run(_call(probe, obj)) + _feed(probe, "length", 4) + assert asyncio.run(_call(probe, obj)) == [""] + + +def test_sidecar_records_the_evidence(probe): + _feed(probe, "length", 4) + sidecar = probe.result_dir / EVAL_PROBE_FILENAME + record = json.loads(sidecar.read_text(encoding="utf-8")) + assert record["reason"] == "model_not_terminating" + assert record["observed_samples"] == 4 + assert record["finish_reason_length"] == 4 + assert record["length_ratio"] == 1.0 + assert record["max_completion_tokens_seen"] == 16384 + # parse_eval_results globs results*.json for the score; a probe sidecar + # matching that name would be read as an lm-eval result file. + assert not sidecar.name.startswith("results") + + +def test_sidecar_is_written_once(probe): + """Every subsequent response would otherwise rewrite it with a diluted + ratio, since short-circuited requests never report a finish_reason.""" + _feed(probe, "length", 4) + first = (probe.result_dir / EVAL_PROBE_FILENAME).read_text(encoding="utf-8") + _feed(probe, "length", 20) + assert (probe.result_dir / EVAL_PROBE_FILENAME).read_text(encoding="utf-8") == first + + +def test_probe_can_be_disabled(monkeypatch, tmp_path): + monkeypatch.setenv("RESULT_DIR", str(tmp_path)) + monkeypatch.setenv("HYPERLOOM_EVAL_PROBE", "0") + api_models, openai_completions = _install_stub_lm_eval(monkeypatch) + pristine = api_models.TemplateAPI.amodel_call + + exec(compile(_EVAL_PROBE_PY, "", "exec"), {"__name__": "sitecustomize"}) + + assert api_models.TemplateAPI.amodel_call is pristine + assert openai_completions.LocalChatCompletion.parse_generations(outputs={}) == ["upstream"] + + +def test_probe_never_raises_when_lm_eval_is_absent(monkeypatch): + """sitecustomize runs at interpreter startup; raising there would break + every python3 the benchmark shells out to, not just lm-eval.""" + for name in ("lm_eval", "lm_eval.models", "lm_eval.models.api_models"): + monkeypatch.setitem(sys.modules, name, None) + exec(compile(_EVAL_PROBE_PY, "", "exec"), {"__name__": "sitecustomize"}) + + +def test_probe_survives_malformed_responses(probe): + """A server that answers with something unexpected must not take the eval + down with it.""" + obj = _StubTemplateAPI() + for junk in (None, [], {"choices": "not-a-list"}, {"choices": [None]}, {"usage": "nope"}): + probe.openai_completions.LocalChatCompletion.parse_generations(outputs=junk) + assert asyncio.run(_call(probe, obj)) == ["real answer"] + + +def test_read_eval_probe_finds_a_nested_sidecar(probe, tmp_path): + """The baseline double-run evaluates in the warmup round, whose RESULT_DIR + nests under the task workspace.""" + nested = tmp_path / "warmup_round" + nested.mkdir() + (nested / EVAL_PROBE_FILENAME).write_text(json.dumps({"reason": "model_not_terminating"}), encoding="utf-8") + + record = read_eval_probe(tmp_path) + + assert record is not None + assert record["kind"] == EVAL_KIND_GENERATION_PATHOLOGY + assert record["source_file"].endswith(EVAL_PROBE_FILENAME) + + +def test_read_eval_probe_is_none_without_a_sidecar(tmp_path): + """No sidecar is the ordinary case: the model terminated its answers.""" + assert read_eval_probe(tmp_path) is None + + +def test_read_eval_probe_tolerates_corrupt_json(tmp_path): + (tmp_path / EVAL_PROBE_FILENAME).write_text("{not json", encoding="utf-8") + assert read_eval_probe(tmp_path) is None + + +def test_eval_probe_summary_is_empty_without_a_probe(): + assert eval_probe_summary(None) == "" + + +def test_eval_probe_summary_names_the_kind_and_the_evidence(): + summary = eval_probe_summary( + {"observed_samples": 16, "finish_reason_length": 16, "max_completion_tokens_seen": 16384} + ) + assert EVAL_KIND_GENERATION_PATHOLOGY in summary + assert "16/16" in summary + assert "16384" in summary + + +def test_probe_record_reaches_session_breakdown(tmp_path): + """End of the traceability chain: the writeback audit stores the record in + the attempt's ``extras``, and the collector must carry it into + ``session_breakdown.json``. Without this, a baseline accuracy of 0 gives a + reader no way to tell a broken generation loop from wrong answers.""" + from hyperloom.inference_optimizer.breakdown.collectors.sessions import collect_baseline + + probe_record = { + "kind": EVAL_KIND_GENERATION_PATHOLOGY, + "reason": "model_not_terminating", + "observed_samples": 16, + "finish_reason_length": 16, + } + state = { + "baseline_tput": 1234.0, + "baseline_accuracy": 0.0, + "baseline_attempts": [ + { + "ts": "2026-08-03T00:00:00+00:00", + "task_id": "t1", + "status": "succeeded", + "decision": "promoted", + "key_metric": 1234.0, + "error_class": None, + "extras": {"eval_probe": probe_record}, + } + ], + } + + section = collect_baseline(tmp_path, state, []) + + assert section["attempts_history"][0]["extras"]["eval_probe"] == probe_record + + +def test_breakdown_attempt_extras_default_to_empty(tmp_path): + """Attempts recorded before this field existed must still render.""" + from hyperloom.inference_optimizer.breakdown.collectors.sessions import collect_baseline + + state = {"baseline_attempts": [{"ts": "2026-08-03T00:00:00+00:00", "task_id": "t1", "status": "failed"}]} + + section = collect_baseline(tmp_path, state, []) + + assert section["attempts_history"][0]["extras"] == {} diff --git a/src/hyperloom/inference_optimizer/tests/test_inferencex_patcher.py b/src/hyperloom/inference_optimizer/tests/test_inferencex_patcher.py index bf0e892dbb..266f380445 100644 --- a/src/hyperloom/inference_optimizer/tests/test_inferencex_patcher.py +++ b/src/hyperloom/inference_optimizer/tests/test_inferencex_patcher.py @@ -653,3 +653,163 @@ def test_baseline_after_materialize_applies_eval_start_patch(tmp_path, monkeypat assert out is None assert 'echo "HYPERLOOM_EVAL_START" >&2' in lib.read_text(encoding="utf-8") + + +# Verbatim upstream ``_patch_lm_eval`` shape: the probe is appended to the same +# sitecustomize InferenceX already writes, anchored on the PYTHONPATH export. +_EVAL_PROBE_FIXTURE = """#!/usr/bin/env bash +_patch_lm_eval() { + local patch_dir + patch_dir="$(mktemp -d)" + cat > "$patch_dir/sitecustomize.py" <<'PY' +from lm_eval.models.openai_completions import LocalChatCompletion as _LCC +_LCC.parse_generations = staticmethod(lambda outputs, **kw: [""]) +PY + export PYTHONPATH="${patch_dir}:${PYTHONPATH:-}" +} +""" + + +def _write_eval_probe_lib(root: Path) -> Path: + bench_dir = root / "benchmarks" + bench_dir.mkdir(parents=True) + lib = bench_dir / "benchmark_lib.sh" + lib.write_text(_EVAL_PROBE_FIXTURE, encoding="utf-8") + return lib + + +def test_eval_probe_patch_appends_after_upstream_sitecustomize(tmp_path, monkeypatch): + """The probe must be appended to the same sitecustomize AFTER InferenceX's + own monkeypatches, and still before the PYTHONPATH export that publishes + it — otherwise it would wrap an unpatched parse_generations, or not load.""" + from hyperloom.orchestrator.actions.executors._inferencex_patcher import ( + ensure_benchmark_lib_eval_probe_patched, + ) + + lib = _write_eval_probe_lib(tmp_path) + monkeypatch.setenv("INFERENCEX_PATH", str(tmp_path)) + monkeypatch.delenv("MAGPIE_PATH", raising=False) + + rc = ensure_benchmark_lib_eval_probe_patched(tmp_path) + assert rc is True + text = lib.read_text(encoding="utf-8") + assert "HYPERLOOM_EVAL_PROBE" in text + upstream_at = text.index("_LCC.parse_generations = staticmethod") + probe_at = text.index("_hl_eval_probe_install") + export_at = text.index('export PYTHONPATH="${patch_dir}') + assert upstream_at < probe_at < export_at + # Appends (>>) so the upstream heredoc body survives. + assert 'cat >> "$patch_dir/sitecustomize.py"' in text + assert 'cat > "$patch_dir/sitecustomize.py"' in text + + +def test_eval_probe_patch_heredoc_terminator_is_unindented(tmp_path, monkeypatch): + """A leading space on the terminator makes bash swallow the rest of the + file, so the eval would die at parse time rather than run unprobed.""" + from hyperloom.orchestrator.actions.executors._inferencex_patcher import ( + ensure_benchmark_lib_eval_probe_patched, + ) + + lib = _write_eval_probe_lib(tmp_path) + monkeypatch.setenv("INFERENCEX_PATH", str(tmp_path)) + ensure_benchmark_lib_eval_probe_patched(tmp_path) + + lines = lib.read_text(encoding="utf-8").splitlines() + assert lines.count("HYPERLOOM_PY") == 1, "heredoc terminator must appear once, at column 0" + + +def test_eval_probe_patch_emits_valid_python(tmp_path, monkeypatch): + """The injected body is a string constant, so no linter or import ever + sees it — compiling it here is the only thing standing between a typo and + a sitecustomize that raises on every lm-eval start.""" + from hyperloom.orchestrator.actions.executors._inferencex_patcher import ( + _EVAL_PROBE_PY, + ensure_benchmark_lib_eval_probe_patched, + ) + + lib = _write_eval_probe_lib(tmp_path) + monkeypatch.setenv("INFERENCEX_PATH", str(tmp_path)) + ensure_benchmark_lib_eval_probe_patched(tmp_path) + + compile(_EVAL_PROBE_PY, "", "exec") + body = lib.read_text(encoding="utf-8").split("<<'HYPERLOOM_PY'\n", 1)[1].split("\nHYPERLOOM_PY\n", 1)[0] + compile(body + "\n", "", "exec") + + +def test_eval_probe_patch_is_idempotent(tmp_path, monkeypatch): + from hyperloom.orchestrator.actions.executors._inferencex_patcher import ( + ensure_benchmark_lib_eval_probe_patched, + ) + + lib = _write_eval_probe_lib(tmp_path) + monkeypatch.setenv("INFERENCEX_PATH", str(tmp_path)) + monkeypatch.delenv("MAGPIE_PATH", raising=False) + + ensure_benchmark_lib_eval_probe_patched(tmp_path) + after_first = lib.read_text(encoding="utf-8") + ensure_benchmark_lib_eval_probe_patched(tmp_path) + assert lib.read_text(encoding="utf-8") == after_first + assert after_first.count("_hl_eval_probe_install()") == 2 # one def, one call + + +def test_eval_probe_patch_fails_soft_when_anchor_missing(tmp_path, monkeypatch): + """An upstream that no longer exports PYTHONPATH here must leave the eval + running unprobed, not break it.""" + from hyperloom.orchestrator.actions.executors._inferencex_patcher import ( + ensure_benchmark_lib_eval_probe_patched, + ) + + bench_dir = tmp_path / "benchmarks" + bench_dir.mkdir(parents=True) + lib = bench_dir / "benchmark_lib.sh" + lib.write_text("#!/usr/bin/env bash\nrun_lm_eval() { :; }\n", encoding="utf-8") + monkeypatch.setenv("INFERENCEX_PATH", str(tmp_path)) + + assert ensure_benchmark_lib_eval_probe_patched(tmp_path) is False + assert "HYPERLOOM_EVAL_PROBE" not in lib.read_text(encoding="utf-8") + + +def test_eval_probe_patch_is_concurrency_safe(tmp_path, monkeypatch): + """Several executors can patch one shared checkout at once.""" + from hyperloom.orchestrator.actions.executors._inferencex_patcher import ( + ensure_benchmark_lib_eval_probe_patched, + ) + + lib = _write_eval_probe_lib(tmp_path) + monkeypatch.setenv("INFERENCEX_PATH", str(tmp_path)) + monkeypatch.delenv("MAGPIE_PATH", raising=False) + + results: list[bool] = [] + threads = [ + threading.Thread(target=lambda: results.append(ensure_benchmark_lib_eval_probe_patched(tmp_path))) + for _ in range(8) + ] + for t in threads: + t.start() + for t in threads: + t.join() + + assert all(results) + assert lib.read_text(encoding="utf-8").splitlines().count("HYPERLOOM_PY") == 1 + + +def test_baseline_after_materialize_applies_eval_probe_patch(tmp_path, monkeypatch): + """Explore/sweep re-assert this in _grid_runner, but the baseline hook is + the one that matters: a non-terminating model there stops the whole run.""" + import yaml + + from hyperloom.orchestrator.actions.executors.baseline import BaselineExecutor + + ix_root = tmp_path / "InferenceX@deadbeef" + lib = _write_eval_probe_lib(ix_root) + config_path = tmp_path / "baseline.yaml" + config_path.write_text( + yaml.safe_dump({"benchmark": {"inferencex_path": str(ix_root)}}), + encoding="utf-8", + ) + monkeypatch.delenv("INFERENCEX_PATH", raising=False) + + out = BaselineExecutor()._after_materialize_config(config_path, tmp_path / "out") + + assert out is None + assert "HYPERLOOM_EVAL_PROBE" in lib.read_text(encoding="utf-8") diff --git a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py index e79cfd2fe6..66749619d2 100644 --- a/src/hyperloom/orchestrator/actions/executors/_grid_runner.py +++ b/src/hyperloom/orchestrator/actions/executors/_grid_runner.py @@ -45,7 +45,10 @@ harvest_leaked_artifacts, ) from .benchmark_backend import build_benchmark_command -from ._inferencex_patcher import ensure_benchmark_lib_eval_start_patched +from ._inferencex_patcher import ( + ensure_benchmark_lib_eval_probe_patched, + ensure_benchmark_lib_eval_start_patched, +) # Re-exported from sibling modules to keep the module namespace intact. from ._grid_base import ( @@ -926,7 +929,8 @@ def _run_magpie( env["MAGPIE_INFERENCEX_PATH"] = inferencex_path # Baseline patches its own checkout, but explore / sweep never pass # through that hook: re-assert here so a resumed session or a re-cloned - # checkout still emits the eval-start marker. Idempotent. + # checkout still emits the eval-start marker and installs the + # generation-pathology probe. Both idempotent. try: ensure_benchmark_lib_eval_start_patched(Path(inferencex_path)) except Exception as exc: # noqa: BLE001 — patch is best-effort @@ -935,6 +939,14 @@ def _run_magpie( inferencex_path, exc, ) + try: + ensure_benchmark_lib_eval_probe_patched(Path(inferencex_path)) + except Exception as exc: # noqa: BLE001 — patch is best-effort + log.warning( + "_grid_runner: eval-probe patch skipped for %s: %s", + inferencex_path, + exc, + ) # AgentX: deploy the aiperf client into InferenceX ``benchmarks/`` + preflight # aiperf right before Magpie runs it, via the shared helper (also used by the # baseline/profile shell-out). No-op under pytest / when AgentX is off (the diff --git a/src/hyperloom/orchestrator/actions/executors/_inferencex_patcher.py b/src/hyperloom/orchestrator/actions/executors/_inferencex_patcher.py index 446150fbe4..4ffde5ba85 100644 --- a/src/hyperloom/orchestrator/actions/executors/_inferencex_patcher.py +++ b/src/hyperloom/orchestrator/actions/executors/_inferencex_patcher.py @@ -1,14 +1,12 @@ # SPDX-FileCopyrightText: 2026 Advanced Micro Devices, Inc. # SPDX-License-Identifier: MIT -"""Idempotent, backward-compatible patcher for InferenceX -``benchmarks/benchmark_lib.sh``. +"""Idempotent, backward-compatible patchers for the InferenceX checkout. -Upstream resets ``num_prompts="$max_concurrency"`` under ``PROFILE=1``, -stomping ``--num-prompts`` so the engine finishes before the steady-state -profiling window opens. The patch makes that line honour -``${NUM_PROMPTS:-$max_concurrency}`` — bit-for-bit identical when the env is -unset. +Each ``ensure_*`` function rewrites one upstream line in place: ``$NUM_PROMPTS`` +support and ``PROFILE_EXTRA_BODY`` consumption for profiling, the eval-artifact +redirect to ``$RESULT_DIR``, the ``HYPERLOOM_EVAL_START`` phase marker, and the +generation-pathology probe injected into lm-eval's ``sitecustomize.py``. Applied in place, once: idempotent via a sentinel substring, serialized across processes via ``fcntl.flock``, written atomically. Returns ``False`` @@ -77,6 +75,131 @@ _EVAL_START_SENTINEL = "HYPERLOOM_EVAL_START" _EVAL_START_LOCK_PATH = str(Path(tempfile.gettempdir()) / "hyperloom_benchmark_lib_eval_start_patcher.lock") +# Early-exit probe for a model that never emits EOS. InferenceX runs lm-eval +# with ``--gen_kwargs max_tokens=min(16384, ctx-4096)``, so a degenerate model +# burns that budget on every one of GSM8K's 1319 docs and takes the whole +# baseline timeout with it. Injected as source rather than imported: the +# lm-eval subprocess shares an interpreter with Hyperloom only by accident. +_EVAL_PROBE_PY = """ +# --- HYPERLOOM_EVAL_PROBE --------------------------------------------------- +import json as _hl_json +import os as _hl_os +import sys as _hl_sys + + +def _hl_eval_probe_install(): + if (_hl_os.environ.get("HYPERLOOM_EVAL_PROBE") or "1").strip().lower() in ("0", "false", "no", "off"): + return + + def _num(name, default, cast): + try: + return cast((_hl_os.environ.get(name) or "").strip()) + except (TypeError, ValueError): + return default + + min_samples = max(1, _num("HYPERLOOM_EVAL_PROBE_MIN_SAMPLES", 16, int)) + ratio_limit = _num("HYPERLOOM_EVAL_PROBE_LENGTH_RATIO", 0.75, float) + + import asyncio as _hl_asyncio + from lm_eval.models import api_models as _hl_api + from lm_eval.models.openai_completions import LocalChatCompletion as _hl_lcc + + state = {"observed": 0, "length": 0, "max_tokens_seen": 0, "tripped": False} + + def _emit(): + record = { + "reason": "model_not_terminating", + "observed_samples": state["observed"], + "finish_reason_length": state["length"], + "length_ratio": round(float(state["length"]) / state["observed"], 4), + "max_completion_tokens_seen": state["max_tokens_seen"], + "min_samples": min_samples, + "length_ratio_threshold": ratio_limit, + } + blob = _hl_json.dumps(record, sort_keys=True) + print("HYPERLOOM_EVAL_PROBE_TRIPPED " + blob, file=_hl_sys.stderr, flush=True) + # $RESULT_DIR, never $EVAL_RESULT_DIR: append_lm_eval_summary rm -rf's + # the latter. The name must not match results*.json -- that glob is how + # parse_eval_results finds the accuracy score. + out_dir = (_hl_os.environ.get("RESULT_DIR") or "").strip() or "." + _hl_os.makedirs(out_dir, exist_ok=True) + with open(_hl_os.path.join(out_dir, "hyperloom_eval_probe.json"), "w", encoding="utf-8") as fh: + fh.write(blob) + + def _observe(outputs): + for out in outputs if isinstance(outputs, list) else [outputs]: + seen = int((out.get("usage") or {}).get("completion_tokens") or 0) + state["max_tokens_seen"] = max(state["max_tokens_seen"], seen) + for choice in out.get("choices") or []: + state["observed"] += 1 + if choice.get("finish_reason") == "length": + state["length"] += 1 + if state["observed"] >= min_samples and float(state["length"]) / state["observed"] >= ratio_limit: + state["tripped"] = True + _emit() + + # Wrap whatever is installed now so InferenceX's own parse_generations + # patch (appended just above) stays in effect. Observation must never break + # the eval it is watching, hence the guard. + _hl_prev_parse = _hl_lcc.parse_generations + + def _hl_probe_parse_generations(outputs, **kwargs): + if not state["tripped"]: + try: + _observe(outputs) + except Exception: + pass + return _hl_prev_parse(outputs, **kwargs) + + _hl_lcc.parse_generations = staticmethod(_hl_probe_parse_generations) + + # get_batched_requests creates one task per request up front, and + # amodel_call builds its payload BEFORE awaiting the inner semaphore, so + # every payload already carries the large max_tokens by the time the probe + # trips. Park the tasks in an equally sized outer gate instead. asyncio.run + # builds a fresh loop per batch and a Semaphore binds to the first loop + # that awaits it, so the gate is loop-keyed. + _hl_prev_amodel_call = _hl_api.TemplateAPI.amodel_call + gate = {"loop": None, "sem": None} + + async def _hl_probe_amodel_call(self, session, sem, messages, **kwargs): + loop = _hl_asyncio.get_running_loop() + if gate["loop"] is not loop: + gate["loop"] = loop + gate["sem"] = _hl_asyncio.Semaphore(max(1, int(self._concurrent or 1))) + async with gate["sem"]: + if not (state["tripped"] and kwargs.get("generate", True)): + return await _hl_prev_amodel_call(self, session, sem, messages, **kwargs) + answers = [""] * len(messages) + for answer, cache_key in zip(answers, kwargs.get("cache_keys") or []): + self.cache_hook.add_partial("generate_until", cache_key, answer) + return answers + + _hl_api.TemplateAPI.amodel_call = _hl_probe_amodel_call + + +# sitecustomize runs at interpreter startup: raising here would break every +# python3 the benchmark shells out to, not just lm-eval. +try: + _hl_eval_probe_install() +except Exception: + pass +# --- end HYPERLOOM_EVAL_PROBE ----------------------------------------------- +""" + +# Anchored on the unique ``export PYTHONPATH`` line that closes +# ``_patch_lm_eval``. The heredoc is quoted so nothing inside it is +# shell-expanded, and its terminator must sit at column 0. +_EVAL_PROBE_LEGACY = ' export PYTHONPATH="${patch_dir}:${PYTHONPATH:-}"' +_EVAL_PROBE_PATCHED = ( + " cat >> \"$patch_dir/sitecustomize.py\" <<'HYPERLOOM_PY'\n" + + _EVAL_PROBE_PY.lstrip("\n") + + "HYPERLOOM_PY\n" + + _EVAL_PROBE_LEGACY +) +_EVAL_PROBE_SENTINEL = "HYPERLOOM_EVAL_PROBE" +_EVAL_PROBE_LOCK_PATH = str(Path(tempfile.gettempdir()) / "hyperloom_benchmark_lib_eval_probe_patcher.lock") + def _discover_inferencex_roots( inferencex_path: Path | str | None, @@ -543,9 +666,74 @@ def ensure_benchmark_lib_eval_start_patched( ) +def _is_eval_probe_patched(src: Path) -> bool: + """Return whether ``benchmark_lib.sh`` already injects the eval probe. + + Args: + src (Path): The ``benchmark_lib.sh`` file to inspect. + + Returns: + bool: ``True`` if the eval-probe sentinel is present; ``False`` on a + miss or read error. + """ + return file_contains_sentinel(src, _EVAL_PROBE_SENTINEL, log, "_inferencex_patcher") + + +def ensure_benchmark_lib_eval_probe_patched( + inferencex_path: Path | str | None = None, +) -> bool: + """Ensure ``_patch_lm_eval`` also installs Hyperloom's early-exit probe. + + The probe watches ``finish_reason`` on completed responses; once the sample + is decisive it answers the remaining requests with an empty string, so + lm-eval still finishes normally and writes a ``results*.json`` scoring ~0 -- + the verdict a non-terminating model would have earned hours later anyway. + Returns ``True`` when patched at exit, ``False`` (non-fatal) when the file + is missing or the anchor line is absent — the eval then runs unbounded as + before. Concurrency-safe; independent lock so it does not serialize with + the other patches on the same file. + + Args: + inferencex_path: Caller-provided override root; defaults to env-based + discovery when ``None``. + + Returns: + True when at least one discovered ``benchmark_lib.sh`` is patched (or + already patched), False when none could be patched. + """ + return _ensure_patched( + _resolve_benchmark_lib_paths(inferencex_path), + _is_eval_probe_patched, + partial( + _apply_line_replacement_atomic, + legacy=_EVAL_PROBE_LEGACY, + patched_line=_EVAL_PROBE_PATCHED, + tmp_prefix=".benchmark_lib.sh.eval_probe_", + missing_msg=( + "_inferencex_patcher: expected _patch_lm_eval PYTHONPATH export " + "not found in %s; upstream layout may have changed. A model that " + "never emits EOS will run the accuracy eval to the full " + "max_tokens budget on every sample instead of exiting early." + ), + success_msg=("_inferencex_patcher: installed eval generation-pathology probe in %s"), + ), + _EVAL_PROBE_LOCK_PATH, + empty_msg=( + "_inferencex_patcher: no InferenceX root discovered " + "(checked $INFERENCEX_PATH, $MAGPIE_PATH/InferenceX) or " + "benchmark_lib.sh missing — skipping eval-probe patch (fine for " + "tests and dry-runs without a real InferenceX tree)" + ), + failure_msg=( + "_inferencex_patcher: failed to eval-probe-patch %s; other discovered roots will still be attempted" + ), + ) + + __all__ = [ "ensure_benchmark_lib_patched", "ensure_benchmark_lib_eval_dest_patched", + "ensure_benchmark_lib_eval_probe_patched", "ensure_benchmark_lib_eval_start_patched", "ensure_benchmark_serving_patched", ] diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index d2a4e1c6db..fa6a175458 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -62,6 +62,7 @@ ) from ._inferencex_patcher import ( ensure_benchmark_lib_eval_dest_patched, + ensure_benchmark_lib_eval_probe_patched, ensure_benchmark_lib_eval_start_patched, ) from ._magpie_patcher import ensure_eval_concurrency_compat @@ -1155,6 +1156,14 @@ def _after_materialize_config( ix_root, exc, ) + try: + ensure_benchmark_lib_eval_probe_patched(Path(ix_root)) + except Exception as exc: # noqa: BLE001 — patch is best-effort + log.warning( + "baseline_executor: eval-probe patch skipped for %s: %s", + ix_root, + exc, + ) # Fail LOUDLY (never warn-and-continue) when the fatal eval flag cannot # be removed AND this run is meant to execute lm-eval: the benchmark is # guaranteed to abort in run_lm_eval, and the accuracy gate then stops @@ -2916,7 +2925,7 @@ async def _run_single_benchmark( "baseline_executor: RUN_EVAL disabled this run (serving); skipping accuracy parse (no lm-eval executed)" ) else: - from ._accuracy_gate import parse_eval_results + from ._accuracy_gate import eval_probe_summary, parse_eval_results, read_eval_probe # Search from ``$RESULT_DIR`` so serving runs survive benchmark_lib.sh # moving/cleaning ``$EVAL_RESULT_DIR`` and scriptable quality gates @@ -2931,6 +2940,11 @@ async def _run_single_benchmark( log.info("baseline_executor: accuracy=%.4f (%s)", result["accuracy"], result["accuracy_task"]) else: log.warning("baseline_executor: accuracy eval not found: %s", eval_data.get("error", "unknown")) + # Records why the score is ~0; the score itself is already correct. + eval_probe = read_eval_probe(eval_search_root) + if eval_probe: + result["eval_probe"] = eval_probe + log.warning("baseline_executor: %s", eval_probe_summary(eval_probe)) log.info( "baseline_executor: %s %s (output) e2el=%.1fms", diff --git a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py index c17e59aab8..14a43c5ff8 100644 --- a/src/hyperloom/orchestrator/actions/executors/integrate_patch.py +++ b/src/hyperloom/orchestrator/actions/executors/integrate_patch.py @@ -31,7 +31,9 @@ accuracy_meets_floor, accuracy_passed, classify_accuracy_failure, + eval_probe_summary, parse_eval_results, + read_eval_probe, ) from ._apply_feedback import ApplyFeedback, build_apply_feedback from ._git import _run_git_cp @@ -2712,6 +2714,9 @@ async def _gate_perf( reasons.append(f"throughput delta {delta_pct:+.2f}% < keep_threshold {keep_threshold_pct:.2f}%") if acc_block and acc_reason: reasons.append(acc_reason) + _probe_reason = eval_probe_summary(gate_evidence.get("eval_probe")) + if _probe_reason: + reasons.append(_probe_reason) _tput_ok = delta_pct is not None and delta_pct >= keep_threshold_pct revert_status = ( "accuracy_unavailable_reject" if (acc_block and accuracy_pass is None and _tput_ok) else "reverted" @@ -3172,9 +3177,6 @@ async def _bench_patch( ) -> tuple[dict[str, Any], dict[str, Any]]: """Run a 1-variant Magpie bench under the patched server + accuracy gate. - Returns ``(bench_result_dict, gate_evidence)`` where gate_evidence - carries ``accuracy_pass`` (True / False / None). - Args: params: The task params (config / model / bench knobs). output_root: The per-task workspace root for the bench. @@ -3185,7 +3187,8 @@ async def _bench_patch( Returns: A ``(bench_result_dict, gate_evidence)`` tuple where - ``gate_evidence`` carries ``accuracy_pass`` (True / False / None). + ``gate_evidence`` carries ``accuracy_pass`` (True / False / None) + and ``eval_probe`` (the generation-pathology record, or ``None``). """ config_path = Path(params.get("config_path") or self.default_config_path or default_baseline_config()) if not config_path.exists(): @@ -3324,12 +3327,16 @@ async def _bench_patch( except Exception: # noqa: BLE001 — eval may not produce a result log.debug("integrate_patch: enablement eval parse failed", exc_info=True) + # Guarded: an empty root would send the recursive scan over the cwd. + eval_probe = read_eval_probe(eval_search_root) if eval_search_root else None + return bench, { "accuracy_pass": accuracy_pass, "accuracy": measured_accuracy, "enablement_accuracy": enablement_accuracy, "enablement_accuracy_task": enablement_accuracy_task, "enablement_accuracy_metric": enablement_accuracy_metric, + "eval_probe": eval_probe, } @staticmethod From c73efc734e5548cec5ca9a7507840a3a8f2af435 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Mon, 3 Aug 2026 09:35:04 +0000 Subject: [PATCH 16/22] fix(tests): stop seeding baseline_tput before the baseline proposal baseline_phase_singleton (254fd8bf8) denies a `baseline` propose_action once baseline_tput > 0. Two tests seeded baseline_tput up front for convenience and then proposed `baseline` as part of their normal flow, so PolicyGate silently dropped that proposal and the tests undercounted/indexed past the bus. Set baseline_tput only after the baseline proposal lands, matching what the real baseline action would do on completion. --- .../tests/test_kernel_integrate_and_report.py | 5 ++++- src/hyperloom/inference_optimizer/tests/test_resume.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_kernel_integrate_and_report.py b/src/hyperloom/inference_optimizer/tests/test_kernel_integrate_and_report.py index 64a97e70c5..92ad8a5b23 100644 --- a/src/hyperloom/inference_optimizer/tests/test_kernel_integrate_and_report.py +++ b/src/hyperloom/inference_optimizer/tests/test_kernel_integrate_and_report.py @@ -1013,7 +1013,6 @@ async def test_report_executor_writes_md_and_json(session_dir): session_id=session_dir.name, model_name="Qwen-Qwen3-8B", model_path="/path/models/Qwen-Qwen3-8B", - baseline_tput=800.0, cumulative_gain=12.5, current_best={ "action": "backends", @@ -1036,6 +1035,9 @@ async def test_report_executor_writes_md_and_json(session_dir): payload={"action_name": "baseline", "predicted_gain_pct": 0.0}, ), ) + # The real baseline action would have set this on completion; explore + # requires baseline_tput > 0 (execution_order) to be proposable next. + c.shared_state.baseline_tput = 800.0 await c._handle_intent( "orchestration", Intent( @@ -1050,6 +1052,7 @@ async def test_report_executor_writes_md_and_json(session_dir): payload={"severity": "low", "summary": "noise"}, ), ) + c.shared_state.save(session_dir) finally: await c.stop() diff --git a/src/hyperloom/inference_optimizer/tests/test_resume.py b/src/hyperloom/inference_optimizer/tests/test_resume.py index 8d191f0059..39d90cb057 100644 --- a/src/hyperloom/inference_optimizer/tests/test_resume.py +++ b/src/hyperloom/inference_optimizer/tests/test_resume.py @@ -209,7 +209,6 @@ async def test_replay_mixed_pending_and_decided(session_dir): c1 = Coordinator(session_dir, backends=backends) try: # Seed prerequisites so arbitrary proposals are accepted. - c1.shared_state.baseline_tput = 100.0 c1.shared_state.last_profile_trace = "/tmp/profile.trace.json.gz" c1.shared_state.last_trace_analyze = { "trace_input": "/tmp/profile.trace.json.gz", @@ -227,6 +226,10 @@ async def test_replay_mixed_pending_and_decided(session_dir): ) tail = await c1.bus.tail(topic="proposal", n=1) proposal_ids.append(tail[0].msg_id) + if action == "baseline": + # profile/explore require baseline_tput > 0 (execution_order); + # the real baseline action would have set this on completion. + c1.shared_state.baseline_tput = 100.0 await c1._handle_intent( "critic", From 54e9f7456e3cd2f11a692d676e1e83f684e5af8a Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Mon, 3 Aug 2026 11:18:45 +0000 Subject: [PATCH 17/22] fix(orchestrator): require salvaged sibling accuracy to meet floor before setting shared baseline Salvaged sibling accuracy is always recorded on the result as evidence, but it is now only copied to `SharedState.baseline_accuracy` when `accuracy_meets_floor` passes. This prevents a zero or negative salvaged score from becoming the "no baseline, skip the check" sentinel and silently bypassing every subsequent accuracy gate. --- .../orchestrator/actions/executors/baseline.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index 49ae37a149..95a3a00f8d 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -1739,10 +1739,16 @@ def _apply_salvaged_accuracy( salvaged: dict[str, Any], shared_state: Any, ) -> float: - """Record a salvaged sibling accuracy on the result and SharedState. + """Record a salvaged sibling accuracy as evidence, and as the gate input + only when it can serve as one. - Records the score whatever its value; deciding whether it is *usable* - belongs to the caller, which knows the applicable floor. + The ``result`` fields are evidence and carry the score whatever its + value. ``SharedState.baseline_accuracy`` is not evidence — it is the + reference every later accuracy gate compares against, and ``<= 0`` is + that gate's "no baseline, skip the check" sentinel + (:func:`accuracy_passed`). Publishing a measured zero there would turn a + real quality failure into a silent gate bypass for every subsequent + candidate, so the two are written under different conditions. Args: result: The baseline result dict, mutated in place. @@ -1753,6 +1759,8 @@ def _apply_salvaged_accuracy( Returns: float: The salvaged accuracy. """ + from ._accuracy_gate import accuracy_meets_floor + acc_val = float(salvaged["accuracy"]) result["accuracy"] = acc_val result["accuracy_task"] = salvaged.get("task", "gsm8k") @@ -1760,7 +1768,7 @@ def _apply_salvaged_accuracy( result["accuracy_source"] = salvaged.get("source_file", "") result.setdefault("nonfatal_warnings", []) result["nonfatal_warnings"].append("baseline_accuracy_salvaged_from_sibling_attempt") - if shared_state is not None: + if shared_state is not None and accuracy_meets_floor(acc_val, 0.0): try: shared_state.baseline_accuracy = acc_val except Exception: # noqa: BLE001 — salvage must never break baseline From 53cbc821b267685b28731b949df96ad770b633da Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Mon, 3 Aug 2026 11:30:57 +0000 Subject: [PATCH 18/22] fix(eval): require ceiling hits before the probe cuts an eval short The trip test was a bare finish_reason=length ratio over the first 16 responses, which cannot separate a model that never terminates from one whose long answers are truncated: lm-eval sizes max_tokens per request from the remaining context, so a terminating model legitimately produces capped responses at several lengths. The ceiling was already tracked and then never consulted. The trip now counts only responses that stopped AT the largest observed cap, over a default window of 128. The knobs had no range validation either. LENGTH_RATIO=0 -- the value an operator reaches for to turn the probe off -- made the ratio test vacuously true and ended every eval at min_samples. Out-of-range values now fall back to the default rather than to the nearest legal one, since clamping RATIO=0 would do the opposite of what it asks for. Two artifact fixes alongside. With no $RESULT_DIR the sidecar landed in the cwd, i.e. InferenceX's checkout, which is the escape the _EVAL_DEST_* patch exists to prevent; stderr already carries the record, so nothing is written there now. And the probe drops a sidecar left in its $RESULT_DIR by a previous attempt, since the eval-failure retry reuses the slot, while read_eval_probe picks the newest by mtime rather than by path -- integrate_patch searches a grid slot whose sibling variants each own one. Co-authored-by: Cursor --- .../tests/test_eval_probe.py | 132 ++++++++++++++++-- .../actions/executors/_accuracy_gate.py | 20 ++- .../actions/executors/_inferencex_patcher.py | 52 ++++++- 3 files changed, 181 insertions(+), 23 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_eval_probe.py b/src/hyperloom/inference_optimizer/tests/test_eval_probe.py index e9bd54da21..b9450886a6 100644 --- a/src/hyperloom/inference_optimizer/tests/test_eval_probe.py +++ b/src/hyperloom/inference_optimizer/tests/test_eval_probe.py @@ -22,6 +22,7 @@ import asyncio import json +import os import sys import types from pathlib import Path @@ -88,11 +89,35 @@ def _install_stub_lm_eval(monkeypatch: pytest.MonkeyPatch) -> tuple[Any, Any]: return api_models, openai_completions +def _install(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, **env: str | None): + """Install the probe with an explicit env; ``None`` unsets a variable.""" + env.setdefault("RESULT_DIR", str(tmp_path)) + monkeypatch.delenv("HYPERLOOM_EVAL_PROBE", raising=False) + for key, val in env.items(): + if val is None: + monkeypatch.delenv(key, raising=False) + else: + monkeypatch.setenv(key, val) + api_models, openai_completions = _install_stub_lm_eval(monkeypatch) + monkeypatch.setattr( + _StubLocalChatCompletion, + "parse_generations", + staticmethod(_StubLocalChatCompletion.__dict__["parse_generations"].__func__), + ) + monkeypatch.setattr(_StubTemplateAPI, "amodel_call", _StubTemplateAPI.amodel_call) + exec(compile(_EVAL_PROBE_PY, "", "exec"), {"__name__": "sitecustomize"}) + return types.SimpleNamespace( + api_models=api_models, + openai_completions=openai_completions, + result_dir=tmp_path, + ) + + @pytest.fixture def probe(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): """Install the probe over stub lm-eval modules with a low trip threshold.""" monkeypatch.setenv("RESULT_DIR", str(tmp_path)) - monkeypatch.setenv("HYPERLOOM_EVAL_PROBE_MIN_SAMPLES", "4") + monkeypatch.setenv("HYPERLOOM_EVAL_PROBE_MIN_SAMPLES", "8") monkeypatch.setenv("HYPERLOOM_EVAL_PROBE_LENGTH_RATIO", "0.75") monkeypatch.delenv("HYPERLOOM_EVAL_PROBE", raising=False) api_models, openai_completions = _install_stub_lm_eval(monkeypatch) @@ -140,16 +165,16 @@ def test_probe_installs_over_upstream_patches(probe): def test_below_min_samples_does_not_short_circuit(probe): - """Three capped responses is not yet evidence; the eval must run on.""" + """One sample short of the minimum is not yet evidence; the eval runs on.""" obj = _StubTemplateAPI() - _feed(probe, "length", 3) + _feed(probe, "length", 7) assert asyncio.run(_call(probe, obj)) == ["real answer"] assert obj.inner_calls == 1 def test_trips_and_short_circuits_once_decisive(probe): obj = _StubTemplateAPI() - _feed(probe, "length", 4) + _feed(probe, "length", 8) assert asyncio.run(_call(probe, obj)) == [""] assert obj.inner_calls == 0, "a tripped probe must not reach the server at all" @@ -167,7 +192,7 @@ def test_terminating_model_is_never_short_circuited(probe): def test_loglikelihood_requests_are_never_short_circuited(probe): """Loglikelihood scoring emits no tokens, so the pathology cannot apply.""" obj = _StubTemplateAPI() - _feed(probe, "length", 4) + _feed(probe, "length", 8) assert asyncio.run(_call(probe, obj, generate=False)) == ["real answer"] assert obj.inner_calls == 1 @@ -176,7 +201,7 @@ def test_short_circuit_still_populates_the_harness_cache(probe): """lm-eval reconciles answers against cache_keys; skipping the hook would desync the run it is supposed to let finish cleanly.""" obj = _StubTemplateAPI() - _feed(probe, "length", 4) + _feed(probe, "length", 8) asyncio.run(_call(probe, obj, cache_keys=[("ctx", "kwargs")])) assert obj.cached == [("generate_until", ("ctx", "kwargs"), "")] @@ -186,18 +211,21 @@ def test_gate_survives_a_fresh_event_loop(probe): first loop that awaits it, so a non-loop-keyed gate would raise here.""" obj = _StubTemplateAPI() asyncio.run(_call(probe, obj)) - _feed(probe, "length", 4) + _feed(probe, "length", 8) assert asyncio.run(_call(probe, obj)) == [""] def test_sidecar_records_the_evidence(probe): - _feed(probe, "length", 4) + _feed(probe, "length", 8) sidecar = probe.result_dir / EVAL_PROBE_FILENAME record = json.loads(sidecar.read_text(encoding="utf-8")) assert record["reason"] == "model_not_terminating" - assert record["observed_samples"] == 4 - assert record["finish_reason_length"] == 4 + assert record["observed_samples"] == 8 + assert record["finish_reason_length"] == 8 assert record["length_ratio"] == 1.0 + assert record["cap_hits"] == 8 + assert record["cap_hit_ratio"] == 1.0 + assert record["written_at"] > 0 assert record["max_completion_tokens_seen"] == 16384 # parse_eval_results globs results*.json for the score; a probe sidecar # matching that name would be read as an lm-eval result file. @@ -207,7 +235,7 @@ def test_sidecar_records_the_evidence(probe): def test_sidecar_is_written_once(probe): """Every subsequent response would otherwise rewrite it with a diluted ratio, since short-circuited requests never report a finish_reason.""" - _feed(probe, "length", 4) + _feed(probe, "length", 8) first = (probe.result_dir / EVAL_PROBE_FILENAME).read_text(encoding="utf-8") _feed(probe, "length", 20) assert (probe.result_dir / EVAL_PROBE_FILENAME).read_text(encoding="utf-8") == first @@ -256,6 +284,88 @@ def test_read_eval_probe_finds_a_nested_sidecar(probe, tmp_path): assert record["source_file"].endswith(EVAL_PROBE_FILENAME) +def test_long_answers_below_the_ceiling_do_not_trip(probe): + """``finish_reason=length`` alone is not the pathology. lm-eval sizes + max_tokens per request from the remaining context, so a truncated-but- + terminating model produces capped responses at several different lengths; + only the ones piled on the ceiling are evidence of a runaway loop.""" + obj = _StubTemplateAPI() + for tokens in (1024,) * 4 + (2048,) * 4: + probe.openai_completions.LocalChatCompletion.parse_generations(outputs=_response("length", tokens)) + assert asyncio.run(_call(probe, obj)) == ["real answer"] + assert not (probe.result_dir / EVAL_PROBE_FILENAME).exists() + + +def test_length_ratio_zero_falls_back_to_the_default(monkeypatch, tmp_path): + """0 is exactly what an operator reaches for to disable the probe. Taken + literally it makes the ratio test vacuously true and guillotines every eval, + so an out-of-range value must fall back to the default, not be clamped.""" + p = _install( + monkeypatch, + tmp_path, + HYPERLOOM_EVAL_PROBE_MIN_SAMPLES="8", + HYPERLOOM_EVAL_PROBE_LENGTH_RATIO="0", + ) + obj = _StubTemplateAPI() + _feed(p, "stop", 8) + assert asyncio.run(_call(p, obj)) == ["real answer"] + assert not (tmp_path / EVAL_PROBE_FILENAME).exists() + + +def test_min_samples_below_the_floor_falls_back_to_the_default(monkeypatch, tmp_path): + """A one-sample window would end the eval on the first capped response.""" + p = _install( + monkeypatch, + tmp_path, + HYPERLOOM_EVAL_PROBE_MIN_SAMPLES="1", + HYPERLOOM_EVAL_PROBE_LENGTH_RATIO="0.75", + ) + obj = _StubTemplateAPI() + _feed(p, "length", 8) + assert asyncio.run(_call(p, obj)) == ["real answer"] + + +def test_no_result_dir_keeps_the_sidecar_out_of_the_cwd(monkeypatch, tmp_path): + """Without ``$RESULT_DIR`` the cwd is InferenceX's checkout, and writing + there is the artifact escape the _EVAL_DEST_* patch exists to prevent. The + record still reaches stderr, and the eval is still cut short.""" + monkeypatch.chdir(tmp_path) + p = _install(monkeypatch, tmp_path, RESULT_DIR=None, HYPERLOOM_EVAL_PROBE_MIN_SAMPLES="8") + obj = _StubTemplateAPI() + _feed(p, "length", 8) + assert asyncio.run(_call(p, obj)) == [""] + assert list(tmp_path.iterdir()) == [] + + +def test_install_drops_a_stale_sidecar(monkeypatch, tmp_path): + """The eval-failure retry reuses ``$RESULT_DIR``, so a sidecar left by the + previous attempt would be read as this run's verdict.""" + stale = tmp_path / EVAL_PROBE_FILENAME + stale.write_text(json.dumps({"reason": "model_not_terminating"}), encoding="utf-8") + + _install(monkeypatch, tmp_path, HYPERLOOM_EVAL_PROBE_MIN_SAMPLES="8") + + assert not stale.exists() + + +def test_read_eval_probe_prefers_the_newest_sidecar(tmp_path): + """``integrate_patch`` searches the grid slot, where sibling variants each + own a sidecar, and attempt dirs are hash-named — so path order says nothing + about which eval ran last.""" + older = tmp_path / "zzz_first" / EVAL_PROBE_FILENAME + newer = tmp_path / "aaa_second" / EVAL_PROBE_FILENAME + for path, ratio in ((older, 0.1), (newer, 0.9)): + path.parent.mkdir() + path.write_text(json.dumps({"length_ratio": ratio}), encoding="utf-8") + os.utime(older, (1_000_000, 1_000_000)) + os.utime(newer, (2_000_000, 2_000_000)) + + record = read_eval_probe(tmp_path) + + assert record is not None + assert record["length_ratio"] == 0.9 + + def test_read_eval_probe_is_none_without_a_sidecar(tmp_path): """No sidecar is the ordinary case: the model terminated its answers.""" assert read_eval_probe(tmp_path) is None diff --git a/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py b/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py index b2e4bac4e3..81b58291e1 100644 --- a/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py +++ b/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py @@ -576,6 +576,13 @@ def read_eval_probe(workspace: Path | str) -> dict[str, Any] | None: because the baseline double-run evaluates in the warmup round, whose ``$RESULT_DIR`` nests under the task workspace. + Several sidecars can be in scope at once — ``integrate_patch`` searches the + grid slot, whose sibling variant dirs each own one — so the newest by mtime + wins. Path order would not do: attempt dirs are hash-named, so sorting them + is unrelated to which eval ran last. The probe also removes any sidecar left + in its ``$RESULT_DIR`` before it starts, which covers the retry that reuses + one slot. + Args: workspace (Path | str): Benchmark workspace to search recursively. @@ -583,11 +590,11 @@ def read_eval_probe(workspace: Path | str) -> dict[str, Any] | None: dict[str, Any] | None: The probe record stamped with ``kind`` and ``source_file``, or ``None`` when no readable sidecar exists. """ - matches = sorted(Path(workspace).rglob(EVAL_PROBE_FILENAME)) + matches = list(Path(workspace).rglob(EVAL_PROBE_FILENAME)) if not matches: return None - latest = matches[-1] try: + latest = max(matches, key=lambda p: p.stat().st_mtime) record = json.loads(latest.read_text(encoding="utf-8")) except (json.JSONDecodeError, OSError): return None @@ -607,10 +614,13 @@ def eval_probe_summary(probe: dict[str, Any] | None) -> str: """ if not probe: return "" + hits = probe.get("cap_hits") + if hits is None: + hits = probe.get("finish_reason_length", 0) return ( - f"{EVAL_KIND_GENERATION_PATHOLOGY}: {probe.get('finish_reason_length', 0)}/" - f"{probe.get('observed_samples', 0)} sampled responses hit the max_tokens cap " - f"(up to {probe.get('max_completion_tokens_seen', 0)} tokens); the model never " + f"{EVAL_KIND_GENERATION_PATHOLOGY}: {hits}/" + f"{probe.get('observed_samples', 0)} sampled responses stopped at the " + f"{probe.get('max_completion_tokens_seen', 0)}-token cap; the model never " "emitted EOS, so the eval was cut short and scored ~0" ) diff --git a/src/hyperloom/orchestrator/actions/executors/_inferencex_patcher.py b/src/hyperloom/orchestrator/actions/executors/_inferencex_patcher.py index 4ffde5ba85..5ffef51898 100644 --- a/src/hyperloom/orchestrator/actions/executors/_inferencex_patcher.py +++ b/src/hyperloom/orchestrator/actions/executors/_inferencex_patcher.py @@ -85,26 +85,47 @@ import json as _hl_json import os as _hl_os import sys as _hl_sys +import time as _hl_time def _hl_eval_probe_install(): if (_hl_os.environ.get("HYPERLOOM_EVAL_PROBE") or "1").strip().lower() in ("0", "false", "no", "off"): return - def _num(name, default, cast): + def _num(name, default, cast, ok): try: - return cast((_hl_os.environ.get(name) or "").strip()) + val = cast((_hl_os.environ.get(name) or "").strip()) except (TypeError, ValueError): return default + return val if ok(val) else default - min_samples = max(1, _num("HYPERLOOM_EVAL_PROBE_MIN_SAMPLES", 16, int)) - ratio_limit = _num("HYPERLOOM_EVAL_PROBE_LENGTH_RATIO", 0.75, float) + # Out-of-range falls back to the default rather than to the nearest legal + # value. RATIO=0 is what an operator reaches for to "turn the probe off", + # and clamping it to the smallest legal ratio would do the opposite: cut + # every eval short the moment min_samples is reached. + min_samples = _num("HYPERLOOM_EVAL_PROBE_MIN_SAMPLES", 128, int, lambda v: v >= 8) + ratio_limit = _num("HYPERLOOM_EVAL_PROBE_LENGTH_RATIO", 0.75, float, lambda v: 0.0 < v <= 1.0) import asyncio as _hl_asyncio from lm_eval.models import api_models as _hl_api from lm_eval.models.openai_completions import LocalChatCompletion as _hl_lcc - state = {"observed": 0, "length": 0, "max_tokens_seen": 0, "tripped": False} + # Reaching here proves this is the lm-eval process, not one of the other + # python3 invocations sitecustomize also runs in -- so only here is it safe + # to drop a sidecar left by a previous attempt. The eval-failure retry + # reuses $RESULT_DIR, and a stale file would be read as this run's verdict. + _hl_dir = (_hl_os.environ.get("RESULT_DIR") or "").strip() + if _hl_dir: + try: + _hl_os.remove(_hl_os.path.join(_hl_dir, "hyperloom_eval_probe.json")) + except OSError: + pass + + state = {"observed": 0, "length": 0, "max_tokens_seen": 0, "cap_hits": 0, "tripped": False} + # completion_tokens -> count, over responses the server stopped on length. + # The cap is the largest such value: a model that never terminates piles + # every capped response onto exactly that number. + capped = {} def _emit(): record = { @@ -112,16 +133,24 @@ def _emit(): "observed_samples": state["observed"], "finish_reason_length": state["length"], "length_ratio": round(float(state["length"]) / state["observed"], 4), + "cap_hits": state["cap_hits"], + "cap_hit_ratio": round(float(state["cap_hits"]) / state["observed"], 4), "max_completion_tokens_seen": state["max_tokens_seen"], "min_samples": min_samples, "length_ratio_threshold": ratio_limit, + "written_at": _hl_time.time(), } blob = _hl_json.dumps(record, sort_keys=True) print("HYPERLOOM_EVAL_PROBE_TRIPPED " + blob, file=_hl_sys.stderr, flush=True) # $RESULT_DIR, never $EVAL_RESULT_DIR: append_lm_eval_summary rm -rf's # the latter. The name must not match results*.json -- that glob is how # parse_eval_results finds the accuracy score. - out_dir = (_hl_os.environ.get("RESULT_DIR") or "").strip() or "." + out_dir = (_hl_os.environ.get("RESULT_DIR") or "").strip() + if not out_dir: + # Without it the sidecar would land in the cwd, i.e. InferenceX's + # checkout -- the artifact escape the _EVAL_DEST_* patch exists to + # prevent. stderr above already carries the whole record. + return _hl_os.makedirs(out_dir, exist_ok=True) with open(_hl_os.path.join(out_dir, "hyperloom_eval_probe.json"), "w", encoding="utf-8") as fh: fh.write(blob) @@ -134,7 +163,16 @@ def _observe(outputs): state["observed"] += 1 if choice.get("finish_reason") == "length": state["length"] += 1 - if state["observed"] >= min_samples and float(state["length"]) / state["observed"] >= ratio_limit: + capped[seen] = capped.get(seen, 0) + 1 + if state["observed"] < min_samples: + return + # Count only the responses that stopped AT the ceiling. A bare + # finish_reason=length ratio cannot separate "never terminates" from + # "legitimately long answers under a small cap", and the ceiling was + # already being tracked without being used. + cap = max(capped) if capped else 0 + state["cap_hits"] = capped.get(cap, 0) + if cap > 0 and float(state["cap_hits"]) / state["observed"] >= ratio_limit: state["tripped"] = True _emit() From 32c50f17c144dcd46989688e29a193a21ff1125f Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Mon, 3 Aug 2026 11:31:39 +0000 Subject: [PATCH 19/22] fix(enablement): tell the specialist the eval was cut short, not answered wrong EVAL_KIND_GENERATION_PATHOLOGY was defined and stamped onto the probe record, but no decision ever read it: classify_accuracy_failure never returns it, and its only consumers append it to a log line or to a revert reason that had already been decided. A truncated eval was therefore indistinguishable from a model that answered and got them wrong -- same ~0 score, same accuracy_below_floor routing -- and the authoring specialist received evidence reading only "accuracy=0.0", sending it after a quality regression that never happened. The baseline executor now stamps the pathology kind and appends the probe summary to the evidence when the probe tripped. classify_accuracy_failure stays pure: it is handed a number and cannot see the probe. Co-authored-by: Cursor --- .../tests/test_baseline_eval_fallback.py | 27 +++++++++++++++++++ .../actions/executors/baseline.py | 16 ++++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_baseline_eval_fallback.py b/src/hyperloom/inference_optimizer/tests/test_baseline_eval_fallback.py index 0bb021900d..8ae28bb2a8 100644 --- a/src/hyperloom/inference_optimizer/tests/test_baseline_eval_fallback.py +++ b/src/hyperloom/inference_optimizer/tests/test_baseline_eval_fallback.py @@ -741,6 +741,7 @@ def fake_run(cmd, *args, **kwargs): BASELINE_EVAL_OBSERVED_ACCURACY_KEY, EVAL_KIND_ACCURACY_BELOW_FLOOR, EVAL_KIND_ACCURACY_UNAVAILABLE, + EVAL_KIND_GENERATION_PATHOLOGY, ) @@ -793,6 +794,32 @@ def test_eval_enablement_zero_accuracy_below_floor(monkeypatch): assert result[BASELINE_EVAL_OBSERVED_ACCURACY_KEY] == 0.0 +def test_eval_enablement_probe_reports_generation_pathology(monkeypatch): + """A tripped probe changes what a ~0 score means: the eval was cut short + because the model never stopped generating, not because it answered and got + them wrong. Without this the specialist is handed a bare ``accuracy=0.0`` + and goes looking for a quality regression that never happened. + """ + result = { + "status": "succeeded", + "accuracy": 0.0, + "run_eval_disabled": False, + "eval_probe": { + "kind": EVAL_KIND_GENERATION_PATHOLOGY, + "observed_samples": 128, + "cap_hits": 128, + "max_completion_tokens_seen": 16384, + }, + } + reason = _route(monkeypatch, "sglang", result) + assert reason == "" + assert result[BASELINE_EVAL_FAILURE_KIND_KEY] == EVAL_KIND_GENERATION_PATHOLOGY + assert result[BASELINE_EVAL_OBSERVED_ACCURACY_KEY] == 0.0 + evidence = result[BASELINE_EVAL_EVIDENCE_KEY] + assert "128/128" in evidence + assert "16384" in evidence + + def test_eval_enablement_positive_below_floor(monkeypatch): observed = DEFAULT_ENABLEMENT_ACCURACY_FLOOR / 2 result = {"status": "succeeded", "accuracy": observed, "run_eval_disabled": False} diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index 95a3a00f8d..edfb1e2de0 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -1649,8 +1649,10 @@ def _maybe_stop_on_missing_baseline_accuracy( eval_enablement = self._eval_enablement_active(ctx) from ._accuracy_gate import ( DEFAULT_ENABLEMENT_ACCURACY_FLOOR, + EVAL_KIND_GENERATION_PATHOLOGY, accuracy_meets_floor, classify_accuracy_failure, + eval_probe_summary, ) floor = DEFAULT_ENABLEMENT_ACCURACY_FLOOR @@ -1718,14 +1720,26 @@ def _maybe_stop_on_missing_baseline_accuracy( f"task={result.get('accuracy_task')} metric={result.get('accuracy_metric')} " f"source={result.get('accuracy_source')}" ) + # A tripped probe changes what the score means: the eval was cut + # short because the model never stopped generating, so the ~0 is a + # broken generation loop and not a model that answered and got them + # wrong. ``classify_accuracy_failure`` cannot see this -- it only + # gets a number -- and an authoring specialist told nothing but + # "accuracy=0.0" goes looking for a quality regression that never + # happened. + probe = result.get("eval_probe") + if probe: + kind = EVAL_KIND_GENERATION_PATHOLOGY + evidence = f"{evidence}; {eval_probe_summary(probe)}" self._stamp_eval_failure_contract( ctx, result, kind=kind or "", observed_accuracy=observed, evidence=evidence ) log.warning( - "baseline_executor: accuracy %s below floor %.4f; routing to " + "baseline_executor: accuracy %s below floor %.4f (kind=%s); routing to " "enablement instead of stopping the run.", acc, floor, + kind or "unclassified", ) return request_baseline_accuracy_stop( From aa0d09aea6e2384e4787a8be99b2455b988f3a98 Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Mon, 3 Aug 2026 11:41:59 +0000 Subject: [PATCH 20/22] refactor(eval): trim the probe record and its comments to what is read Follow-up to 53cbc821b / 32c50f17c. Three sidecar fields were written and never read: written_at (read_eval_probe orders by mtime), cap_hit_ratio and length_ratio (both derivable from counts already in the record). The remaining threshold is renamed to say which ratio it bounds, now that the trip test keys off cap hits. eval_probe_summary drops its finish_reason_length fallback and the routing drops an "unclassified" default for a kind that cannot be empty there -- neither case is reachable, since the probe writes and reads within one run and the classifier only returns None above the floor. The rest is comment length: six explanatory blocks cut to the constraint they state, and _maybe_stop_on_missing_baseline_accuracy now documents that a tripped probe re-labels the eval-failure contract, which the code already did without saying so. Co-authored-by: Cursor --- .../tests/test_eval_probe.py | 13 +++----- .../actions/executors/_accuracy_gate.py | 14 +++----- .../actions/executors/_inferencex_patcher.py | 31 +++++------------ .../actions/executors/baseline.py | 33 ++++++++----------- 4 files changed, 31 insertions(+), 60 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_eval_probe.py b/src/hyperloom/inference_optimizer/tests/test_eval_probe.py index b9450886a6..bf22de2517 100644 --- a/src/hyperloom/inference_optimizer/tests/test_eval_probe.py +++ b/src/hyperloom/inference_optimizer/tests/test_eval_probe.py @@ -222,10 +222,7 @@ def test_sidecar_records_the_evidence(probe): assert record["reason"] == "model_not_terminating" assert record["observed_samples"] == 8 assert record["finish_reason_length"] == 8 - assert record["length_ratio"] == 1.0 assert record["cap_hits"] == 8 - assert record["cap_hit_ratio"] == 1.0 - assert record["written_at"] > 0 assert record["max_completion_tokens_seen"] == 16384 # parse_eval_results globs results*.json for the score; a probe sidecar # matching that name would be read as an lm-eval result file. @@ -354,16 +351,16 @@ def test_read_eval_probe_prefers_the_newest_sidecar(tmp_path): about which eval ran last.""" older = tmp_path / "zzz_first" / EVAL_PROBE_FILENAME newer = tmp_path / "aaa_second" / EVAL_PROBE_FILENAME - for path, ratio in ((older, 0.1), (newer, 0.9)): + for path, hits in ((older, 11), (newer, 22)): path.parent.mkdir() - path.write_text(json.dumps({"length_ratio": ratio}), encoding="utf-8") + path.write_text(json.dumps({"cap_hits": hits}), encoding="utf-8") os.utime(older, (1_000_000, 1_000_000)) os.utime(newer, (2_000_000, 2_000_000)) record = read_eval_probe(tmp_path) assert record is not None - assert record["length_ratio"] == 0.9 + assert record["cap_hits"] == 22 def test_read_eval_probe_is_none_without_a_sidecar(tmp_path): @@ -381,9 +378,7 @@ def test_eval_probe_summary_is_empty_without_a_probe(): def test_eval_probe_summary_names_the_kind_and_the_evidence(): - summary = eval_probe_summary( - {"observed_samples": 16, "finish_reason_length": 16, "max_completion_tokens_seen": 16384} - ) + summary = eval_probe_summary({"observed_samples": 16, "cap_hits": 16, "max_completion_tokens_seen": 16384}) assert EVAL_KIND_GENERATION_PATHOLOGY in summary assert "16/16" in summary assert "16384" in summary diff --git a/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py b/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py index 81b58291e1..e35b1f4203 100644 --- a/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py +++ b/src/hyperloom/orchestrator/actions/executors/_accuracy_gate.py @@ -576,12 +576,9 @@ def read_eval_probe(workspace: Path | str) -> dict[str, Any] | None: because the baseline double-run evaluates in the warmup round, whose ``$RESULT_DIR`` nests under the task workspace. - Several sidecars can be in scope at once — ``integrate_patch`` searches the - grid slot, whose sibling variant dirs each own one — so the newest by mtime - wins. Path order would not do: attempt dirs are hash-named, so sorting them - is unrelated to which eval ran last. The probe also removes any sidecar left - in its ``$RESULT_DIR`` before it starts, which covers the retry that reuses - one slot. + Newest by mtime wins: ``integrate_patch`` searches a grid slot whose sibling + variants each own a sidecar, and attempt dirs are hash-named, so path order + says nothing about which eval ran last. Args: workspace (Path | str): Benchmark workspace to search recursively. @@ -614,11 +611,8 @@ def eval_probe_summary(probe: dict[str, Any] | None) -> str: """ if not probe: return "" - hits = probe.get("cap_hits") - if hits is None: - hits = probe.get("finish_reason_length", 0) return ( - f"{EVAL_KIND_GENERATION_PATHOLOGY}: {hits}/" + f"{EVAL_KIND_GENERATION_PATHOLOGY}: {probe.get('cap_hits', 0)}/" f"{probe.get('observed_samples', 0)} sampled responses stopped at the " f"{probe.get('max_completion_tokens_seen', 0)}-token cap; the model never " "emitted EOS, so the eval was cut short and scored ~0" diff --git a/src/hyperloom/orchestrator/actions/executors/_inferencex_patcher.py b/src/hyperloom/orchestrator/actions/executors/_inferencex_patcher.py index 5ffef51898..b56fcec849 100644 --- a/src/hyperloom/orchestrator/actions/executors/_inferencex_patcher.py +++ b/src/hyperloom/orchestrator/actions/executors/_inferencex_patcher.py @@ -85,7 +85,6 @@ import json as _hl_json import os as _hl_os import sys as _hl_sys -import time as _hl_time def _hl_eval_probe_install(): @@ -99,10 +98,8 @@ def _num(name, default, cast, ok): return default return val if ok(val) else default - # Out-of-range falls back to the default rather than to the nearest legal - # value. RATIO=0 is what an operator reaches for to "turn the probe off", - # and clamping it to the smallest legal ratio would do the opposite: cut - # every eval short the moment min_samples is reached. + # Out of range falls back to the default, not to the nearest legal value: + # RATIO=0 means "turn the probe off", and clamping would do the opposite. min_samples = _num("HYPERLOOM_EVAL_PROBE_MIN_SAMPLES", 128, int, lambda v: v >= 8) ratio_limit = _num("HYPERLOOM_EVAL_PROBE_LENGTH_RATIO", 0.75, float, lambda v: 0.0 < v <= 1.0) @@ -110,10 +107,9 @@ def _num(name, default, cast, ok): from lm_eval.models import api_models as _hl_api from lm_eval.models.openai_completions import LocalChatCompletion as _hl_lcc - # Reaching here proves this is the lm-eval process, not one of the other - # python3 invocations sitecustomize also runs in -- so only here is it safe - # to drop a sidecar left by a previous attempt. The eval-failure retry - # reuses $RESULT_DIR, and a stale file would be read as this run's verdict. + # The imports above prove this is lm-eval, not one of the other python3 + # invocations sitecustomize runs in, so any sidecar here is a stale one from + # the attempt that reused this $RESULT_DIR. _hl_dir = (_hl_os.environ.get("RESULT_DIR") or "").strip() if _hl_dir: try: @@ -123,8 +119,6 @@ def _num(name, default, cast, ok): state = {"observed": 0, "length": 0, "max_tokens_seen": 0, "cap_hits": 0, "tripped": False} # completion_tokens -> count, over responses the server stopped on length. - # The cap is the largest such value: a model that never terminates piles - # every capped response onto exactly that number. capped = {} def _emit(): @@ -132,13 +126,10 @@ def _emit(): "reason": "model_not_terminating", "observed_samples": state["observed"], "finish_reason_length": state["length"], - "length_ratio": round(float(state["length"]) / state["observed"], 4), "cap_hits": state["cap_hits"], - "cap_hit_ratio": round(float(state["cap_hits"]) / state["observed"], 4), "max_completion_tokens_seen": state["max_tokens_seen"], "min_samples": min_samples, - "length_ratio_threshold": ratio_limit, - "written_at": _hl_time.time(), + "cap_hit_ratio_threshold": ratio_limit, } blob = _hl_json.dumps(record, sort_keys=True) print("HYPERLOOM_EVAL_PROBE_TRIPPED " + blob, file=_hl_sys.stderr, flush=True) @@ -147,9 +138,7 @@ def _emit(): # parse_eval_results finds the accuracy score. out_dir = (_hl_os.environ.get("RESULT_DIR") or "").strip() if not out_dir: - # Without it the sidecar would land in the cwd, i.e. InferenceX's - # checkout -- the artifact escape the _EVAL_DEST_* patch exists to - # prevent. stderr above already carries the whole record. + # The cwd is InferenceX's checkout; stderr above already has it all. return _hl_os.makedirs(out_dir, exist_ok=True) with open(_hl_os.path.join(out_dir, "hyperloom_eval_probe.json"), "w", encoding="utf-8") as fh: @@ -166,10 +155,8 @@ def _observe(outputs): capped[seen] = capped.get(seen, 0) + 1 if state["observed"] < min_samples: return - # Count only the responses that stopped AT the ceiling. A bare - # finish_reason=length ratio cannot separate "never terminates" from - # "legitimately long answers under a small cap", and the ceiling was - # already being tracked without being used. + # A model that never terminates piles every capped response onto the + # same ceiling; cap 0 means no usage was reported, so it is unknown. cap = max(capped) if capped else 0 state["cap_hits"] = capped.get(cap, 0) if cap > 0 and float(state["cap_hits"]) / state["observed"] >= ratio_limit: diff --git a/src/hyperloom/orchestrator/actions/executors/baseline.py b/src/hyperloom/orchestrator/actions/executors/baseline.py index edfb1e2de0..d2db859d80 100644 --- a/src/hyperloom/orchestrator/actions/executors/baseline.py +++ b/src/hyperloom/orchestrator/actions/executors/baseline.py @@ -1619,7 +1619,9 @@ def _maybe_stop_on_missing_baseline_accuracy( ``kind="baseline"`` opt out earlier, via ``quality_ref_exempt``. With eval-on-fail enablement active (the default), the result is stamped - as an eval-failure contract and routed to enablement rather than + as an eval-failure contract -- ``eval_generation_pathology`` when the + generation probe tripped, else whatever the score classifies as -- and + routed to enablement rather than stopping; ``_is_promotable_result`` then blocks it from anchoring ``baseline_tput`` / ``baseline_accuracy`` / ``baseline_config_path``. Otherwise the run stops. Throughput-level baseline failures are handled @@ -1720,13 +1722,8 @@ def _maybe_stop_on_missing_baseline_accuracy( f"task={result.get('accuracy_task')} metric={result.get('accuracy_metric')} " f"source={result.get('accuracy_source')}" ) - # A tripped probe changes what the score means: the eval was cut - # short because the model never stopped generating, so the ~0 is a - # broken generation loop and not a model that answered and got them - # wrong. ``classify_accuracy_failure`` cannot see this -- it only - # gets a number -- and an authoring specialist told nothing but - # "accuracy=0.0" goes looking for a quality regression that never - # happened. + # A tripped probe means the eval was cut short because the model + # never stopped generating, not that it answered and got them wrong. probe = result.get("eval_probe") if probe: kind = EVAL_KIND_GENERATION_PATHOLOGY @@ -1739,7 +1736,7 @@ def _maybe_stop_on_missing_baseline_accuracy( "enablement instead of stopping the run.", acc, floor, - kind or "unclassified", + kind, ) return request_baseline_accuracy_stop( @@ -1753,16 +1750,14 @@ def _apply_salvaged_accuracy( salvaged: dict[str, Any], shared_state: Any, ) -> float: - """Record a salvaged sibling accuracy as evidence, and as the gate input - only when it can serve as one. - - The ``result`` fields are evidence and carry the score whatever its - value. ``SharedState.baseline_accuracy`` is not evidence — it is the - reference every later accuracy gate compares against, and ``<= 0`` is - that gate's "no baseline, skip the check" sentinel - (:func:`accuracy_passed`). Publishing a measured zero there would turn a - real quality failure into a silent gate bypass for every subsequent - candidate, so the two are written under different conditions. + """Record a salvaged sibling accuracy, publishing it as the gate + reference only when it can serve as one. + + ``result`` carries the score whatever its value: that is evidence. + ``SharedState.baseline_accuracy`` is the reference later gates compare + against, where ``<= 0`` is :func:`accuracy_passed`'s "no baseline, skip + the check" sentinel -- a measured zero there bypasses the gate for every + later candidate. Args: result: The baseline result dict, mutated in place. From 4d77e4230eee2bd778aefd43e6aa1494f100536a Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Mon, 3 Aug 2026 12:14:18 +0000 Subject: [PATCH 21/22] fix(enablement): classify a cut-short eval as its own failure kind The baseline stamps eval_generation_pathology, but the specialist's kind comes from classify_failure over the evidence text, and that text also carries the "accuracy did not meet floor" phrasing -- so a truncated eval arrived labelled accuracy_below_floor and the specialist was pointed at answer quality instead of at generation that never terminates. Declaring the kind in FAILURE_KINDS is not enough on its own: classify_failure elects a primary by position in _RULES, so the kind needs a rule of its own, ahead of the below-floor rule it shadows. The ladder book now names the kind alongside the other eval triggers, since a kind the classifier can return with no methodology entry is the same gap one step further along. Co-authored-by: Cursor --- src/hyperloom/agents/framework/README.md | 3 ++- src/hyperloom/agents/framework/enablement.py | 15 ++++++++++++++- src/hyperloom/agents/framework/enablement_ops.py | 5 +++-- .../agents/framework/tests/test_enablement.py | 12 ++++++++++++ 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/hyperloom/agents/framework/README.md b/src/hyperloom/agents/framework/README.md index 089d232367..2968393db7 100644 --- a/src/hyperloom/agents/framework/README.md +++ b/src/hyperloom/agents/framework/README.md @@ -29,7 +29,8 @@ bridging patch, gated on *does it run correctly* rather than *is it faster*: launch/import/build/eval log into a `FailureSignature` (`missing_model_arch` / `unsupported_dtype` / `hip_kernel_missing` / `import_error` / `shape_mismatch` / `not_implemented` / - `capability_disabled` / `accuracy_below_floor` / `eval_runtime_failure`) + `capability_disabled` / `accuracy_below_floor` / + `eval_generation_pathology` / `eval_runtime_failure`) with the offending file/symbol and a `bridge_layer`. 2. **Discover** — `hyperloom.agents.framework.enablement_ops.build_search_plan(...)` picks the repos to scout (the framework repo, plus ROCm/HIP/aiter via diff --git a/src/hyperloom/agents/framework/enablement.py b/src/hyperloom/agents/framework/enablement.py index dd13b46f55..18066f07a6 100644 --- a/src/hyperloom/agents/framework/enablement.py +++ b/src/hyperloom/agents/framework/enablement.py @@ -33,14 +33,17 @@ # Resource constraints (OOM, TP/GPU count) are NOT code acquisition targets. RESOURCE_CONSTRAINT = "resource_constraint" # Accuracy-eval triggers (values match _accuracy_gate EVAL_KIND_*): a booting -# baseline whose accuracy is below the floor, and a crashed eval run. +# baseline whose accuracy is below the floor, an eval cut short because +# generation never terminated, and a crashed eval run. ACCURACY_BELOW_FLOOR = "accuracy_below_floor" +EVAL_GENERATION_PATHOLOGY = "eval_generation_pathology" EVAL_RUNTIME_FAILURE = "eval_runtime_failure" UNKNOWN = "unknown" # Ordered most-specific to least-specific. FAILURE_KINDS: tuple[str, ...] = ( MISSING_MODEL_ARCH, + EVAL_GENERATION_PATHOLOGY, ACCURACY_BELOW_FLOOR, RESOURCE_CONSTRAINT, HIP_KERNEL_MISSING, @@ -192,6 +195,15 @@ def _grp(match: re.Match[str]) -> str: confidence=0.95, symbol_from=_grp, ), + _Rule( + # The eval was cut short because generation never terminated, so the ~0 + # score says nothing about answer quality. Precedes ACCURACY_BELOW_FLOOR, + # whose evidence string it also carries. + kind=EVAL_GENERATION_PATHOLOGY, + bridge_layer="", + patterns=(re.compile(r"eval_generation_pathology"),), + confidence=0.95, + ), _Rule( # A booting baseline whose accuracy is below the floor. Not a bridge-repo # target — the fix is correctness of the model's real output. @@ -714,6 +726,7 @@ def is_targeted_build_candidate( __all__ = [ "ACCURACY_BELOW_FLOOR", "CAPABILITY_DISABLED", + "EVAL_GENERATION_PATHOLOGY", "EVAL_RUNTIME_FAILURE", "FAILURE_KINDS", "HIP_KERNEL_MISSING", diff --git a/src/hyperloom/agents/framework/enablement_ops.py b/src/hyperloom/agents/framework/enablement_ops.py index 04490311cf..82d5cb5baa 100644 --- a/src/hyperloom/agents/framework/enablement_ops.py +++ b/src/hyperloom/agents/framework/enablement_ops.py @@ -389,8 +389,9 @@ def _resolve_actual_root_hints(framework: str) -> list[str]: "import_error / merged-PR closure -> Rung 4", "hip_kernel_missing / native unsupported_dtype / missing compiled symbol -> Rung 5", "resource_constraint (OOM / GPU count) -> NOT a code gap; cannot be patched", - "accuracy_below_floor / eval_runtime_failure -> re-diagnose against the failing " - "eval contract (accuracy target), then enter at the rung the underlying gap implies", + "accuracy_below_floor / eval_generation_pathology / eval_runtime_failure -> " + "re-diagnose against the failing eval contract (answer quality, or generation " + "that never terminates), then enter at the rung the underlying gap implies", ) diff --git a/src/hyperloom/agents/framework/tests/test_enablement.py b/src/hyperloom/agents/framework/tests/test_enablement.py index fc6dc49461..571337b807 100644 --- a/src/hyperloom/agents/framework/tests/test_enablement.py +++ b/src/hyperloom/agents/framework/tests/test_enablement.py @@ -14,6 +14,7 @@ from hyperloom.agents.framework.enablement import ( ACCURACY_BELOW_FLOOR, CAPABILITY_DISABLED, + EVAL_GENERATION_PATHOLOGY, EVAL_RUNTIME_FAILURE, HIP_KERNEL_MISSING, IMPORT_ERROR, @@ -47,6 +48,17 @@ def test_accuracy_below_floor_kind() -> None: assert sig.bridge_layer == "" +def test_generation_pathology_outranks_accuracy_below_floor() -> None: + """The probe's evidence carries the below-floor phrasing too, but a truncated + eval is a different repair from a model that answered and got them wrong.""" + sig = classify_failure( + "baseline accuracy did not meet floor: accuracy=0.0 floor=0.05 task=gsm8k; " + "eval_generation_pathology: 128/128 sampled responses stopped at the 16384-token cap" + ) + assert sig.kind == EVAL_GENERATION_PATHOLOGY + assert ACCURACY_BELOW_FLOOR in sig.secondary_kinds + + def test_eval_runtime_failure_kind() -> None: sig = classify_failure("benchmark_stderr.log: ERROR: run_eval failed with exit code 1") assert sig.kind == EVAL_RUNTIME_FAILURE From 8e15515473a59f89247865d0b7ffe80def9f7c2c Mon Sep 17 00:00:00 2001 From: ZhengGong-amd Date: Mon, 3 Aug 2026 12:14:29 +0000 Subject: [PATCH 22/22] fix(trace): keep gateway credentials out of the orchestration trace urlsplit().netloc includes URL userinfo, so a base URL configured as https://user:key@gateway/... put the key straight into an on-disk trace row. The identifier is the host, so read parts.hostname instead. gateway_endpoint was also the one free-text string in to_row() not passed through _safe_value, in a method whose whole contract is "serialize a redacted row"; every other string field there is wrapped. Co-authored-by: Cursor --- .../tests/test_claude_backend_branches_unit.py | 14 ++++++++++++++ src/hyperloom/orchestrator/roles/claude.py | 3 ++- .../orchestrator/trace/orchestration_trace.py | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/hyperloom/inference_optimizer/tests/test_claude_backend_branches_unit.py b/src/hyperloom/inference_optimizer/tests/test_claude_backend_branches_unit.py index b067091976..f762d7e4a4 100644 --- a/src/hyperloom/inference_optimizer/tests/test_claude_backend_branches_unit.py +++ b/src/hyperloom/inference_optimizer/tests/test_claude_backend_branches_unit.py @@ -264,6 +264,20 @@ async def test_run_skips_diagnostics_when_not_requested(): assert b.get_turn_diagnostic() == {} +# ---- gateway endpoint identifier ----------------------------------------- +def test_gateway_endpoint_drops_url_userinfo(monkeypatch): + """The diagnostic is appended to an on-disk trace, and a base URL of the + form ``https://user:key@gw/...`` puts the key in netloc.""" + monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://user:s3cret@gw.example.com:8443/api/v1") + assert _backend()._gateway_endpoint_identifier() == "gw.example.com" + + +def test_gateway_endpoint_is_none_without_a_base_url(monkeypatch): + for var in ("ANTHROPIC_BASE_URL", "DEEPSEEK_BASE_URL", "OPENAI_BASE_URL"): + monkeypatch.delenv(var, raising=False) + assert _backend()._gateway_endpoint_identifier() is None + + # ---- _invoke_and_collect: error-result-success tolerance ------------------ async def test_invoke_error_result_success_with_intents(): msg = _Msg(content=[_emit_tool_block()]) diff --git a/src/hyperloom/orchestrator/roles/claude.py b/src/hyperloom/orchestrator/roles/claude.py index 335d617b00..9a3383d119 100644 --- a/src/hyperloom/orchestrator/roles/claude.py +++ b/src/hyperloom/orchestrator/roles/claude.py @@ -638,7 +638,8 @@ def _gateway_endpoint_identifier(self) -> str | None: if not raw: return None parts = urlsplit(raw) - return parts.netloc or "configured" + # hostname, not netloc: netloc carries any ``user:secret@`` userinfo. + return parts.hostname or "configured" @staticmethod def _session_hash(session_id: str | None) -> str | None: diff --git a/src/hyperloom/orchestrator/trace/orchestration_trace.py b/src/hyperloom/orchestrator/trace/orchestration_trace.py index 8970d61654..5b68a294c9 100644 --- a/src/hyperloom/orchestrator/trace/orchestration_trace.py +++ b/src/hyperloom/orchestrator/trace/orchestration_trace.py @@ -142,7 +142,7 @@ def to_row(self) -> dict[str, Any]: "sdk_name": self.sdk_name, "sdk_version": self.sdk_version, "cli_version": self.cli_version, - "gateway_endpoint": self.gateway_endpoint, + "gateway_endpoint": _safe_value(self.gateway_endpoint), "request_id": self.request_id, "resume_requested": bool(self.resume_requested), "previous_session_id_hash": self.previous_session_id_hash,