[STEPPING] grade against the live anchor, retire the baseline, and make a zero accuracy explain itself

src/hyperloom/agents/framework/README.md

@@ -29,7 +29,8 @@ bridging patch, gated on *does it run correctly* rather than *is it faster*:
    launch/import/build/eval log into a `FailureSignature`
    (`missing_model_arch` / `unsupported_dtype` / `hip_kernel_missing` /
    `import_error` / `shape_mismatch` / `not_implemented` /
-   `capability_disabled` / `accuracy_below_floor` / `eval_runtime_failure`)
+   `capability_disabled` / `accuracy_below_floor` /
+   `eval_generation_pathology` / `eval_runtime_failure`)
    with the offending file/symbol and a `bridge_layer`.
 2. **Discover** — `hyperloom.agents.framework.enablement_ops.build_search_plan(...)`
    picks the repos to scout (the framework repo, plus ROCm/HIP/aiter via

src/hyperloom/agents/framework/enablement.py

@@ -33,14 +33,17 @@
 # Resource constraints (OOM, TP/GPU count) are NOT code acquisition targets.
 RESOURCE_CONSTRAINT = "resource_constraint"
 # Accuracy-eval triggers (values match _accuracy_gate EVAL_KIND_*): a booting
-# baseline whose accuracy is below the floor, and a crashed eval run.
+# baseline whose accuracy is below the floor, an eval cut short because
+# generation never terminated, and a crashed eval run.
 ACCURACY_BELOW_FLOOR = "accuracy_below_floor"
+EVAL_GENERATION_PATHOLOGY = "eval_generation_pathology"
 EVAL_RUNTIME_FAILURE = "eval_runtime_failure"
 UNKNOWN = "unknown"
 
 # Ordered most-specific to least-specific.
 FAILURE_KINDS: tuple[str, ...] = (
     MISSING_MODEL_ARCH,
+    EVAL_GENERATION_PATHOLOGY,
     ACCURACY_BELOW_FLOOR,
     RESOURCE_CONSTRAINT,
     HIP_KERNEL_MISSING,
@@ -192,6 +195,15 @@ def _grp(match: re.Match[str]) -> str:
         confidence=0.95,
         symbol_from=_grp,
     ),
+    _Rule(
+        # The eval was cut short because generation never terminated, so the ~0
+        # score says nothing about answer quality. Precedes ACCURACY_BELOW_FLOOR,
+        # whose evidence string it also carries.
+        kind=EVAL_GENERATION_PATHOLOGY,
+        bridge_layer="",
+        patterns=(re.compile(r"eval_generation_pathology"),),
+        confidence=0.95,
+    ),
     _Rule(
         # A booting baseline whose accuracy is below the floor. Not a bridge-repo
         # target — the fix is correctness of the model's real output.
@@ -714,6 +726,7 @@ def is_targeted_build_candidate(
 __all__ = [
     "ACCURACY_BELOW_FLOOR",
     "CAPABILITY_DISABLED",
+    "EVAL_GENERATION_PATHOLOGY",
     "EVAL_RUNTIME_FAILURE",
     "FAILURE_KINDS",
     "HIP_KERNEL_MISSING",

src/hyperloom/agents/framework/enablement_ops.py

@@ -389,8 +389,9 @@ def _resolve_actual_root_hints(framework: str) -> list[str]:
     "import_error / merged-PR closure -> Rung 4",
     "hip_kernel_missing / native unsupported_dtype / missing compiled symbol -> Rung 5",
     "resource_constraint (OOM / GPU count) -> NOT a code gap; cannot be patched",
-    "accuracy_below_floor / eval_runtime_failure -> re-diagnose against the failing "
-    "eval contract (accuracy target), then enter at the rung the underlying gap implies",
+    "accuracy_below_floor / eval_generation_pathology / eval_runtime_failure -> "
+    "re-diagnose against the failing eval contract (answer quality, or generation "
+    "that never terminates), then enter at the rung the underlying gap implies",
 )
 
 

src/hyperloom/agents/framework/tests/test_enablement.py

@@ -14,6 +14,7 @@
 from hyperloom.agents.framework.enablement import (
     ACCURACY_BELOW_FLOOR,
     CAPABILITY_DISABLED,
+    EVAL_GENERATION_PATHOLOGY,
     EVAL_RUNTIME_FAILURE,
     HIP_KERNEL_MISSING,
     IMPORT_ERROR,
@@ -47,6 +48,17 @@ def test_accuracy_below_floor_kind() -> None:
     assert sig.bridge_layer == ""
 
 
+def test_generation_pathology_outranks_accuracy_below_floor() -> None:
+    """The probe's evidence carries the below-floor phrasing too, but a truncated
+    eval is a different repair from a model that answered and got them wrong."""
+    sig = classify_failure(
+        "baseline accuracy did not meet floor: accuracy=0.0 floor=0.05 task=gsm8k; "
+        "eval_generation_pathology: 128/128 sampled responses stopped at the 16384-token cap"
+    )
+    assert sig.kind == EVAL_GENERATION_PATHOLOGY
+    assert ACCURACY_BELOW_FLOOR in sig.secondary_kinds
+
+
 def test_eval_runtime_failure_kind() -> None:
     sig = classify_failure("benchmark_stderr.log: ERROR: run_eval failed with exit code 1")
     assert sig.kind == EVAL_RUNTIME_FAILURE

src/hyperloom/inference_optimizer/breakdown/collectors/sessions.py

@@ -854,6 +854,7 @@ def collect_baseline(
                 "error_excerpt": a.get("error_excerpt"),
                 "stderr_tail": a.get("stderr_tail"),
                 "stderr_log_path": a.get("stderr_log_path"),
+                "extras": a.get("extras") or {},
             }
         )
 
@@ -1010,6 +1011,8 @@ def _reconstruct_baseline_attempts(
                 "error_excerpt": None,
                 "stderr_tail": None,
                 "stderr_log_path": None,
+                # Reconstruction reads the on-disk reports; audit extras are state-only.
+                "extras": {},
             }
         )
     return out

src/hyperloom/inference_optimizer/breakdown/schema.py

@@ -239,6 +239,9 @@ class BaselineAttemptSummary(TypedDict, total=False):
         key_metric (float | None): Headline metric value, or None if absent.
         workspace (str | None): Benchmark workspace path, or None.
         error_class (str | None): Error classification on failure, or None.
+        extras (dict[str, Any]): Attempt-specific fields from the writeback
+            audit: ``fingerprint``, ``anchor_kept_tput``, and ``eval_probe``
+            (why an accuracy of ~0 was a runaway generation, not wrong answers).
     """
 
     ts: str
@@ -252,6 +255,7 @@ class BaselineAttemptSummary(TypedDict, total=False):
     error_excerpt: str | None
     stderr_tail: str | None
     stderr_log_path: str | None
+    extras: dict[str, Any]
 
 
 class BenchmarkInvocation(TypedDict, total=False):

src/hyperloom/inference_optimizer/cli/backends.py

@@ -254,6 +254,7 @@ def _build_backends(
             model=claude_model,
             max_turns_default=4,
             conversational=True,
+            capture_turn_diagnostics=True,
         )
 
     backends: dict[str, Any] = {

src/hyperloom/inference_optimizer/reference_script.py

@@ -204,7 +204,7 @@ def parse_reference_script(source: str, *, framework: str) -> ReferenceRecipe:
 
 
 def _extract_envs(text: str) -> dict[str, str]:
-    """Pull whitelisted ``export KEY=VALUE`` lines whose value has no ``$``."""
+    """Pull whitelisted literal exports, including self-referential defaults."""
     envs: dict[str, str] = {}
     pat = re.compile(r"^\s*export\s+([A-Za-z_][A-Za-z0-9_]*)=(\S+)\s*$")
     for line in text.splitlines():
@@ -214,15 +214,35 @@ def _extract_envs(text: str) -> dict[str, str]:
         key, val = m.group(1), m.group(2)
         if key not in _ENV_WHITELIST:
             continue
-        if _has_var(val):
-            continue
         # strip surrounding quotes if present
         if len(val) >= 2 and val[0] == val[-1] and val[0] in ("'", '"'):
             val = val[1:-1]
+        if _has_var(val):
+            resolved = _resolve_self_default(key, val)
+            if resolved is None:
+                continue
+            val = resolved
         envs[key] = val
     return envs
 
 
+# ``${FOO:-1}`` / ``${FOO-1}``, capturing the name and the default.
+_SELF_DEFAULT_RE = re.compile(r"^\$\{([A-Za-z_][A-Za-z0-9_]*):?-(.*)\}$")
+
+
+def _resolve_self_default(key: str, val: str) -> str | None:
+    """Return the literal default of ``${key:-default}``, else ``None``.
+
+    Only the *self*-referential form counts: ``export FOO=${BAR:-1}`` depends on
+    an unrelated variable, so its default is not FOO's effective value here.
+    """
+    m = _SELF_DEFAULT_RE.match(val)
+    if not m or m.group(1) != key:
+        return None
+    default = m.group(2)
+    return None if _has_var(default) else default
+
+
 def _extract_server_args(
     tokens: list[str],
     framework: str,

src/hyperloom/inference_optimizer/session/session_paths.py

@@ -387,6 +387,11 @@ def conversations_path(session_dir: Path) -> Path:
     return trace_dir(session_dir) / "conversations.jsonl"
 
 
+def orchestration_turns_path(session_dir: Path) -> Path:
+    """``<sd>/reports/trace/orchestration_turns.jsonl``."""
+    return trace_dir(session_dir) / "orchestration_turns.jsonl"
+
+
 def research_hints_md(session_dir: Path) -> Path:
     """``<sd>/research_hints.md`` — human-readable proven-prior hints
     collected by the research scout.
@@ -457,6 +462,11 @@ def agent_prompt_snapshot(session_dir: Path, role: str) -> Path:
     return agent_dir(session_dir, role) / "system_prompt.snapshot.md"
 
 
+def agent_mcp_setup_path(session_dir: Path, role: str) -> Path:
+    """Compute the per-agent MCP setup snapshot path."""
+    return agent_dir(session_dir, role) / "mcp_setup.json"
+
+
 # External baseline comparison artefacts. Dedicated top-level subdir (not
 # runs/) because target_analysis is a prep-phase action.
 def target_analysis_dir(session_dir: Path) -> Path:
@@ -758,6 +768,7 @@ def allocate_turn_workdir(session_dir: Path, subdir: str, turn_idx: int, *, keep
 __all__ = [
     "allocate_turn_workdir",
     "agent_dir",
+    "agent_mcp_setup_path",
     "agent_prompt_snapshot",
     "breakdown_parts_dir",
     "competitor_target_json",
@@ -780,6 +791,7 @@ def allocate_turn_workdir(session_dir: Path, subdir: str, turn_idx: int, *, keep
     "kernel_agent_runs_root",
     "llm_calls_path",
     "manifest_path",
+    "orchestration_turns_path",
     "patches_dir",
     "reports_dir",
     "research_hints_json",

src/hyperloom/inference_optimizer/tests/test_baseline_eval_fallback.py

@@ -741,6 +741,7 @@ def fake_run(cmd, *args, **kwargs):
     BASELINE_EVAL_OBSERVED_ACCURACY_KEY,
     EVAL_KIND_ACCURACY_BELOW_FLOOR,
     EVAL_KIND_ACCURACY_UNAVAILABLE,
+    EVAL_KIND_GENERATION_PATHOLOGY,
 )
 
 
@@ -793,6 +794,32 @@ def test_eval_enablement_zero_accuracy_below_floor(monkeypatch):
     assert result[BASELINE_EVAL_OBSERVED_ACCURACY_KEY] == 0.0
 
 
+def test_eval_enablement_probe_reports_generation_pathology(monkeypatch):
+    """A tripped probe changes what a ~0 score means: the eval was cut short
+    because the model never stopped generating, not because it answered and got
+    them wrong. Without this the specialist is handed a bare ``accuracy=0.0``
+    and goes looking for a quality regression that never happened.
+    """
+    result = {
+        "status": "succeeded",
+        "accuracy": 0.0,
+        "run_eval_disabled": False,
+        "eval_probe": {
+            "kind": EVAL_KIND_GENERATION_PATHOLOGY,
+            "observed_samples": 128,
+            "cap_hits": 128,
+            "max_completion_tokens_seen": 16384,
+        },
+    }
+    reason = _route(monkeypatch, "sglang", result)
+    assert reason == ""
+    assert result[BASELINE_EVAL_FAILURE_KIND_KEY] == EVAL_KIND_GENERATION_PATHOLOGY
+    assert result[BASELINE_EVAL_OBSERVED_ACCURACY_KEY] == 0.0
+    evidence = result[BASELINE_EVAL_EVIDENCE_KEY]
+    assert "128/128" in evidence
+    assert "16384" in evidence
+
+
 def test_eval_enablement_positive_below_floor(monkeypatch):
     observed = DEFAULT_ENABLEMENT_ACCURACY_FLOOR / 2
     result = {"status": "succeeded", "accuracy": observed, "run_eval_disabled": False}

src/hyperloom/inference_optimizer/tests/test_claude_backend_branches_unit.py

@@ -246,10 +246,36 @@ async def test_run_conversational_session_capture(monkeypatch):
 # ---- run(): no-intent raises ----------------------------------------------
 async def test_run_no_intent_raises():
     msg = _Msg(content=[TextBlock("just text")], result="hi")
-    b = _backend()
+    b = _backend(capture_turn_diagnostics=True)
     b.sdk_query_factory = _query([msg])
     with pytest.raises(NoIntentEmitted):
         await b.run("hi")
+    diag = b.get_turn_diagnostic()
+    assert diag["outcome"] == "no_intent"
+    assert diag["raw_text"] == "hi"
+    assert diag["messages"] == [{"type": "_Msg", "result": "hi"}]
+
+
+async def test_run_skips_diagnostics_when_not_requested():
+    msg = _Msg(content=[TextBlock("just text")], result="hi")
+    b = _backend()
+    b.sdk_query_factory = _query([msg])
+    await b.run("hi", allow_no_intent=True)
+    assert b.get_turn_diagnostic() == {}
+
+
+# ---- gateway endpoint identifier -----------------------------------------
+def test_gateway_endpoint_drops_url_userinfo(monkeypatch):
+    """The diagnostic is appended to an on-disk trace, and a base URL of the
+    form ``https://user:key@gw/...`` puts the key in netloc."""
+    monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://user:s3cret@gw.example.com:8443/api/v1")
+    assert _backend()._gateway_endpoint_identifier() == "gw.example.com"
+
+
+def test_gateway_endpoint_is_none_without_a_base_url(monkeypatch):
+    for var in ("ANTHROPIC_BASE_URL", "DEEPSEEK_BASE_URL", "OPENAI_BASE_URL"):
+        monkeypatch.delenv(var, raising=False)
+    assert _backend()._gateway_endpoint_identifier() is None
 
 
 # ---- _invoke_and_collect: error-result-success tolerance ------------------

src/hyperloom/inference_optimizer/tests/test_coordinator_async_batch2_unit.py

@@ -7,6 +7,7 @@
 
 from __future__ import annotations
 
+import json
 from pathlib import Path
 from types import SimpleNamespace
 
@@ -69,6 +70,57 @@ def test_context_inbox_reader_empty(coord: Coordinator) -> None:
     assert out == "(no inbox events)"
 
 
+def test_trace_orchestration_turn_persists_diagnostics(coord: Coordinator) -> None:
+    backend = SimpleNamespace(
+        model="claude-test",
+        get_turn_diagnostic=lambda: {
+            "backend": "ClaudeBackend",
+            "model": "claude-test",
+            "sdk_name": "claude_agent_sdk",
+            "sdk_version": "1.2.3",
+            "resume_requested": True,
+            "previous_session_id_hash": "old",
+            "session_id_hash": "new",
+            "new_session": False,
+            "max_turns": 12,
+            "timeout_sec": 300.0,
+            "prompt": "prompt",
+            "system_prompt": "system",
+            "allowed_tools": ["mcp__inference_optimizer__emit_intent"],
+            "mcp_servers": ["inference_optimizer"],
+            "emit_intent_registered": True,
+            "messages": [{"type": "ResultMessage", "is_error": False, "result": "done"}],
+            "result": "done",
+            "raw_text": "done",
+            "tool_blocks": [],
+            "parse_errors": [],
+            "usage": {"input_tokens": 3},
+            "stderr_tail": [],
+        },
+        get_mcp_setup_diagnostic=lambda: {
+            "sdk_name": "claude_agent_sdk",
+            "emit_intent": {"registered": True},
+        },
+    )
+
+    coord._trace_orchestration_turn(
+        agent_name="orchestration",
+        backend=backend,
+        prompt="prompt",
+        system_prompt="system",
+        tools=["emit_intent"],
+        outcome="no_intent",
+        error=RuntimeError("missing intent"),
+    )
+
+    row = json.loads((coord.session_dir / "reports" / "trace" / "orchestration_turns.jsonl").read_text())
+    setup = json.loads((coord.session_dir / "agents" / "orchestration" / "mcp_setup.json").read_text())
+    assert row["outcome"] == "no_intent"
+    assert row["resume_requested"] is True
+    assert row["error_type"] == "RuntimeError"
+    assert setup["emit_intent"]["registered"] is True
+
+
 @pytest.mark.asyncio
 async def test_context_inbox_reader_with_events(coord: Coordinator) -> None:
     await coord.bus.append_and_seq(Message.new("kernel_agent", "orchestration", "heartbeat", {"body_md": "hi"}))