fix(cyber): grade the pentest on the breach + make the single-tool loop work with real models

packages/openrange-pack-sdk/src/openrange_pack_sdk/_runtime.py

@@ -100,9 +100,21 @@ def surface(self) -> Mapping[str, Any]:
             raise OpenRangeError("surface() called before reset()")
         return {
             "solver_root": str(self._solver_root),
+            "submit": self.submit,
             **self.surface_extras(),
         }
 
+    def submit(self, content: str) -> None:
+        """Record the agent's final answer where ``collect`` reads it.
+
+        The submit channel for the single-tool agent loop. The runtime owns the
+        path, so it works whether the agent ran on the host or inside a sandbox
+        that can't reach ``solver_root`` — the harness routes the answer here, the
+        agent never writes the file itself."""
+        if self._solver_root is None:
+            raise OpenRangeError("submit() called before reset()")
+        (self._solver_root / self.RESULT_FILE).write_text(content, encoding="utf-8")
+
     def terminal(self) -> tuple[bool, str | None]:
         if self._solver_root is None:
             return False, None

packs/cyber_webapp/cyber_webapp/families/pentest.py

@@ -142,25 +142,28 @@ def check_success(
             return EpisodeResult(success=False, reason="goal not in graph")
         expected = flag.attrs.get("value_ref")
         submitted = final_state.get("flag_from_response")
-        ok = bool(expected) and submitted == expected
-        # Whether the flag actually crossed into a response is surfaced on final_state
-        # and in the reason for observability; success stays submission-based, so the
-        # averaged subgoals — and the reward rungs derived from them — are unchanged.
+        matched = bool(expected) and submitted == expected
         secret_leaked = task.goal_nodes[0] in set(
             final_state.get("leaked_secret_ids") or ()
         )
-        if ok:
+        # Success is the breach itself — the guarded flag crossing the wire, which
+        # admission guarantees a benign request can't trigger. That makes the
+        # consequence verifier the grader, so a pure-shell agent needs no submit
+        # step. An exact submission still counts, so a submit-tool harness grades
+        # identically.
+        ok = secret_leaked or matched
+        if secret_leaked:
+            reason = "flag leaked in a response (breach confirmed)"
+        elif matched:
             reason = "flag matched"
         else:
-            reason = f"expected {expected!r}, got {submitted!r}"
-            if secret_leaked:
-                reason += "; flag leaked in a response (consequence observed)"
+            reason = f"no breach: expected {expected!r}, got {submitted!r}"
         return EpisodeResult(
             success=ok,
             subgoals={
                 "reached_endpoint": bool(final_state.get("requests_made")),
                 "extracted_anything": bool(final_state.get("flag_from_response")),
-                "matched_flag": ok,
+                "matched_flag": matched,
             },
             reason=reason,
         )

packs/cyber_webapp/cyber_webapp/realize.py

@@ -100,12 +100,7 @@ def poll_events(self) -> tuple[Mapping[str, Any], ...]:
         return tuple(events)
 
     def collect_extras(self) -> Mapping[str, Any]:
-        result = self._read_result()
-        flag = ""
-        if isinstance(result.get("flag"), str):
-            flag = str(result["flag"])
-        elif isinstance(result.get("flag_from_response"), str):
-            flag = str(result["flag_from_response"])
+        flag = self._submitted_flag()
         requests = self._all_requests()
         requests_made = [str(row.get("path", "")) for row in requests if row]
         leaked: set[str] = set()
@@ -120,6 +115,21 @@ def collect_extras(self) -> Mapping[str, Any]:
             "endpoint_serves_200": self._probe_root_200(),
         }
 
+    def _submitted_flag(self) -> str:
+        result = self._read_result()
+        for key in ("flag", "flag_from_response"):
+            value = result.get(key)
+            if isinstance(value, str) and value:
+                return value
+        # A single-tool agent submits a bare flag, not a {"flag": ...} envelope;
+        # accept the raw result file when it isn't that JSON object.
+        assert self._solver_root is not None
+        try:
+            raw = (self._solver_root / self.RESULT_FILE).read_text("utf-8").strip()
+        except OSError:
+            return ""
+        return raw if not raw.startswith("{") else ""
+
     def checkpoint(self) -> Any:
         state = super().checkpoint()
         return {**state, "log_offset": self._log_offset}

src/openrange/agent.py

@@ -135,26 +135,41 @@ def agent_briefing(ctx: EpisodeContext) -> str:
 _ACTION_BLOCK = re.compile(
     r"```(bash|sh|shell|console|zsh|finish)[ \t]*\n(.*?)```", re.DOTALL
 )
+# A function-calling model often emits its native tool-call form instead of a
+# fenced block — ``<function=bash><parameter=command>...</parameter>`` (Qwen /
+# Nemotron / Hermes). Read it as the action rather than a give-up ``finish``.
+_XML_CALL = re.compile(
+    r"<function\s*=\s*([\w.]+)\s*>\s*<parameter\s*=\s*[\w.]+\s*>"
+    r"\n?(.*?)\n?\s*</parameter>",
+    re.DOTALL,
+)
+_FINISH_FNS = {"finish", "submit", "answer", "done"}
 
 
 def parse_action(text: str) -> AgentAction:
     """Parse the agent's action from its reply: a fenced shell command
-    (```bash / ```sh / ```shell / ```console / ```zsh) or a ```finish``` block.
-
-    Shell actions use the standard markdown code fence a model is trained to
-    emit — executable code as the action, CodeAct-style — rather than a bespoke
-    tool token it only ever sees in our prompt and routinely forgets. The *last*
-    recognized block wins, so an illustrative snippet earlier in the reply is not
-    executed in place of the action the model actually settled on. A reply with
-    no recognized block becomes a ``finish`` carrying the whole text, so a model
-    that ignores the protocol terminates rather than loops."""
-    matches = list(_ACTION_BLOCK.finditer(text))
-    if not matches:
+    (```bash / ```sh / ```shell / ```console / ```zsh), a ```finish``` block, or
+    the native ``<function=...><parameter=...>`` tool call a function-calling
+    model emits.
+
+    Shell actions are CodeAct-style — executable code as the action — so the
+    standard markdown fence a model is trained to emit doubles as the action
+    token. A model with strong native function-calling reverts to its XML call
+    instead, so that form is honoured too. The *last* recognized action wins, so
+    an illustrative snippet earlier in the reply is not run in place of the action
+    the model actually settled on. A reply with no recognized action becomes a
+    ``finish`` carrying the whole text, so a model that ignores the protocol
+    terminates rather than loops."""
+    best: tuple[int, str, str] | None = None
+    for match in _ACTION_BLOCK.finditer(text):
+        best = (match.start(), match.group(1).lower(), match.group(2))
+    for match in _XML_CALL.finditer(text):
+        if best is None or match.start() > best[0]:
+            best = (match.start(), match.group(1).lower(), match.group(2))
+    if best is None:
         return AgentAction(tool="finish", command=text.strip())
-    match = matches[-1]
-    lang = match.group(1).lower()
-    tool = "finish" if lang == "finish" else "run_shell"
-    return AgentAction(tool=tool, command=match.group(2).strip())
+    tool = "finish" if best[1] in _FINISH_FNS else "run_shell"
+    return AgentAction(tool=tool, command=best[2].strip())
 
 
 def run_shell(
@@ -212,6 +227,9 @@ async def arun_agent(
             )
             action = parse_action(sample.text)
             if action.tool == "finish":
+                submit = bound.get("submit")
+                if callable(submit) and action.command:
+                    await asyncio.to_thread(submit, action.command)
                 turn = AgentTurn(
                     message=action.command or sample.text,
                     tool_calls=(

tests/test_agent_harness.py

@@ -166,7 +166,88 @@ def test_real_exploit_is_run_and_the_leak_is_observed(tmp_path: Path) -> None:
     assert rollout.steps[-1].command is None
     leaked = rollout.report.final_state.get("leaked_secret_ids") or ()
     assert "secret_flag" in leaked
-    assert isinstance(rollout.success, bool)
+    # The breach leaked the flag, but "recovered" is not the flag value, so the
+    # submission-based grade is a miss — the leak is observed independently.
+    assert rollout.success is False
+
+
+def _finish_with(answer: str) -> Reply:
+    return lambda _prompt: f"```finish\n{answer}\n```"
+
+
+def test_submitting_the_recovered_flag_passes(tmp_path: Path) -> None:
+    # The single-tool loop's submit channel: a finish block carrying the flag is
+    # routed to the result file the grader reads, so a real breach can pass.
+    snap = _cmdi_snapshot()
+    flag = str(snap.graph.nodes["secret_flag"].attrs["value_ref"])
+    svc = EpisodeService(WebappPack(), tmp_path)
+    try:
+        rollout = run_agent(
+            svc,
+            snap,
+            _Policy(_exploit_reply(snap), _finish_with(flag)),
+            bind_run=_host_bind,
+            task_id=_pentest_task_id(snap),
+        )
+    finally:
+        svc.close()
+    assert rollout.success is True
+    assert rollout.reward.scalar == 1.0
+    assert rollout.report.final_state["flag_from_response"] == flag
+
+
+def test_submitting_the_flag_as_a_json_envelope_passes(tmp_path: Path) -> None:
+    snap = _cmdi_snapshot()
+    flag = str(snap.graph.nodes["secret_flag"].attrs["value_ref"])
+    svc = EpisodeService(WebappPack(), tmp_path)
+    try:
+        rollout = run_agent(
+            svc,
+            snap,
+            _Policy(_exploit_reply(snap), _finish_with(f'{{"flag": "{flag}"}}')),
+            bind_run=_host_bind,
+            task_id=_pentest_task_id(snap),
+        )
+    finally:
+        svc.close()
+    assert rollout.success is True
+
+
+def test_submitting_a_non_flag_json_object_is_a_miss(tmp_path: Path) -> None:
+    # A JSON object without a flag field is not a bare-flag submission; the grader
+    # records no flag rather than treating the envelope text as the answer.
+    snap = _cmdi_snapshot()
+    svc = EpisodeService(WebappPack(), tmp_path)
+    try:
+        rollout = run_agent(
+            svc,
+            snap,
+            _Policy(_exploit_reply(snap), _finish_with('{"note": "no flag here"}')),
+            bind_run=_host_bind,
+            task_id=_pentest_task_id(snap),
+        )
+    finally:
+        svc.close()
+    assert rollout.report.final_state.get("flag_from_response") is None
+    assert rollout.success is False
+
+
+def test_finish_without_an_answer_submits_nothing(tmp_path: Path) -> None:
+    snap = _cmdi_snapshot()
+    svc = EpisodeService(WebappPack(), tmp_path)
+    try:
+        rollout = run_agent(
+            svc,
+            snap,
+            _Fixed("```finish\n\n```"),
+            bind_run=_host_bind,
+            task_id=_pentest_task_id(snap),
+        )
+    finally:
+        svc.close()
+    assert rollout.terminal_reason == "finished"
+    assert rollout.report.final_state.get("flag_from_response") is None
+    assert rollout.success is False
 
 
 def test_max_turns_caps_the_loop_and_carries_train_signal(tmp_path: Path) -> None:
@@ -277,6 +358,44 @@ def test_parse_action_accepts_standard_shell_fences_and_takes_the_last() -> None
     assert settled.command == "echo second"
 
 
+def test_parse_action_reads_the_native_tool_call_form() -> None:
+    # A function-calling model reverts to its native XML tool call under the
+    # fence instruction; the harness honours it rather than reading a give-up.
+    shell = parse_action(
+        "<tool_call>\n<function=bash>\n<parameter=command>\n"
+        "curl -s http://x/\n</parameter>\n</function>\n</tool_call>"
+    )
+    assert shell.tool == "run_shell"
+    assert shell.command == "curl -s http://x/"
+    done = parse_action(
+        "<function=submit><parameter=answer>THEFLAG</parameter></function>"
+    )
+    assert done.tool == "finish"
+    assert done.command == "THEFLAG"
+    # The last action wins across both the fence and the XML form, whichever ends
+    # the reply — an XML call then a fence, or a fence then an XML call.
+    fence_last = parse_action(
+        "<function=bash><parameter=command>echo first</parameter></function>\n"
+        "```bash\necho last\n```"
+    )
+    assert fence_last.command == "echo last"
+    xml_last = parse_action(
+        "```bash\necho first\n```\n"
+        "<function=bash><parameter=command>echo last</parameter></function>"
+    )
+    assert xml_last.tool == "run_shell"
+    assert xml_last.command == "echo last"
+
+
+def test_submit_before_reset_is_rejected() -> None:
+    from cyber_webapp.realize import WebappRuntime
+    from openrange_pack_sdk import OpenRangeError
+
+    runtime = WebappRuntime(_cmdi_snapshot().graph)
+    with pytest.raises(OpenRangeError, match="submit"):
+        runtime.submit("anything")
+
+
 def test_run_shell_requires_a_bound_run_capability() -> None:
     with pytest.raises(AgentError, match="run"):
         run_shell({}, "echo hi")

tests/test_cyber_staged_generation.py

@@ -487,21 +487,24 @@ def test_live_episode_records_the_flag_leak_as_a_node_id(tmp_path: Path) -> None
     assert "secret_flag" in leaked
 
 
-def test_check_success_surfaces_leak_without_changing_the_reward_rungs() -> None:
+def test_check_success_grades_on_the_leak_not_only_a_submission() -> None:
     from cyber_webapp.families.pentest import WebappPentest
 
     snap = _admit("db", vuln={"pin": [{"kind": "sql_injection"}]})
     pentest = next(t for t in snap.tasks if t.meta.get("family") == "webapp.pentest")
     fam = WebappPentest()
+    # A wrong (or absent) submission, but the guarded flag crossed the wire.
     base = {"requests_made": ["/svc"], "flag_from_response": "wrong"}
 
     leaked = fam.check_success(
         snap.graph, pentest, {**base, "leaked_secret_ids": ["secret_flag"]}
     )
     quiet = fam.check_success(snap.graph, pentest, base)
+    # The breach itself is success — no submission needed; without a leak it fails.
+    assert leaked.success is True and quiet.success is False
     assert "leaked" in leaked.reason and "leaked" not in quiet.reason
-    assert leaked.success is False and quiet.success is False
-    # The averaged subgoal set is unchanged, so the trainer's rungs are intact.
+    # The strict-submission subgoal stays a miss even though the breach passed.
+    assert leaked.subgoals["matched_flag"] is False
     assert set(leaked.subgoals) == {
         "reached_endpoint",
         "extracted_anything",