diff --git a/CLAUDE.md b/CLAUDE.md index 68f17bbc..b2c4001b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -243,7 +243,7 @@ include/naab/ All headers - BSD event types: `TOOL_CALL`, `TOOL_RESULT`, `TOOL_ERROR`, `TOOL_BLOCKED`. Default patterns include `tool_data_exfil`, `tool_env_harvest`, `tool_shell_escape`, `tool_rapid_fire` - Dashboard: `Tools: N calls (N blocked, Nms)` when tool execution occurred - Tool config fields are ratchet-enforced (can only tighten mid-run, never loosen) -- **`agent.extract_code(response, lang)`** — extract code from markdown fences. Searches for `` ```lang `` or `` ``` `` fences, prefers matching `lang` hint, returns longest block if multiple found. Strips surrounding conversational text. Returns input unchanged if no fence found. More powerful than the auto-strip applied by `agent.send()`. +- **`agent.extract_code(response, lang)`** — extract code from markdown fences. Scans ALL fenced blocks and prefers the longest whose language matches the `lang` hint; if none match (wrong tag or bare `` ``` `` with no language), falls back to the longest block of any language; returns input unchanged if no fence found. Bare no-language fences are handled. Strips surrounding conversational text. Pure function (no handle/governance/API). More powerful than the auto-strip applied by `agent.send()` (`stripMarkdownFences`, which only de-fences a whole-response single block). Test: `tests/governance_v4/test_extract_code.sh`. - **Codegen module** (`src/stdlib/codegen_impl.cpp`): `codegen.run(lang, code)` — governed dynamic code execution. Routes runtime-generated code through the same 39+ governance checks as static polyglot blocks. `codegen.run_with_args(lang, code, args)` — same with variable bindings. `codegen.run_strict(lang, code, args)` — throws `std::runtime_error` on non-zero exit code (catchable by NAAb `try/catch`). `codegen.supported_languages()` — list available languages. `codegen.is_enabled()` — check if codegen is enabled in govern.json. Config: `codegen` section in govern.json with per-call limits, cumulative limits, taint policy, nesting prevention. - **Orchestra module** (`src/stdlib/orchestra_impl.cpp`): multi-agent workflow building blocks. - `orchestra.sequential_refinement(handles, prompt [, iterations])` — returns a plan dict `{pattern, handles, prompt, iterations, description}`. NAAb code uses the plan to drive `agent.send()` loops. diff --git a/run-all-tests.sh b/run-all-tests.sh index 563e7588..e1010d1e 100755 --- a/run-all-tests.sh +++ b/run-all-tests.sh @@ -1530,6 +1530,19 @@ else echo " test_challenge_fail_path.sh: not found, skipping" fi +# agent.extract_code() fence extraction (pure function, no API) +EXTRACT_CODE_SCRIPT="tests/governance_v4/test_extract_code.sh" +if [ -f "$EXTRACT_CODE_SCRIPT" ]; then + if bash "$EXTRACT_CODE_SCRIPT" 2>&1; then + echo " test_extract_code.sh: ALL PASSED" + else + FAILED=$((FAILED + 1)) + FAILED_TESTS+=("test_extract_code.sh") + fi +else + echo " test_extract_code.sh: not found, skipping" +fi + # S23 response_degenerate + adaptive absorption cap + propose diversity (stub-backed) ABSORB_SCRIPT="tests/governance_v4/test_absorption_degenerate.sh" if [ -f "$ABSORB_SCRIPT" ]; then diff --git a/src/stdlib/agent_impl.cpp b/src/stdlib/agent_impl.cpp index 5c3e092f..455031ab 100644 --- a/src/stdlib/agent_impl.cpp +++ b/src/stdlib/agent_impl.cpp @@ -1143,74 +1143,87 @@ static NaabVal agentExtractCode(std::vector& args) { return NaabVal::makeString(""); } - // Search for code fences (```language ... ```) - auto fence_start = response.find("```"); - if (fence_start == std::string::npos) { - // No fence found — return response as-is - return NaabVal::makeString(response); - } + // Enumerate ALL fenced blocks, then select per the documented contract: + // "prefers matching lang hint, returns longest block if multiple found." + // (The old implementation examined only the first fence and, on a language + // mismatch, returned the entire response — so a wrong-tag or bare fence, or + // a leading non-target block, leaked prose + fences to the caller.) + struct FenceBlock { std::string lang; std::string body; }; + std::vector blocks; + + auto trim = [](std::string& s) { + while (!s.empty() && (s.back() == '\n' || s.back() == '\r' || + s.back() == ' ' || s.back() == '\t')) { + s.pop_back(); + } + while (!s.empty() && (s.front() == '\n' || s.front() == '\r' || + s.front() == ' ' || s.front() == '\t')) { + s.erase(s.begin()); + } + }; - // Find the end of the opening fence line (language tag) - auto line_end = response.find('\n', fence_start); - if (line_end == std::string::npos || line_end <= fence_start + 3) { - // No newline after fence or invalid fence format - return NaabVal::makeString(response); - } + size_t scan = 0; + while (true) { + auto fence_start = response.find("```", scan); + if (fence_start == std::string::npos) break; - // Extract the language tag (if any) - std::string fence_line = response.substr(fence_start + 3, line_end - fence_start - 3); - // Trim whitespace from fence line to get language - while (!fence_line.empty() && (fence_line.back() == ' ' || fence_line.back() == '\t' || - fence_line.back() == '\r' || fence_line.back() == '\n')) { - fence_line.pop_back(); - } - while (!fence_line.empty() && (fence_line.front() == ' ' || fence_line.front() == '\t')) { - fence_line.erase(fence_line.begin()); - } + // The opening fence must be followed by a newline (the language tag, if + // any, sits between ``` and that newline). A bare ```\n is valid — its + // newline lands exactly at fence_start+3, which the old `<=` guard wrongly + // rejected; require only that the newline exists at/after fence_start+3. + auto line_end = response.find('\n', fence_start); + if (line_end == std::string::npos) break; // dangling ``` with no newline + if (line_end < fence_start + 3) { scan = fence_start + 3; continue; } - // If lang_hint is provided and fence has a language, prefer matching - // If no lang_hint, use the fence's language (or empty for no language) - bool lang_matches = true; - if (!lang_hint.empty() && !fence_line.empty()) { - // Check if the fence language matches the hint (case-insensitive, substring) - std::string hint_lower = lang_hint; - std::string fence_lower = fence_line; - std::transform(hint_lower.begin(), hint_lower.end(), hint_lower.begin(), ::tolower); - std::transform(fence_lower.begin(), fence_lower.end(), fence_lower.begin(), ::tolower); - lang_matches = (fence_lower.find(hint_lower) != std::string::npos || - hint_lower.find(fence_lower) != std::string::npos); - } + // Closing fence is the next ``` after the opening fence line. + auto fence_end = response.find("```", line_end + 1); + if (fence_end == std::string::npos) break; // no closing fence — stop - if (!lang_matches && !lang_hint.empty()) { - // Fence language doesn't match hint — skip this fence and look for next - // This is a more sophisticated version that can handle multiple fences - // For now, fall back to returning the response as-is if lang doesn't match - return NaabVal::makeString(response); + std::string lang = response.substr(fence_start + 3, line_end - fence_start - 3); + trim(lang); + std::string body = response.substr(line_end + 1, fence_end - line_end - 1); + trim(body); + blocks.push_back({lang, body}); + + scan = fence_end + 3; // resume after the closing fence } - // Find the closing fence - auto fence_end = response.find("```", line_end); - if (fence_end == std::string::npos || fence_end <= line_end) { - // No closing fence or malformed + if (blocks.empty()) { + // No fenced block found — return the response unchanged. Preserves the + // contract and the agent.send() auto-strip → extract_code chain, where + // a clean single block has already been de-fenced upstream. return NaabVal::makeString(response); } - // Extract code between opening and closing fence lines - std::string extracted = response.substr(line_end + 1, fence_end - line_end - 1); + // Case-insensitive bidirectional-substring language match (unchanged from + // the original single-fence logic). + auto lang_matches = [](const std::string& fence_lang, const std::string& hint) -> bool { + if (hint.empty() || fence_lang.empty()) return false; + std::string h = hint, f = fence_lang; + std::transform(h.begin(), h.end(), h.begin(), ::tolower); + std::transform(f.begin(), f.end(), f.begin(), ::tolower); + return f.find(h) != std::string::npos || h.find(f) != std::string::npos; + }; - // Strip trailing whitespace/newlines from extracted code - while (!extracted.empty() && (extracted.back() == '\n' || extracted.back() == '\r' || - extracted.back() == ' ' || extracted.back() == '\t')) { - extracted.pop_back(); + // Prefer the longest block whose language matches the hint; if none match + // (or no hint given), fall back to the longest block of any language — a + // model that mislabeled or omitted the tag still yields usable code. + const FenceBlock* best = nullptr; + if (!lang_hint.empty()) { + for (const auto& b : blocks) { + if (lang_matches(b.lang, lang_hint) && + (!best || b.body.size() > best->body.size())) { + best = &b; + } + } } - - // Strip leading whitespace/newlines from extracted code - while (!extracted.empty() && (extracted.front() == '\n' || extracted.front() == '\r' || - extracted.front() == ' ' || extracted.front() == '\t')) { - extracted.erase(extracted.begin()); + if (!best) { + for (const auto& b : blocks) { + if (!best || b.body.size() > best->body.size()) best = &b; + } } - return NaabVal::makeString(extracted); + return NaabVal::makeString(best->body); } // ============================================================================ diff --git a/tests/governance_v4/test_extract_code.sh b/tests/governance_v4/test_extract_code.sh new file mode 100755 index 00000000..f5d3a4e1 --- /dev/null +++ b/tests/governance_v4/test_extract_code.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# ============================================================ +# test_extract_code.sh — agent.extract_code() fence extraction +# +# agent.extract_code is a PURE function (no handle, no governance, no API): +# it takes a string + optional language hint and returns the extracted code. +# So this test drives it directly on literal strings with sentinel bodies — +# no stub, no keys — and asserts the documented contract: +# "prefers matching lang hint, returns longest block if multiple found; +# strips surrounding conversational text; returns input unchanged if no +# fence found." +# +# Regression targets (all failed before the multi-fence rewrite): +# T3 — a non-target fenced block BEFORE the target one (old: whole response) +# T4 — a bare ``` fence with no language tag (old: guard rejected it) +# T5 — multiple matching blocks (old: only the first) +# ============================================================ +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +NAAB="$SCRIPT_DIR/../../build/naab-lang" + +if [ -d "/data/data/com.termux/files/usr/tmp" ]; then + _SYSTMP="${TMPDIR:-/data/data/com.termux/files/usr/tmp}" +else + _SYSTMP="${TMPDIR:-/tmp}" +fi +TEST_TMP="${_SYSTMP}/extract-code-$$" + +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; CYAN='\033[0;36m'; NC='\033[0m' +PASS_COUNT=0; FAIL_COUNT=0; SKIP_COUNT=0; FAILURES="" + +pass() { PASS_COUNT=$((PASS_COUNT + 1)); echo -e " ${GREEN}PASS${NC} [$1] $2"; } +fail() { FAIL_COUNT=$((FAIL_COUNT + 1)); echo -e " ${RED}FAIL${NC} [$1] $2"; [ -n "${3:-}" ] && echo -e " ${RED}-> $3${NC}"; FAILURES="${FAILURES}\n [$1] $2"; } +skip() { SKIP_COUNT=$((SKIP_COUNT + 1)); echo -e " ${YELLOW}SKIP${NC} [$1] $2"; } + +cleanup() { rm -rf "$TEST_TMP"; } +trap cleanup EXIT +mkdir -p "$TEST_TMP" + +# Single-line sentinel bodies so each extracted result is exactly one output +# line — easy to assert, and any leaked fence/prose is immediately visible. +# Quoted heredoc: backticks and \n pass through literally; NAAb's lexer turns +# \n into newlines inside the string literals. +cat > "$TEST_TMP/t.naab" <<'NAABEOF' +use agent +main { + // T1: single python block + print("T1|" + agent.extract_code("```python\nPYBODY1\n```", "python")) + // T2: prose around a python block + print("T2|" + agent.extract_code("Here is the code:\n```python\nPYBODY2\n```\nHope it helps.", "python")) + // T3: a non-target (text) block BEFORE the target python block + print("T3|" + agent.extract_code("Notes:\n```text\nTEXTBODY\n```\n```python\nPYBODY3\n```", "python")) + // T4: a BARE fence (no language tag) with surrounding prose + print("T4|" + agent.extract_code("Here:\n```\nBAREBODY\n```", "python")) + // T5: two matching blocks of different length — expect the longer + print("T5|" + agent.extract_code("```python\nSHORT\n```\n```python\nLONGER_BODY_HERE\n```", "python")) + // T6: json hint selects the json block, not the python one + print("T6|" + agent.extract_code("```json\nJSONBODY\n```\n```python\nPYBODY6\n```", "json")) + // T7: hint matches no block; one untagged block — fallback to longest + print("T7|" + agent.extract_code("```\nUNTAGGED\n```", "python")) + // T8: no fence at all — return input unchanged + print("T8|" + agent.extract_code("just plain text no fences", "python")) +} +NAABEOF + +echo "" +echo -e "${CYAN}+==============================================================+${NC}" +echo -e "${CYAN}| agent.extract_code() fence extraction (pure, no API) |${NC}" +echo -e "${CYAN}+==============================================================+${NC}" +echo "" + +# extract_code needs no governance; run without it. Fall back to a minimal +# inline govern.json only if the agent module refuses to load unconfigured. +OUT=$(cd "$TEST_TMP" && timeout 30s "$NAAB" --no-governance t.naab 2>&1) || true +if ! echo "$OUT" | grep -q "^T1|"; then + cat > "$TEST_TMP/govern.json" <<'GEOF' +{ "version": "5.0", "mode": "advisory", "security": { "sandbox_level": "elevated" } } +GEOF + OUT=$(cd "$TEST_TMP" && timeout 30s "$NAAB" t.naab 2>&1) || true +fi + +if ! echo "$OUT" | grep -q "^T1|"; then + fail "T0" "extract_code test harness did not run" "$(echo "$OUT" | head -4)" +else + pass "T0" "test program executed" + + check() { # $1=id $2=expected exact value after the pipe $3=desc + local got; got=$(echo "$OUT" | grep "^$1|" | head -1 | sed "s/^$1|//") + if [ "$got" = "$2" ]; then + pass "$1" "$3" + else + fail "$1" "$3" "expected [$2] got [$got]" + fi + } + + check "T1" "PYBODY1" "single python block extracted" + check "T2" "PYBODY2" "prose stripped around python block" + check "T3" "PYBODY3" "target block found past a leading non-target fence" + check "T4" "BAREBODY" "bare (no-language) fence extracted" + check "T5" "LONGER_BODY_HERE" "longest of multiple matching blocks" + check "T6" "JSONBODY" "json hint selects the json block" + check "T7" "UNTAGGED" "fallback to longest block when hint matches none" + check "T8" "just plain text no fences" "no fence — input returned unchanged" + + # Explicit anti-leak assertions for the two headline bugs. + if echo "$OUT" | grep "^T3|" | grep -q "TEXTBODY\|\`\`\`"; then + fail "T3-leak" "T3 leaked the non-target block or fence markers" + else + pass "T3-leak" "T3 output free of the text block and fence markers" + fi + if echo "$OUT" | grep "^T4|" | grep -q "Here:\|\`\`\`"; then + fail "T4-leak" "T4 leaked prose or fence markers" + else + pass "T4-leak" "T4 output free of prose and fence markers" + fi +fi + +echo "" +echo -e "${CYAN}+==============================================================+${NC}" +TOTAL=$((PASS_COUNT + FAIL_COUNT + SKIP_COUNT)) +echo -e " Total: $TOTAL | ${GREEN}Pass: $PASS_COUNT${NC} | ${RED}Fail: $FAIL_COUNT${NC} | ${YELLOW}Skip: $SKIP_COUNT${NC}" +if [ "$FAIL_COUNT" -gt 0 ]; then echo -e "${RED}Failures:${NC}$FAILURES"; exit 1; fi +exit 0