diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index cfa62ada..bdd05151 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -4,7 +4,6 @@ on: push: branches: [main] pull_request: - branches: [main] schedule: # Weekly deeper run (longer per-target budget via FUZZ_SECONDS). - cron: "41 4 * * 2" @@ -26,6 +25,7 @@ jobs: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} persist-credentials: false - name: Set up Python @@ -50,6 +50,7 @@ jobs: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} persist-credentials: false - name: Set up Python @@ -83,6 +84,9 @@ jobs: - name: Fuzz orchestration engine run: python fuzz/fuzz_orchestration.py -max_total_time=${FUZZ_SECONDS} -artifact_prefix=crash- fuzz/corpus/orchestration + - name: Fuzz NIM model-catalog parser + run: python fuzz/fuzz_nim_catalog.py -max_total_time=${FUZZ_SECONDS} -artifact_prefix=crash- fuzz/corpus/nim_catalog + - name: Upload crash artifacts if: failure() uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # actions/upload-artifact@v5 diff --git a/.github/workflows/nim-benchmark.yml b/.github/workflows/nim-benchmark.yml new file mode 100644 index 00000000..b30f515c --- /dev/null +++ b/.github/workflows/nim-benchmark.yml @@ -0,0 +1,149 @@ +name: NIM benchmark + +# Evidence-grade NVIDIA NIM model discovery and cost-quality benchmark. +# Manual dispatch defaults to a deterministic dry run. Monthly scheduled runs +# are live but use a conservative hard request cap. Dry execution receives no +# provider credential; only the live step can read NVIDIA_NIM_API_KEY. + +on: + workflow_dispatch: + inputs: + dry_run: + description: "Dry run without contacting NVIDIA" + type: boolean + default: true + max_total_requests: + description: "Hard cap on provider requests for this run" + type: number + default: 2000 + pricing_scenario: + description: "Optional reviewed pricing-scenario JSON path" + type: string + default: "" + schedule: + - cron: "23 3 5 * *" + +permissions: + contents: read + +concurrency: + group: nim-benchmark + cancel-in-progress: false + +jobs: + dry_run_benchmark: + name: Deterministic NIM benchmark dry run + if: github.event_name == 'workflow_dispatch' && inputs.dry_run == true + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install pinned runtime + run: | + python -m pip install --require-hashes -r requirements.lock + python -m pip install --no-deps -e . + + - name: Run dry benchmark + env: + MAX_REQUESTS: ${{ inputs.max_total_requests }} + PRICING_SCENARIO: ${{ inputs.pricing_scenario }} + PROVENANCE_GIT_SHA: ${{ github.sha }} + PROVENANCE_RUN_ID: ${{ github.run_id }} + run: | + set -euo pipefail + extra_args=() + if [ -n "$PRICING_SCENARIO" ]; then + extra_args+=(--pricing-scenario "$PRICING_SCENARIO") + fi + python -m contextual_orchestrator nim-benchmark \ + --dry-run \ + "${extra_args[@]}" \ + --task-manifest examples/nim_task_manifest.json \ + --output-dir benchmark_artifacts \ + --max-total-requests "$MAX_REQUESTS" \ + --git-sha "$PROVENANCE_GIT_SHA" \ + --workflow-run-id "$PROVENANCE_RUN_ID" + + - name: Upload dry-run artifacts + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # actions/upload-artifact@v5 + with: + name: nim-benchmark-dry-${{ github.run_id }} + path: benchmark_artifacts/ + retention-days: 30 + if-no-files-found: error + + live_benchmark: + name: Live NIM catalog benchmark + if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.dry_run != true) + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install pinned runtime + run: | + python -m pip install --require-hashes -r requirements.lock + python -m pip install --no-deps -e . + + - name: Resolve live parameters + id: live_params + env: + EVENT_NAME: ${{ github.event_name }} + INPUT_MAX_REQUESTS: ${{ inputs.max_total_requests }} + INPUT_PRICING: ${{ inputs.pricing_scenario }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" = "schedule" ]; then + echo "max_requests=2000" >> "$GITHUB_OUTPUT" + echo "pricing_scenario=" >> "$GITHUB_OUTPUT" + else + echo "max_requests=${INPUT_MAX_REQUESTS}" >> "$GITHUB_OUTPUT" + echo "pricing_scenario=${INPUT_PRICING}" >> "$GITHUB_OUTPUT" + fi + + - name: Run live benchmark + env: + NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} + MAX_REQUESTS: ${{ steps.live_params.outputs.max_requests }} + PRICING_SCENARIO: ${{ steps.live_params.outputs.pricing_scenario }} + PROVENANCE_GIT_SHA: ${{ github.sha }} + PROVENANCE_RUN_ID: ${{ github.run_id }} + run: | + set -euo pipefail + extra_args=() + if [ -n "$PRICING_SCENARIO" ]; then + extra_args+=(--pricing-scenario "$PRICING_SCENARIO") + fi + python -m contextual_orchestrator nim-benchmark \ + "${extra_args[@]}" \ + --task-manifest examples/nim_task_manifest.json \ + --output-dir benchmark_artifacts \ + --max-total-requests "$MAX_REQUESTS" \ + --git-sha "$PROVENANCE_GIT_SHA" \ + --workflow-run-id "$PROVENANCE_RUN_ID" + + - name: Upload live benchmark artifacts + if: always() + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # actions/upload-artifact@v5 + with: + name: nim-benchmark-live-${{ github.run_id }} + path: benchmark_artifacts/ + retention-days: 90 + if-no-files-found: error diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 631503e1..fa76a22e 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -9,7 +9,6 @@ on: push: branches: [main] pull_request: - branches: [main] schedule: - cron: "17 3 * * 1" workflow_dispatch: @@ -34,6 +33,7 @@ jobs: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} persist-credentials: false - name: Initialize CodeQL @@ -54,6 +54,7 @@ jobs: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} persist-credentials: false - name: Set up Python diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 02605dd7..6d001bca 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,7 +4,6 @@ on: push: branches: [main] pull_request: - branches: [main] permissions: contents: read @@ -21,6 +20,7 @@ jobs: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} persist-credentials: false - name: Set up Python @@ -36,3 +36,63 @@ jobs: - name: Run full test suite run: python -m pytest -q + + nim_benchmark_quality: + name: NIM benchmark coverage, docstrings, and package smoke + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install hash-locked quality tools + run: python -m pip install --require-hashes -r requirements-opencode-review-ci.txt + + - name: Prove complete benchmark coverage and public docstrings + run: | + set -euo pipefail + python -m coverage erase + python -m coverage run --branch \ + --source=contextual_orchestrator.nim_benchmark,contextual_orchestrator.nim_csv_evidence,contextual_orchestrator.nim_strict_scoring \ + -m pytest \ + tests/test_nim_benchmark.py \ + tests/test_nim_benchmark_budget_view.py \ + tests/test_nim_benchmark_release_acceptance.py \ + tests/test_nim_benchmark_review_regressions.py \ + tests/test_nim_benchmark_workflow_contract.py \ + tests/test_nim_artifact_publication.py \ + tests/test_nim_artifact_publication_edges.py \ + tests/test_nim_assignment_step_ids.py \ + tests/test_nim_csv_evidence.py \ + tests/test_nim_csv_evidence_edges.py \ + tests/test_nim_strict_scorer_validity.py \ + tests/test_nim_strict_scoring_bounds.py \ + tests/test_nim_strict_scoring_integration.py \ + tests/test_nim_strict_scoring_leakage.py \ + -q + python -m coverage report \ + --include=contextual_orchestrator/nim_benchmark.py,contextual_orchestrator/nim_csv_evidence.py,contextual_orchestrator/nim_strict_scoring.py \ + --show-missing \ + --fail-under=100 + python -m interrogate -f 100 contextual_orchestrator/nim_benchmark.py + python -m interrogate -f 100 contextual_orchestrator/nim_csv_evidence.py + python -m interrogate -f 100 contextual_orchestrator/nim_strict_scoring.py + + - name: Build, install, and import the wheel + run: | + set -euo pipefail + rm -rf dist "$RUNNER_TEMP/nim-wheel-site" + python -m pip wheel --no-deps . --wheel-dir dist + python -m pip install --no-deps \ + --target "$RUNNER_TEMP/nim-wheel-site" \ + dist/contextual_orchestrator-*.whl + cd "$RUNNER_TEMP" + PYTHONPATH="$RUNNER_TEMP/nim-wheel-site" \ + python -c "import contextual_orchestrator; import contextual_orchestrator.nim_benchmark; import contextual_orchestrator.nim_csv_evidence; import contextual_orchestrator.nim_strict_scoring" \ No newline at end of file diff --git a/.gitignore b/.gitignore index 131b454a..b8f5e84b 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,9 @@ tempcred.txt # hypothesis fuzzing DB .hypothesis/ + +# coverage measurement data +.coverage + +# local benchmark artifacts (uploaded by CI, not committed) +benchmark_artifacts/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e7c96c6..6b4c84dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,19 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and ## [Unreleased] +### Added + +- Add an optional provider-neutral NVIDIA NIM benchmark harness that dynamically discovers the live `/v1/models` catalog, probes every discovered model under bounded concurrency and a complete hard request plan, and compares every eligible direct model, route-once, bounded-conduct, and reviewed pricing-scenario policy under equal total token and call envelopes. +- Add deterministic no-egress dry runs, valid media fixtures, secret-redacted transactional JSON/CSV/Markdown artifacts, complete role/agent/model assignment evidence, paired bootstrap uncertainty, evidence sufficiency, and quality-latency and reviewed-hypothetical-cost Pareto frontiers. +- Add validation-time public-address-pinned provider transport, original-host authority/SNI/certificate verification, redirect and ambient-proxy rejection, credential-forwarding prevention, 8 MiB response bounds, and live-secret isolation to the NIM benchmark path. +- Add versioned strict complete-answer scoring for locked tasks: finite full-response decimal comparison, NFC-normalized declared text alternatives with task-specific case semantics, explicit alias and prompt-disambiguation evidence, bounded value and prompt inputs, decimal-equivalent and declared-alias prompt-leakage rejection, complete token boundaries that avoid larger-word false positives, zero-score handling for oversized or unrepresentable model output, fail-before-egress manifest validation, derived-manifest provenance, and lazy optional-adapter activation. +- Add permanent benchmark contracts for 100% production statement/branch coverage, 100% public docstrings, fuzzing, package build/install/import, exact contributor-head workflows, evidence-status semantics, and release acceptance without automatic routing authorization. + ### Security - Restrict the private plain-HTTP provider seam to `localhost` or literal loopback IP addresses, reject URL userinfo before connection, dial directly without ambient proxy lookup, reject all redirect responses, and close failed resources deterministically. - Pin each HTTPS provider connection to the exact public addresses approved during validation, preserve the original hostname for TLS verification, bypass environment proxy resolution, and reject redirects to close DNS-rebinding and credential-forwarding SSRF paths. +- Bound every provider response to 8 MiB of cumulative consumed bytes, including SSE iteration, reject oversized declared lengths before body consumption, fail closed on malformed or conflicting `Content-Length` and ambiguous `Content-Length` plus `Transfer-Encoding`, redact header-inspection failures, and never silently truncate an untrusted response. - Integrate DNS-pinned provider dispatch directly into `ModelClient` so package import performs no optional-adapter monkey-patching or order-dependent class mutation. - Reject provider hosts that resolve to any non-globally-routable address, including RFC 6598 shared address space, while retaining explicit multicast, private, loopback, link-local, and reserved-address protections. - Document narrowly scoped Semgrep suppressions for parameter-bound database queries, the explicit development-only TLS verification opt-out, and provider URLs that pass the egress guard. @@ -17,8 +26,11 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and ### Changed - Pin Atheris by Python interpreter so the Python 3.11 fuzz job and the newer central coverage-evidence image both install a published, hash-locked wheel. +- Run repository Tests, Fuzz, and Security workflows for stacked pull requests targeting any branch, bind every checkout to the literal contributor-head SHA, and keep checkout credentials non-persistent so local evidence cannot silently become absent or synthetic-merge-only evidence. ### Documentation - Add APA 7 doctoring for Python environment-marker semantics, Atheris artifact availability and hashes, and the supported-platform uncertainty boundary. +- Add provider-response resource-bound doctoring covering the 8 MiB fail-closed limit, HTTP framing preflight, bounded SSE reads, batch-output partitioning, incident handling, and operational rollback. +- Add pull-request exact-head workflow doctoring covering stacked-base support, contributor-head identity, untrusted-code execution, merge-tree separation, cancellation handling, and rollback. - Record the CI trust boundary between generic coverage and native fuzz execution, including the evidence-preserving retry rule for branch-referenced reusable workflows. diff --git a/README.md b/README.md index 65f57dd4..c18b59f3 100644 --- a/README.md +++ b/README.md @@ -199,9 +199,36 @@ is read from a **KV config store**, never `os.getenv`. `pg_tiktoken` counting, and the production batch backend without adding a repository split here. -Grounding papers (LLM cost, routing, load balancing) live in +Grounding papers (LLM cost, routing, load balancing, evaluation) live in [docs/papers](docs/papers/README.md) with citations. +### NIM cost-quality benchmark (optional harness) + +Evidence-grade benchmark of the routing policies against a **dynamically +discovered** NVIDIA NIM catalog. It probes chat, completions, Responses, +embeddings, image/video/audio understanding, transcription, and speech; compares +direct, route-once, and bounded-conduct cells under one equal total-token and +call budget; records paired uncertainty and Pareto frontiers; and keeps reviewed +actual endpoint-access evidence separate from optional hypothetical paid rates. +The bundled manifest contains thirty locked tasks. It may reach +`evidence_review_required` when at least 90% of policy-task cells complete and +the paired-task floor is met; otherwise it reports `insufficient_evidence`. No +benchmark artifact automatically changes production routing. + +The adapter is lazy and optional: ordinary `import contextual_orchestrator` does +not import or mutate it. Deterministic `--dry-run` receives no network access or +NVIDIA secret. Live execution resolves `NVIDIA_NIM_API_KEY` from the credential +registry, pins HTTPS connections to validation-time public addresses, rejects +redirects and proxy routing, and fails closed on missing/expired evidence. See +[docs/nim_benchmark.md](docs/nim_benchmark.md) and the +[engineering decision record](docs/doctoring/nim-benchmark-evidence-grade.md). + +```bash +python -m contextual_orchestrator nim-benchmark --dry-run \ + --pricing-scenario examples/nim_pricing_scenario.json \ + --output-dir benchmark_artifacts +``` + ## Design Artifacts - [Library research](docs/library_research.md) @@ -255,6 +282,7 @@ python tests/test_paper_contracts.py python tests/test_admin_contract.py python tests/test_conventions.py python tests/test_api_contract.py +python tests/test_nim_benchmark.py python tests/test_security_hardening.py python tests/test_repository_security_metadata.py python tests/test_product_planning_contract.py diff --git a/conductor/tracks.md b/conductor/tracks.md index 968c08ef..b522d66a 100644 --- a/conductor/tracks.md +++ b/conductor/tracks.md @@ -4,3 +4,4 @@ |---|---|---| | 001-paper-grounded-orchestrator | active | Implement the source-backed orchestration contract with TDD, DDD, and CDD | | 002-enterprise-design-foundation | active | Add paper-grounded screen design, user stories, REST API, code/DB conventions, and i18n | +| 003-nim-cost-quality-benchmark | active | Evidence-grade NIM catalog discovery, all-modality capability probes, and the route/conduct/single-worker cost-quality benchmark (docs/nim_benchmark.md) | diff --git a/contextual_orchestrator/__main__.py b/contextual_orchestrator/__main__.py index 5f68c3b7..73e711f6 100644 --- a/contextual_orchestrator/__main__.py +++ b/contextual_orchestrator/__main__.py @@ -61,6 +61,22 @@ def main() -> None: _register_credential_command(sys.argv[2:]) return + if len(sys.argv) > 1 and sys.argv[1] == "nim-benchmark": + # Optional benchmark harness (issue #86): dynamic NIM catalog discovery, + # all-modality capability probes, and the cost-quality policy benchmark. + # The explicit composition root derives strict complete-answer scoring, + # while the publication adapter enriches CSV model assignments and emits + # success only after the complete artifact set is transactionally ready. + from .nim_csv_evidence import run_benchmark_cli_with_complete_csv + from .nim_strict_scoring import run_strict_benchmark_cli + + sys.exit( + run_benchmark_cli_with_complete_csv( + sys.argv[2:], + benchmark_cli=run_strict_benchmark_cli, + ) + ) + parser = argparse.ArgumentParser(description="Route or conduct chat requests across model agents.") parser.add_argument("prompt", nargs="?", help="User prompt for CLI mode.") parser.add_argument("--agents", default="examples/agents.mock.json", help="Agent config JSON.") diff --git a/contextual_orchestrator/nim_benchmark.py b/contextual_orchestrator/nim_benchmark.py new file mode 100644 index 00000000..143f3986 --- /dev/null +++ b/contextual_orchestrator/nim_benchmark.py @@ -0,0 +1,2786 @@ +"""Evidence-grade NVIDIA NIM model discovery and cost-quality benchmark harness. + +This is the optional benchmark adapter demanded by the "[Product Gap] +Evidence-grade NVIDIA NIM model discovery and cost-quality benchmark" issue. +It is NOT part of the runtime request path: the gateway keeps its +provider-neutral, standard-library-only contract, and this module simply +reuses the same stdlib HTTP/KV seams to measure the repo's own policies +(``route_once`` vs ``conduct`` vs single-worker baselines) against a +dynamically discovered NIM catalog. + +Design contract (mirrors the issue): + +* **Dynamic catalog** — models come from the OpenAI-compatible + ``GET /v1/models`` endpoint; nothing here hard-codes a model inventory. +* **All-modality capability probes** — every discovered model is probed, + under bounded concurrency and a hard request budget, for every contract + NIM can host: chat completions, legacy text completions, the Responses + API, embeddings, image understanding (vision), video understanding, + audio understanding (omni-style ``input_audio``), audio transcription, + and audio speech synthesis. ``omni_capable`` is derived, never probed + separately. A run that cannot execute every cell fails before capability egress. +* **Fair comparison** — the same task manifest, scorers, call caps, + workflow-depth cap (five), timeout, and output-token budget apply to all + compared systems. +* **Honest cost accounting** — actual cost is recorded as ``0`` while the + hosted catalog is free to the caller; hypothetical paid cost is computed + only from an explicit versioned pricing scenario and is ``"unknown"`` + for any model the scenario does not price. The two never mix. +* **Fail closed** — a live run refuses to start without the KV-resolvable + ``NVIDIA_NIM_API_KEY`` credential, complete provenance, and a request + budget large enough for the planned evaluation. The secret is never + accepted via argv and never serialized into artifacts. +* **Deterministic dry run** — ``--dry-run`` drives the entire pipeline + against an in-process synthetic provider covering every modality class, + so manifests, pricing assumptions, scorer registration, budgets, and + output schemas are validated without any network egress. +""" + +from __future__ import annotations + +import argparse +import base64 +import csv +import dataclasses +import datetime as datetime_module +import hashlib +import http.client +import io +import json +import math +import os +import random +import re +import socket +import ssl +import struct +import threading +import time +import urllib.error +import urllib.parse +import wave +from concurrent.futures import ThreadPoolExecutor +from typing import Any, Callable + +from .conventions import is_two_word_snake_case +from .credentials import NotConfigured, get_credential, register_credential +from .orchestrator import ( + ModelAgent, + ModelClient, + OrchestrationPolicy, + TaskOrchestrator, + estimate_tokens, +) +from .provider_transport import ( + _PinnedHTTPSConnection, + _validated_public_addresses, +) + +BENCHMARK_SCHEMA_VERSION = "1.0.0" +NIM_DEFAULT_ENDPOINT = "https://integrate.api.nvidia.com/v1" +NIM_CREDENTIAL_NAME = "NVIDIA_NIM_API_KEY" +DRY_RUN_PROVENANCE_PLACEHOLDER = "dry_run" +# Fixed epoch for deterministic dry-run artifacts (2026-01-01T00:00:00Z). +DRY_RUN_FIXED_UNIX_TIME = 1767225600.0 +# Issue contract: Conductor/TRINITY-style deep paths are capped at five steps. +MAX_WORKFLOW_DEPTH = 5 +# Bound every provider response before materializing it in memory. Eight MiB is +# ample for model catalogs, JSON probe responses, and the deliberately tiny +# benchmark media outputs while preventing a provider from returning an +# unbounded body to the evidence collector. +MAX_PROVIDER_RESPONSE_BYTES = 8 * 1024 * 1024 +# Smoke manifests can exercise plumbing but cannot justify production routing. +MINIMUM_PAIRED_TASK_COUNT = 30 +REQUIRED_COMPLETION_FRACTION = 0.9 + +ACTUAL_COST_EVIDENCE: dict[str, Any] = { + "evidence_schema_version": "1.0.0", + "source_title": "NVIDIA NIM General FAQ", + "source_url": "https://docs.api.nvidia.com/nim/docs/product", + "reviewed_at_date": "2026-08-05", + "valid_until_date": "2026-09-04", + "access_program": "NVIDIA Developer Program API Catalog hosted endpoints", + "access_scope": "free API endpoint access for prototyping", + "production_access_note": ( + "Production support and licensing require NVIDIA AI Enterprise." + ), + "actual_cost_usd": 0.0, + "uncertainty": ( + "Hosted-endpoint access terms can change. Live runs fail closed after " + "the validity date until the official source is reviewed again." + ), +} + +# Transport seam: (method, url, headers, body_bytes_or_None) -> (status, body). +# Network-level failures raise URLError/TimeoutError/ConnectionError/socket.timeout. +ProviderTransport = Callable[[str, str, dict[str, str], bytes | None], tuple[int, bytes]] + + +class BenchmarkContractError(ValueError): + """A manifest, pricing scenario, schema, or parameter violates the benchmark contract.""" + + +class CatalogDiscoveryError(RuntimeError): + """The provider model catalog could not be discovered or parsed completely.""" + + +class BenchmarkAuthError(RuntimeError): + """The provider rejected the benchmark credential; the run must fail closed.""" + + +class BenchmarkBudgetError(RuntimeError): + """The benchmark would exceed (or has exceeded) its hard request budget.""" + + +class SecretLeakError(RuntimeError): + """A serialized artifact contained the provider secret; writing is refused.""" + + +# -------------------------------------------------------------------------- +# Egress guard + default transport +# -------------------------------------------------------------------------- + + +def require_public_https_endpoint(url: str) -> tuple[str, ...]: + """Resolve and return public addresses approved for one HTTPS request. + + The returned addresses are the only addresses a caller may dial. Combining + resolution and validation in one operation closes the DNS time-of-check to + time-of-use gap caused by a generic URL opener resolving the hostname again. + + Args: + url: Complete provider URL whose origin may receive credentials. + + Returns: + Deduplicated globally routable IPv4 or IPv6 addresses. + + Raises: + BenchmarkContractError: If the URL is not HTTPS, lacks a hostname, or + resolves to any non-global address. + """ + parsed = urllib.parse.urlparse(url) + if parsed.scheme != "https" or not parsed.hostname: + raise BenchmarkContractError(f"benchmark endpoint must use https: {url!r}") + try: + return _validated_public_addresses( + parsed.hostname.lower(), + parsed.port or 443, + "NIM benchmark", + ) + except RuntimeError as exc: + raise BenchmarkContractError(str(exc)) from exc + + +def build_default_transport(timeout_seconds: float) -> ProviderTransport: + """Build direct HTTPS transport pinned to each request's DNS evidence. + + Every request resolves exactly once, validates every answer as globally + routable, and connects only to those validation-time addresses. The original + hostname remains the HTTP authority and TLS SNI/certificate name. Environment + proxies and redirect handlers are never used. + + Args: + timeout_seconds: Socket, TLS, and response timeout for each address. + + Returns: + A provider transport returning HTTP status and raw response bytes. + + Raises: + BenchmarkContractError: If a request URL or redirect violates policy. + urllib.error.URLError: If all validation-time addresses fail. + """ + if ( + isinstance(timeout_seconds, bool) + or not isinstance(timeout_seconds, (int, float)) + or not math.isfinite(timeout_seconds) + or timeout_seconds <= 0 + ): + raise BenchmarkContractError("timeout_seconds must be a positive number") + ssl_context = ssl.create_default_context() + + def transport( + method: str, + url: str, + headers: dict[str, str], + body: bytes | None, + ) -> tuple[int, bytes]: + """Perform one request without proxy lookup, redirect follow, or re-resolution.""" + parsed = urllib.parse.urlparse(url) + approved_addresses = require_public_https_endpoint(url) + port = parsed.port or 443 + target = parsed.path or "/" + if parsed.params: + target = f"{target};{parsed.params}" + if parsed.query: + target = f"{target}?{parsed.query}" + request_headers = dict(headers) + request_headers["Connection"] = "close" + + last_error: BaseException | None = None + for pinned_ip in approved_addresses: + connection = _PinnedHTTPSConnection( + parsed.hostname or "", + pinned_ip, + port, + float(timeout_seconds), + ssl_context, + ) + response = None + try: + connection.request( + method, + target, + body=body, + headers=request_headers, + ) + response = connection.getresponse() + status = int(response.status) + response_body = response.read(MAX_PROVIDER_RESPONSE_BYTES + 1) + if len(response_body) > MAX_PROVIDER_RESPONSE_BYTES: + raise BenchmarkContractError( + "benchmark provider response exceeds " + f"{MAX_PROVIDER_RESPONSE_BYTES} byte limit" + ) + if 300 <= status < 400: + raise BenchmarkContractError( + f"benchmark provider redirects are not permitted (HTTP {status})" + ) + return status, response_body + except BenchmarkContractError: + raise + except (OSError, http.client.HTTPException) as exc: + last_error = exc + finally: + if response is not None: + response.close() + connection.close() + raise urllib.error.URLError(last_error or "benchmark provider connection failed") + + return transport + + +# -------------------------------------------------------------------------- +# Request budget (fail-closed hard cap) +# -------------------------------------------------------------------------- + + +class RequestBudget: + """Thread-safe hard cap on total provider requests for one benchmark run.""" + + def __init__(self, max_total_requests: int) -> None: + """Create a positive integer request allowance. + + Args: + max_total_requests: Maximum provider calls in the complete run. + + Raises: + BenchmarkContractError: If the cap is boolean or not positive. + """ + if ( + isinstance(max_total_requests, bool) + or not isinstance(max_total_requests, int) + or max_total_requests < 1 + ): + raise BenchmarkContractError("max_total_requests must be a positive integer") + self.max_total_requests = max_total_requests + self._spent = 0 + self._lock = threading.Lock() + + def try_spend(self) -> bool: + """Consume one request from the budget; return ``False`` when exhausted.""" + with self._lock: + if self._spent >= self.max_total_requests: + return False + self._spent += 1 + return True + + def spend_or_fail(self) -> None: + """Consume one request or raise for a phase that must complete.""" + if not self.try_spend(): + raise BenchmarkBudgetError( + f"request budget of {self.max_total_requests} exhausted; " + "refusing further provider calls" + ) + + @property + def requests_spent(self) -> int: + """Return the number of provider requests consumed so far.""" + with self._lock: + return self._spent + + @property + def remaining_requests(self) -> int: + """Return the non-negative provider request allowance still available.""" + with self._lock: + return self.max_total_requests - self._spent + + +class _BudgetedModelClient(ModelClient): + """ModelClient that charges every chat call against the shared request budget.""" + + def __init__(self, request_budget: RequestBudget, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._request_budget = request_budget + + def chat(self, agent: ModelAgent, messages: list[dict[str, str]], temperature: float = 0.2) -> str: + """Spend one budgeted request, then delegate to the normal chat path.""" + self._request_budget.spend_or_fail() + return super().chat(agent, messages, temperature) + + +class PolicyTokenBudgetExceeded(RuntimeError): + """A policy cell exhausted its shared token or call allowance.""" + + +class EqualBudgetModelClient: + """Delegate model calls while enforcing an equal per-cell budget. + + Direct, route-once, conduct, and cheapest-worker cells all receive the same + total prompt-plus-completion token allowance and the same declared maximum- + call envelope. The wrapper lowers each provider call's output cap to the + remaining allowance and reconciles estimates with provider-reported usage. + """ + + def __init__( + self, + delegate: ModelClient, + total_token_budget: int, + maximum_calls: int, + ) -> None: + """Create a cell-local limiter around an existing provider client. + + Args: + delegate: Existing request-budgeted provider client. + total_token_budget: Cell-wide prompt-plus-completion allowance. + maximum_calls: Maximum calls available to every compared policy. + + Raises: + ValueError: If either allowance is boolean or not positive. + """ + if ( + isinstance(total_token_budget, bool) + or not isinstance(total_token_budget, int) + or total_token_budget < 1 + ): + raise ValueError("total_token_budget must be a positive integer") + if ( + isinstance(maximum_calls, bool) + or not isinstance(maximum_calls, int) + or maximum_calls < 1 + ): + raise ValueError("maximum_calls must be a positive integer") + self._delegate = delegate + self.total_token_budget = total_token_budget + self.maximum_calls = maximum_calls + self.observed_calls = 0 + self.observed_tokens = 0 + self._pending_estimated_tokens: int | None = None + self._exceeded = False + + @property + def max_output_tokens(self) -> int: + """Expose the delegate cap for compatibility with orchestration clients.""" + return int(self._delegate.max_output_tokens) + + @max_output_tokens.setter + def max_output_tokens(self, value: int) -> None: + """Forward explicit cap changes to the delegated model client.""" + self._delegate.max_output_tokens = value + + @property + def remaining_tokens(self) -> int: + """Return the non-negative token allowance remaining in this cell.""" + return max(0, self.total_token_budget - self.observed_tokens) + + @property + def exceeded(self) -> bool: + """Return whether observed usage crossed the configured allowance.""" + return self._exceeded + + @staticmethod + def _coerce_usage_count(value: Any) -> int | None: + """Return one valid non-negative provider token count, otherwise ``None``.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + if not math.isfinite(value) or value < 0: + return None + return int(value) + + def chat( + self, + agent: ModelAgent, + messages: list[dict[str, Any]], + temperature: float = 0.2, + ) -> str: + """Perform one delegated call within the remaining cell allowance. + + Raises: + PolicyTokenBudgetExceeded: If the call or token allowance is already + exhausted or the prompt cannot fit. + """ + if self._exceeded or self.observed_calls >= self.maximum_calls: + raise PolicyTokenBudgetExceeded( + "policy cell maximum-call allowance exhausted" + ) + prompt_text = json.dumps(messages, ensure_ascii=False, sort_keys=True) + prompt_tokens = estimate_tokens(prompt_text) + output_allowance = self.remaining_tokens - prompt_tokens + if output_allowance < 1: + raise PolicyTokenBudgetExceeded( + "policy cell total-token allowance exhausted" + ) + + original_max_output_tokens = int(self._delegate.max_output_tokens) + self._delegate.max_output_tokens = min( + original_max_output_tokens, + output_allowance, + ) + self.observed_calls += 1 + try: + answer = self._delegate.chat(agent, messages, temperature) + finally: + self._delegate.max_output_tokens = original_max_output_tokens + + estimated_total = prompt_tokens + estimate_tokens(answer) + self.observed_tokens += estimated_total + self._pending_estimated_tokens = estimated_total + self._exceeded = self.observed_tokens > self.total_token_budget + return answer + + def take_usage(self) -> dict[str, Any] | None: + """Return delegated usage and replace the latest estimate when valid.""" + usage = self._delegate.take_usage() + pending_estimate = self._pending_estimated_tokens + self._pending_estimated_tokens = None + if pending_estimate is None or not isinstance(usage, dict): + return usage + prompt_tokens = self._coerce_usage_count(usage.get("prompt_tokens")) + completion_tokens = self._coerce_usage_count( + usage.get("completion_tokens") + ) + if prompt_tokens is None or completion_tokens is None: + return usage + self.observed_tokens += prompt_tokens + completion_tokens - pending_estimate + self._exceeded = self.observed_tokens > self.total_token_budget + return usage + + +# -------------------------------------------------------------------------- +# Catalog discovery +# -------------------------------------------------------------------------- + + +def parse_model_catalog_body(body: bytes) -> dict[str, Any]: + """Parse an OpenAI-compatible ``GET /v1/models`` body into a hygienic inventory. + + Adversarial inputs (non-JSON, wrong shapes, entries without an id, + duplicate ids) never crash: structural failures raise + :class:`CatalogDiscoveryError`; salvageable per-entry problems are recorded + with machine-readable reasons in ``invalid_entries``/``duplicate_model_ids``. + """ + try: + decoded = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, ValueError, RecursionError) as exc: + raise CatalogDiscoveryError(f"model catalog body is not valid JSON: {exc}") from exc + if not isinstance(decoded, dict) or not isinstance(decoded.get("data"), list): + raise CatalogDiscoveryError("model catalog must be a JSON object with a 'data' list") + + models: list[dict[str, Any]] = [] + seen_ids: set[str] = set() + duplicate_model_ids: list[str] = [] + invalid_entries: list[dict[str, Any]] = [] + for index, entry in enumerate(decoded["data"]): + if not isinstance(entry, dict): + invalid_entries.append({"entry_index": index, "invalid_reason": "entry_not_an_object"}) + continue + model_id = entry.get("id") + if not isinstance(model_id, str) or not model_id.strip(): + invalid_entries.append({"entry_index": index, "invalid_reason": "missing_model_id"}) + continue + model_id = model_id.strip() + if model_id in seen_ids: + duplicate_model_ids.append(model_id) + continue + seen_ids.add(model_id) + owned_by = entry.get("owned_by") + models.append( + { + "model_id": model_id, + "owned_by": owned_by if isinstance(owned_by, str) else "", + } + ) + models.sort(key=lambda row: row["model_id"]) + return { + "models": models, + "duplicate_model_ids": sorted(duplicate_model_ids), + "invalid_entries": invalid_entries, + } + + +def discover_model_catalog( + transport: ProviderTransport, + endpoint: str, + api_key: str, + request_budget: RequestBudget, +) -> dict[str, Any]: + """Fetch and parse the live model catalog, failing closed on any discovery gap.""" + request_budget.spend_or_fail() + url = f"{endpoint.rstrip('/')}/models" + try: + status, body = transport("GET", url, _auth_headers(api_key), None) + except (urllib.error.URLError, TimeoutError, ConnectionError, socket.timeout) as exc: + raise CatalogDiscoveryError(f"model catalog request failed: {type(exc).__name__}") from exc + if status in (401, 403): + raise BenchmarkAuthError(f"provider rejected the benchmark credential (HTTP {status})") + if status != 200: + raise CatalogDiscoveryError(f"model catalog request returned HTTP {status}") + catalog = parse_model_catalog_body(body) + if not catalog["models"]: + raise CatalogDiscoveryError("model catalog discovery returned zero usable models") + return catalog + + +def _auth_headers(api_key: str, content_type: str = "application/json") -> dict[str, str]: + """Standard provider headers; the bearer value never appears in artifacts.""" + return {"authorization": f"Bearer {api_key}", "content-type": content_type, "accept": "application/json"} + + +# -------------------------------------------------------------------------- +# Capability probes — every contract NIM can host +# -------------------------------------------------------------------------- + +# 1x1 transparent PNG for vision probes. +_TINY_PNG_BASE64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" +) +# Deterministic one-frame 16x16 H.264 MP4 generated once with bit-exact flags. +_TINY_MP4_BASE64 = """AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1wNDEAAALzbW9vdgAAAGxtdmhkAAAAAAAAAAAA +AAAAAAAD6AAAACgAAQAAAQAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAA +AABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAkJ0cmFrAAAAXHRraGQAAAADAAAA +AAAAAAAAAAABAAAAAAAAACgAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAA +AAAAAAAAAABAAAAAABAAAAAQAAAAAAAkZWR0cwAAABxlbHN0AAAAAAAAAAEAAAAoAAAAAAABAAAA +AAG6bWRpYQAAACBtZGhkAAAAAAAAAAAAAAAAAAAyAAAAAgBVxAAAAAAALWhkbHIAAAAAAAAAAHZp +ZGUAAAAAAAAAAAAAAABWaWRlb0hhbmRsZXIAAAABZW1pbmYAAAAUdm1oZAAAAAEAAAAAAAAAAAAA +ACRkaW5mAAAAHGRyZWYAAAAAAAAAAQAAAAx1cmwgAAAAAQAAASVzdGJsAAAAwXN0c2QAAAAAAAAA +AQAAALFhdmMxAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAABAAEABIAAAASAAAAAAAAAABDExhdmMg +bGlieDI2NAAAAAAAAAAAAAAAAAAAAAAAAAAAGP//AAAAN2F2Y0MBZAAK/+EAGWdkAAqscgRewEQA +AAMABAAAAwDIPEiWEYABAAdo6EOPEyEw/fj4AAAAABBwYXNwAAAAAQAAAAEAAAAUYnRydAAAAAAA +Ai3QAAAAAAAAABhzdHRzAAAAAAAAAAEAAAABAAACAAAAABxzdHNjAAAAAAAAAAEAAAABAAAAAQAA +AAEAAAAUc3RzegAAAAAAAALKAAAAAQAAABRzdGNvAAAAAAAAAAEAAAMjAAAAPXVkdGEAAAA1bWV0 +YQAAAAAAAAAhaGRscgAAAAAAAAAAbWRpcmFwcGwAAAAAAAAAAAAAAAAIaWxzdAAAAAhmcmVlAAAC +0m1kYXQAAAKyBgX//67cRem95tlIt5Ys2CDZI+7veDI2NCAtIGNvcmUgMTY0IHIzMTA4IDMxZTE5 +ZjkgLSBILjI2NC9NUEVHLTQgQVZDIGNvZGVjIC0gQ29weWxlZnQgMjAwMy0yMDIzIC0gaHR0cDov +L3d3dy52aWRlb2xhbi5vcmcveDI2NC5odG1sIC0gb3B0aW9uczogY2FiYWM9MSByZWY9MTYgZGVi +bG9jaz0xOi0zOi0zIGFuYWx5c2U9MHgzOjB4MTMzIG1lPXVtaCBzdWJtZT0xMCBwc3k9MSBwc3lf +cmQ9Mi4wMDowLjcwIG1peGVkX3JlZj0xIG1lX3JhbmdlPTI0IGNocm9tYV9tZT0xIHRyZWxsaXM9 +MiA4eDhkY3Q9MSBjcW09MCBkZWFkem9uZT0yMSwxMSBmYXN0X3Bza2lwPTEgY2hyb21hX3FwX29m +ZnNldD0tNCB0aHJlYWRzPTEgbG9va2FoZWFkX3RocmVhZHM9MSBzbGljZWRfdGhyZWFkcz0wIG5y +PTAgZGVjaW1hdGU9MSBpbnRlcmxhY2VkPTAgYmx1cmF5X2NvbXBhdD0wIGNvbnN0cmFpbmVkX2lu +dHJhPTAgYmZyYW1lcz04IGJfcHlyYW1pZD0yIGJfYWRhcHQ9MiBiX2JpYXM9MCBkaXJlY3Q9MyB3 +ZWlnaHRiPTEgb3Blbl9nb3A9MCB3ZWlnaHRwPTIga2V5aW50PTI1MCBrZXlpbnRfbWluPTI1IHNj +ZW5lY3V0PTQwIGludHJhX3JlZnJlc2g9MCByY19sb29rYWhlYWQ9NjAgcmM9Y3JmIG1idHJlZT0x +IGNyZj0yMy4wIHFjb21wPTAuNjAgcXBtaW49MCBxcG1heD02OSBxcHN0ZXA9NCBpcF9yYXRpbz0x +LjQwIGFxPTE6MS4yMACAAAAAEGWIgQAG5z/+9vD+BTZWBME=""" +VIDEO_PROBE_FIXTURE_SHA256 = "777dda43b5a15162b68a39aa486d5c70c9994d7fe761742fd00d4e13508983c0" +_MULTIPART_BOUNDARY = "nim-benchmark-boundary-7f3a1c" +_MP4_CONTAINER_BOX_TYPES = frozenset( + {b"moov", b"trak", b"mdia", b"minf", b"dinf", b"stbl", b"edts", b"udta"} +) + + +def _iter_mp4_boxes( + data: bytes, + start_offset: int = 0, + end_offset: int | None = None, +): + """Yield validated ISO-BMFF boxes as type and payload/end offsets. + + Args: + data: Complete MP4 bytes. + start_offset: First byte of the bounded box sequence. + end_offset: Exclusive sequence end, defaulting to ``len(data)``. + + Yields: + Tuples of ``(box_type, payload_start, box_end)``. + + Raises: + BenchmarkContractError: If box headers, sizes, or bounds are malformed. + """ + sequence_end = len(data) if end_offset is None else end_offset + offset = start_offset + while offset < sequence_end: + if sequence_end - offset < 8: + raise BenchmarkContractError("video probe MP4 has a truncated box header") + box_size = struct.unpack(">I", data[offset : offset + 4])[0] + box_type = data[offset + 4 : offset + 8] + header_size = 8 + if box_size == 1: + if sequence_end - offset < 16: + raise BenchmarkContractError("video probe MP4 has a truncated extended box") + box_size = struct.unpack(">Q", data[offset + 8 : offset + 16])[0] + header_size = 16 + elif box_size == 0: + box_size = sequence_end - offset + if box_size < header_size or offset + box_size > sequence_end: + raise BenchmarkContractError("video probe MP4 box exceeds its parent bounds") + payload_start = offset + header_size + box_end = offset + box_size + yield box_type, payload_start, box_end + offset = box_end + + +def _walk_mp4_boxes(data: bytes): + """Yield every validated box in the deterministic video fixture.""" + + def walk(start_offset: int, end_offset: int): + """Recursively traverse known ISO-BMFF container boxes.""" + for box_type, payload_start, box_end in _iter_mp4_boxes( + data, + start_offset, + end_offset, + ): + yield box_type, payload_start, box_end + if box_type in _MP4_CONTAINER_BOX_TYPES: + yield from walk(payload_start, box_end) + elif box_type == b"meta": + if box_end - payload_start < 4: + raise BenchmarkContractError( + "video probe MP4 meta box lacks full-box flags" + ) + yield from walk(payload_start + 4, box_end) + + yield from walk(0, len(data)) + + +def validate_video_probe_fixture(data: bytes) -> dict[str, Any]: + """Validate one H.264 video stream, dimensions, and frame count. + + Args: + data: Candidate ISO-BMFF/MP4 bytes. + + Returns: + Codec, width, height, and frame count for the single video stream. + + Raises: + BenchmarkContractError: If required boxes or one-frame video evidence is + missing, inconsistent, or malformed. + """ + top_level_types = {box_type for box_type, _, _ in _iter_mp4_boxes(data)} + if not {b"ftyp", b"moov", b"mdat"} <= top_level_types: + raise BenchmarkContractError("video probe MP4 lacks ftyp, moov, or mdat") + + width: int | None = None + height: int | None = None + frame_count: int | None = None + video_handler_count = 0 + codec_name: str | None = None + for box_type, payload_start, box_end in _walk_mp4_boxes(data): + payload = data[payload_start:box_end] + if box_type == b"tkhd": + if len(payload) < 8: + raise BenchmarkContractError("video probe MP4 tkhd box is truncated") + width_fixed, height_fixed = struct.unpack(">II", payload[-8:]) + width = width_fixed >> 16 + height = height_fixed >> 16 + elif box_type == b"hdlr" and len(payload) >= 12: + if payload[8:12] == b"vide": + video_handler_count += 1 + elif box_type == b"stsz": + if len(payload) < 12: + raise BenchmarkContractError("video probe MP4 stsz box is truncated") + frame_count = struct.unpack(">I", payload[8:12])[0] + elif box_type == b"stsd" and b"avc1" in payload: + codec_name = "h264" + + metadata = { + "codec_name": codec_name, + "width": width, + "height": height, + "frame_count": frame_count, + } + expected = { + "codec_name": "h264", + "width": 16, + "height": 16, + "frame_count": 1, + } + if video_handler_count != 1 or metadata != expected: + raise BenchmarkContractError( + f"video probe MP4 must contain one 16x16 one-frame H.264 stream: {metadata}" + ) + return metadata + + +def _tiny_mp4_bytes() -> bytes: + """Return the validated deterministic one-frame MP4 probe fixture.""" + fixture = base64.b64decode("".join(_TINY_MP4_BASE64.split()), validate=True) + if hashlib.sha256(fixture).hexdigest() != VIDEO_PROBE_FIXTURE_SHA256: + raise BenchmarkContractError("video probe MP4 checksum does not match") + validate_video_probe_fixture(fixture) + return fixture + + +def _tiny_wav_bytes() -> bytes: + """Return a deterministic 10ms silent mono WAV used by the audio probes.""" + buffer = io.BytesIO() + with wave.open(buffer, "wb") as handle: + handle.setnchannels(1) + handle.setsampwidth(2) + handle.setframerate(8000) + handle.writeframes(b"\x00\x00" * 80) + return buffer.getvalue() + + +def _chat_probe_body(model_id: str, content: Any) -> bytes: + """Serialize a minimal single-message chat probe for ``model_id``.""" + payload = { + "model": model_id, + "messages": [{"role": "user", "content": content}], + "max_tokens": 1, + "temperature": 0.0, + } + return json.dumps(payload).encode("utf-8") + + +def _multipart_transcription_body(model_id: str) -> bytes: + """Build a deterministic multipart body for the audio transcription probe.""" + boundary = _MULTIPART_BOUNDARY.encode("ascii") + parts = [ + b"--" + boundary, + b'Content-Disposition: form-data; name="model"', + b"", + model_id.encode("utf-8"), + b"--" + boundary, + b'Content-Disposition: form-data; name="file"; filename="probe.wav"', + b"Content-Type: audio/wav", + b"", + _tiny_wav_bytes(), + b"--" + boundary + b"--", + b"", + ] + return b"\r\n".join(parts) + + +def _has_choice(payload: dict[str, Any]) -> bool: + """True when an OpenAI chat/completions payload carries at least one choice.""" + choices = payload.get("choices") + return isinstance(choices, list) and len(choices) > 0 + + +def _has_embedding(payload: dict[str, Any]) -> bool: + """True when an embeddings payload carries at least one embedding vector.""" + data = payload.get("data") + return ( + isinstance(data, list) + and len(data) > 0 + and isinstance(data[0], dict) + and isinstance(data[0].get("embedding"), list) + ) + + +def _has_response_output(payload: dict[str, Any]) -> bool: + """True when a Responses API payload carries an output field.""" + return any(key in payload for key in ("output", "output_text", "response")) + + +def _has_transcription_text(payload: dict[str, Any]) -> bool: + """True when a transcription payload carries the transcribed text field.""" + return isinstance(payload.get("text"), str) + + +def _image_data_uri() -> str: + """Data URI of the tiny PNG used by the image-understanding probe.""" + return f"data:image/png;base64,{_TINY_PNG_BASE64}" + + +def _video_data_uri() -> str: + """Return a data URI containing the validated one-frame MP4 fixture.""" + return f"data:video/mp4;base64,{base64.b64encode(_tiny_mp4_bytes()).decode('ascii')}" + + +def _audio_probe_base64() -> str: + """Base64 WAV payload used by the omni-style audio-understanding probe.""" + return base64.b64encode(_tiny_wav_bytes()).decode("ascii") + + +def _build_capability_probes() -> dict[str, dict[str, Any]]: + """Registry of every probe contract, in the fixed order they are attempted. + + Each spec: ``path``, ``content_type``, ``body`` (model_id -> bytes), + ``validate`` (decoded JSON -> bool), and ``binary_response`` for endpoints + that answer with raw media instead of JSON. A deterministic validated media fixture is used for each modality; only an + HTTP 200 with the expected response shape counts as contract support. + """ + return { + "chat_completion": { + "path": "/chat/completions", + "content_type": "application/json", + "body": lambda model_id: _chat_probe_body(model_id, "Reply with OK."), + "validate": _has_choice, + "binary_response": False, + }, + "text_completion": { + "path": "/completions", + "content_type": "application/json", + "body": lambda model_id: json.dumps( + {"model": model_id, "prompt": "OK", "max_tokens": 1, "temperature": 0.0} + ).encode("utf-8"), + "validate": _has_choice, + "binary_response": False, + }, + "response_generation": { + "path": "/responses", + "content_type": "application/json", + "body": lambda model_id: json.dumps( + {"model": model_id, "input": "Reply with OK.", "max_output_tokens": 16} + ).encode("utf-8"), + "validate": _has_response_output, + "binary_response": False, + }, + "text_embedding": { + "path": "/embeddings", + "content_type": "application/json", + "body": lambda model_id: json.dumps({"model": model_id, "input": "probe"}).encode("utf-8"), + "validate": _has_embedding, + "binary_response": False, + }, + "image_understanding": { + "path": "/chat/completions", + "content_type": "application/json", + "body": lambda model_id: _chat_probe_body( + model_id, + [ + {"type": "text", "text": "Describe the image in one word."}, + {"type": "image_url", "image_url": {"url": _image_data_uri()}}, + ], + ), + "validate": _has_choice, + "binary_response": False, + }, + "video_understanding": { + "path": "/chat/completions", + "content_type": "application/json", + "body": lambda model_id: _chat_probe_body( + model_id, + [ + {"type": "text", "text": "Describe the video in one word."}, + {"type": "video_url", "video_url": {"url": _video_data_uri()}}, + ], + ), + "validate": _has_choice, + "binary_response": False, + }, + "audio_understanding": { + "path": "/chat/completions", + "content_type": "application/json", + "body": lambda model_id: _chat_probe_body( + model_id, + [ + {"type": "text", "text": "Transcribe the audio."}, + {"type": "input_audio", "input_audio": {"data": _audio_probe_base64(), "format": "wav"}}, + ], + ), + "validate": _has_choice, + "binary_response": False, + }, + "audio_transcription": { + "path": "/audio/transcriptions", + "content_type": f"multipart/form-data; boundary={_MULTIPART_BOUNDARY}", + "body": _multipart_transcription_body, + "validate": _has_transcription_text, + "binary_response": False, + }, + "audio_speech": { + "path": "/audio/speech", + "content_type": "application/json", + "body": lambda model_id: json.dumps( + {"model": model_id, "input": "OK", "voice": "default"} + ).encode("utf-8"), + "validate": lambda payload: True, + "binary_response": True, + }, + } + + +CAPABILITY_PROBES = _build_capability_probes() +CAPABILITY_PROBE_ORDER = tuple(CAPABILITY_PROBES) + +# HTTP statuses meaning "this model does not serve this contract" (not an outage). +_UNSUPPORTED_HTTP_STATUS = frozenset({400, 404, 405, 415, 422, 501}) + + +def classify_probe_status(status: int) -> str: + """Map one probe HTTP status to its machine-readable outcome class.""" + if status == 200: + return "supported" + if status in _UNSUPPORTED_HTTP_STATUS: + return "unsupported" + if status == 401: + return "auth_rejected" + if status == 403: + return "unavailable" + if status == 408: + return "timeout" + if status == 429: + return "rate_limited" + if status >= 500: + return "unavailable" + return "failed" + + +def execute_capability_probe( + transport: ProviderTransport, + endpoint: str, + api_key: str, + model_id: str, + capability_name: str, + timer: Callable[[], float] = time.perf_counter, +) -> dict[str, Any]: + """Run one capability probe against one model and classify the outcome.""" + spec = CAPABILITY_PROBES[capability_name] + url = f"{endpoint.rstrip('/')}{spec['path']}" + headers = _auth_headers(api_key, spec["content_type"]) + started = timer() + try: + status, body = transport("POST", url, headers, spec["body"](model_id)) + except (TimeoutError, socket.timeout) as exc: + return _probe_row(capability_name, "timeout", f"network_timeout:{type(exc).__name__}", None, started, timer) + except (urllib.error.URLError, ConnectionError) as exc: + return _probe_row(capability_name, "failed", f"network_error:{type(exc).__name__}", None, started, timer) + + outcome = classify_probe_status(status) + if outcome == "auth_rejected": + raise BenchmarkAuthError(f"provider rejected the benchmark credential during probes (HTTP {status})") + reason = f"http_status:{status}" + if outcome == "supported" and not spec["binary_response"]: + try: + payload = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, ValueError): + payload = None + if not isinstance(payload, dict) or not spec["validate"](payload): + outcome, reason = "malformed_response", "http_200_with_unexpected_body_shape" + if outcome == "supported" and spec["binary_response"] and not body: + outcome, reason = "malformed_response", "http_200_with_empty_media_body" + return _probe_row(capability_name, outcome, reason, status, started, timer) + + +def _probe_row( + capability_name: str, + probe_outcome: str, + outcome_reason: str, + http_status: int | None, + started: float, + timer: Callable[[], float], +) -> dict[str, Any]: + """Assemble one probe result row with its end-to-end latency.""" + return { + "capability_name": capability_name, + "probe_outcome": probe_outcome, + "outcome_reason": outcome_reason, + "http_status": http_status, + "probe_latency_ms": round((timer() - started) * 1000, 2), + } + + +_CHAT_CLASSIFICATIONS = frozenset({"chat_capable", "vision_chat_capable", "omni_capable"}) + + +def classify_model_capabilities(probe_rows: list[dict[str, Any]]) -> dict[str, Any]: + """Derive one model-level classification from its per-capability probe rows. + + Chat-family support wins (with vision/omni refinements derived from the + modality probes); otherwise the strongest single-contract class applies; + otherwise the dominant failure mode is reported, so a skipped or throttled + model is never silently confused with an unsupported one. + """ + outcomes = {row["capability_name"]: row["probe_outcome"] for row in probe_rows} + supported = sorted(name for name, outcome in outcomes.items() if outcome == "supported") + supported_set = set(supported) + if "chat_completion" in supported_set: + if {"image_understanding", "audio_understanding"} <= supported_set: + classification = "omni_capable" + elif supported_set & {"image_understanding", "video_understanding"}: + classification = "vision_chat_capable" + else: + classification = "chat_capable" + elif "text_embedding" in supported_set: + classification = "embedding_only" + elif "text_completion" in supported_set: + classification = "completion_only" + elif "response_generation" in supported_set: + classification = "responses_only" + elif supported_set & {"audio_transcription", "audio_speech"}: + classification = "audio_only" + else: + observed = set(outcomes.values()) + if observed == {"skipped"}: + classification = "skipped" + elif "rate_limited" in observed: + classification = "rate_limited" + elif "unavailable" in observed: + classification = "unavailable" + elif observed & {"timeout", "failed", "malformed_response"}: + classification = "failed" + else: + classification = "unsupported_for_contract" + return { + "model_classification": classification, + "supported_capabilities": supported, + "chat_eligible": classification in _CHAT_CLASSIFICATIONS, + } + + +def probe_discovered_models( + models: list[dict[str, Any]], + transport: ProviderTransport, + endpoint: str, + api_key: str, + request_budget: RequestBudget, + probe_concurrency: int, + clock: Callable[[], float], + timer: Callable[[], float] = time.perf_counter, +) -> list[dict[str, Any]]: + """Probe every model with deterministic allocation and bounded concurrency. + + The permitted ``(model_id, capability)`` cells are fixed in sorted catalog + and capability order before worker threads start. Thread scheduling can + change completion order but cannot choose which cells run. + + Args: + models: Discovered model rows containing ``model_id`` and ``owned_by``. + transport: Provider request seam. + endpoint: OpenAI-compatible provider base endpoint. + api_key: In-memory credential value, never serialized. + request_budget: Shared hard provider-call cap. + probe_concurrency: Maximum simultaneous model workers. + clock: Provenance timestamp source. + timer: Per-probe monotonic latency source. + + Returns: + Sorted model rows with complete capability evidence for every model. + + Raises: + BenchmarkContractError: If concurrency is boolean or not positive. + """ + if ( + isinstance(probe_concurrency, bool) + or not isinstance(probe_concurrency, int) + or probe_concurrency < 1 + ): + raise BenchmarkContractError("probe_concurrency must be a positive integer") + + sorted_models = sorted(models, key=lambda row: row["model_id"]) + required_probe_requests = len(sorted_models) * len(CAPABILITY_PROBE_ORDER) + if required_probe_requests > request_budget.remaining_requests: + raise BenchmarkBudgetError( + f"complete capability probe plan needs {required_probe_requests} " + f"requests but only {request_budget.remaining_requests} remain" + ) + + def probe_one(model: dict[str, Any]) -> dict[str, Any]: + """Execute every preflighted capability cell for one model.""" + rows: list[dict[str, Any]] = [] + for capability_name in CAPABILITY_PROBE_ORDER: + request_budget.spend_or_fail() + rows.append( + execute_capability_probe( + transport, + endpoint, + api_key, + model["model_id"], + capability_name, + timer, + ) + ) + classified = classify_model_capabilities(rows) + return { + "model_id": model["model_id"], + "owned_by": model["owned_by"], + "endpoint": endpoint, + "discovered_at_unix": round(clock(), 3), + "capability_probe_rows": rows, + **classified, + } + + with ThreadPoolExecutor(max_workers=probe_concurrency) as executor: + results = list(executor.map(probe_one, sorted_models)) + return results + + +# -------------------------------------------------------------------------- +# Task manifest, scorers, pricing scenario +# -------------------------------------------------------------------------- + + +def score_exact_number_match(expected: dict[str, Any], answer_text: str) -> float: + """1.0 when the exact expected number appears as a standalone number in the answer. + + A trailing sentence period ("the answer is 21.") still matches; being part + of a longer number ("210", "21.5", "121") never does. + """ + pattern = rf"(? float: + """1.0 when the expected substring appears (case-insensitive) in the answer.""" + return 1.0 if str(expected["substring"]).lower() in answer_text.lower() else 0.0 + + +SCORER_REGISTRY: dict[tuple[str, str], Callable[[dict[str, Any], str], float]] = { + ("exact_number_match", "1"): score_exact_number_match, + ("substring_match", "1"): score_substring_match, +} + +_VALID_TASK_SPLITS = frozenset({"locked", "exploratory"}) + + +def load_task_manifest(path: str) -> dict[str, Any]: + """Load and validate the versioned task manifest, rejecting leakage and drift. + + Enforces: a manifest version, unique immutable snake_case task ids, known + splits, registered scorer name+version pairs, and the no-leakage rule that + an expected answer value never appears inside its own task prompt. + """ + with open(path, "r", encoding="utf-8") as handle: + try: + manifest = json.load(handle) + except ValueError as exc: + raise BenchmarkContractError(f"task manifest is not valid JSON: {exc}") from exc + if not isinstance(manifest, dict) or not isinstance(manifest.get("manifest_version"), str): + raise BenchmarkContractError("task manifest must be an object with a string 'manifest_version'") + tasks = manifest.get("tasks") + if not isinstance(tasks, list) or not tasks: + raise BenchmarkContractError("task manifest must carry a non-empty 'tasks' list") + seen_task_ids: set[str] = set() + for task in tasks: + if not isinstance(task, dict): + raise BenchmarkContractError("every task manifest entry must be an object") + task_id = task.get("task_id") + if not isinstance(task_id, str) or not is_two_word_snake_case(task_id): + raise BenchmarkContractError(f"task_id must be two-plus-word snake_case: {task_id!r}") + if task_id in seen_task_ids: + raise BenchmarkContractError(f"duplicate task_id in manifest: {task_id!r}") + seen_task_ids.add(task_id) + if task.get("split") not in _VALID_TASK_SPLITS: + raise BenchmarkContractError(f"task {task_id!r} split must be 'locked' or 'exploratory'") + prompt = task.get("prompt") + if not isinstance(prompt, str) or not prompt.strip(): + raise BenchmarkContractError(f"task {task_id!r} must carry a non-empty prompt") + scorer = task.get("scorer") + if not isinstance(scorer, dict): + raise BenchmarkContractError(f"task {task_id!r} must carry a scorer object") + scorer_key = (str(scorer.get("name")), str(scorer.get("version"))) + if scorer_key not in SCORER_REGISTRY: + raise BenchmarkContractError(f"task {task_id!r} names an unregistered scorer: {scorer_key}") + expected = task.get("expected") + if not isinstance(expected, dict) or not expected: + raise BenchmarkContractError(f"task {task_id!r} must carry a non-empty expected object") + # No-leakage rule, defined by the scorer itself: if the registered + # scorer would award the prompt text a point, the expected answer has + # leaked into the prompt and a prompt-echoing model would score. + if SCORER_REGISTRY[scorer_key](expected, prompt) != 0.0: + raise BenchmarkContractError( + f"task {task_id!r} leaks its expected answer into the prompt (test-set leakage)" + ) + return manifest + + +def locked_evaluation_tasks(manifest: dict[str, Any]) -> list[dict[str, Any]]: + """Return only the locked evaluation split, in manifest order.""" + return [task for task in manifest["tasks"] if task["split"] == "locked"] + + +def _require_finite_rate(value: Any, label: str) -> float: + """Validate one USD-per-million-token rate: a finite, non-negative number.""" + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0: + raise BenchmarkContractError(f"pricing scenario rate {label} must be a finite non-negative number") + return float(value) + + +_REVIEWED_PRICING_FIELDS = ( + "source_url", + "reviewed_by", + "reviewed_at_date", + "valid_until_date", + "rate_basis", + "uncertainty", +) + + +def _parse_evidence_date(value: Any, field_name: str) -> datetime_module.date: + """Parse one ISO evidence date or raise a field-specific contract error.""" + if not isinstance(value, str): + raise BenchmarkContractError( + f"pricing scenario {field_name} must be an ISO date string" + ) + try: + return datetime_module.date.fromisoformat(value) + except ValueError as exc: + raise BenchmarkContractError( + f"pricing scenario {field_name} must be a valid ISO date" + ) from exc + + +def _validate_reviewed_pricing_metadata(scenario: dict[str, Any]) -> None: + """Require complete provenance for a scenario labeled ``reviewed``.""" + missing = [field for field in _REVIEWED_PRICING_FIELDS if field not in scenario] + if missing: + raise BenchmarkContractError( + f"reviewed pricing scenario is missing fields: {missing}" + ) + parsed_source = urllib.parse.urlparse(str(scenario["source_url"])) + if parsed_source.scheme != "https" or not parsed_source.hostname: + raise BenchmarkContractError( + "reviewed pricing scenario source_url must use https" + ) + for field_name in ("reviewed_by", "rate_basis", "uncertainty"): + value = scenario[field_name] + if not isinstance(value, str) or not value.strip(): + raise BenchmarkContractError( + f"reviewed pricing scenario {field_name} must be non-empty" + ) + reviewed_at = _parse_evidence_date(scenario["reviewed_at_date"], "reviewed_at_date") + valid_until = _parse_evidence_date(scenario["valid_until_date"], "valid_until_date") + if valid_until < reviewed_at: + raise BenchmarkContractError( + "reviewed pricing scenario valid_until_date precedes reviewed_at_date" + ) + + +def validate_live_pricing_scenario( + scenario: dict[str, Any] | None, + today: datetime_module.date | None = None, +) -> None: + """Fail before egress when supplied live price evidence is not current. + + Omitting a scenario is valid and leaves every hypothetical cost ``unknown``. + Supplying one requires an explicit reviewed status, complete provenance, and + a validity horizon that includes the run date. + """ + if scenario is None: + return + if scenario.get("scenario_status") != "reviewed": + raise BenchmarkContractError( + "live benchmark pricing scenario must be independently reviewed" + ) + _validate_reviewed_pricing_metadata(scenario) + observed_date = today or datetime_module.date.today() + reviewed_at = _parse_evidence_date(scenario["reviewed_at_date"], "reviewed_at_date") + valid_until = _parse_evidence_date(scenario["valid_until_date"], "valid_until_date") + if reviewed_at > observed_date: + raise BenchmarkContractError("reviewed pricing evidence is dated in the future") + if observed_date > valid_until: + raise BenchmarkContractError("reviewed pricing evidence expired") + + +def load_pricing_scenario(path: str | None) -> dict[str, Any] | None: + """Load and validate one explicit hypothetical price-assumption file. + + ``None`` is legal and keeps hypothetical costs ``unknown``. Rates are never + inferred: only finite non-negative input/output USD-per-million-token values + explicitly present in the supplied file are accepted. A scenario labeled + ``reviewed`` must also carry complete source and validity metadata. + + Args: + path: JSON scenario path, or ``None`` to omit paid-cost assumptions. + + Returns: + Validated scenario dictionary or ``None``. + + Raises: + BenchmarkContractError: If JSON, status, provenance, or rates are invalid. + """ + if path is None: + return None + with open(path, "r", encoding="utf-8") as handle: + try: + scenario = json.load(handle) + except ValueError as exc: + raise BenchmarkContractError( + f"pricing scenario is not valid JSON: {exc}" + ) from exc + if not isinstance(scenario, dict) or not isinstance( + scenario.get("scenario_version"), str + ): + raise BenchmarkContractError( + "pricing scenario must be an object with a string 'scenario_version'" + ) + if scenario.get("scenario_status") not in ("example_unreviewed", "reviewed"): + raise BenchmarkContractError( + "pricing scenario_status must be 'example_unreviewed' or 'reviewed'" + ) + rates = scenario.get("usd_per_million_tokens") + if not isinstance(rates, dict): + raise BenchmarkContractError( + "pricing scenario must carry a 'usd_per_million_tokens' object" + ) + for model_id, rate in rates.items(): + if not isinstance(model_id, str) or not model_id.strip(): + raise BenchmarkContractError("pricing model id must be a non-empty string") + if not isinstance(rate, dict): + raise BenchmarkContractError( + f"pricing entry for {model_id!r} must be an object" + ) + _require_finite_rate(rate.get("input"), f"{model_id}.input") + _require_finite_rate(rate.get("output"), f"{model_id}.output") + if scenario["scenario_status"] == "reviewed": + _validate_reviewed_pricing_metadata(scenario) + return scenario + + +def hypothetical_cost_usd( + pricing_scenario: dict[str, Any] | None, + usage_by_model: dict[str, dict[str, int]], +) -> float | str: + """Cost under the pricing scenario, or ``"unknown"`` when any model is unpriced. + + ``usage_by_model`` maps model id to its prompt/completion token counts for + one cell. No authoritative rate for a used model means the whole cell is + honestly ``"unknown"`` — a partial sum would understate cost. + """ + if pricing_scenario is None: + return "unknown" + rates = pricing_scenario["usd_per_million_tokens"] + total = 0.0 + for model_id, usage in usage_by_model.items(): + rate = rates.get(model_id) + if rate is None: + return "unknown" + total += usage["prompt_tokens"] * float(rate["input"]) / 1_000_000 + total += usage["completion_tokens"] * float(rate["output"]) / 1_000_000 + return round(total, 10) + + +# -------------------------------------------------------------------------- +# Policy evaluation +# -------------------------------------------------------------------------- + + +def sanitize_worker_agent_id(model_id: str, taken_ids: set[str]) -> str: + """Deterministically derive a convention-compliant agent id from a model id.""" + base = re.sub(r"[^a-z0-9]+", "_", model_id.lower()).strip("_") or "unnamed_model" + if not is_two_word_snake_case(base): + base = f"nim_{base}" + candidate = base + suffix = 2 + while candidate in taken_ids: + candidate = f"{base}_{suffix}" + suffix += 1 + taken_ids.add(candidate) + return candidate + + +def build_worker_agents( + probed_models: list[dict[str, Any]], + base_url: str, + max_eval_models: int, +) -> list[ModelAgent]: + """Build the evaluation worker pool from chat-eligible probed models. + + Deterministic: models are already sorted by id; the pool is capped at + ``max_eval_models`` so a huge catalog cannot silently explode the budget. + """ + if max_eval_models < 1: + raise BenchmarkContractError("max_eval_models must be a positive integer") + taken_ids: set[str] = set() + agents: list[ModelAgent] = [] + for row in probed_models: + if not row["chat_eligible"]: + continue + if len(agents) >= max_eval_models: + break + agents.append( + ModelAgent( + id=sanitize_worker_agent_id(row["model_id"], taken_ids), + model=row["model_id"], + base_url=base_url, + credential_key=NIM_CREDENTIAL_NAME, + tags=("reasoning", "writing"), + ) + ) + return agents + + +def _coerce_token_count(value: Any) -> int | None: + """Defensively coerce a provider-reported token count; ``None`` when unusable. + + Guards the adversarial cases: booleans, non-numbers, NaN/inf, negatives. + """ + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + if not math.isfinite(value) or value < 0: + return None + return int(value) + + +def _cell_usage( + trace: list[dict[str, Any]], + agents_by_id: dict[str, str], + task_prompt: str, +) -> tuple[dict[str, dict[str, int]], dict[str, Any]]: + """Aggregate per-model token usage for one cell, labeling its source honestly. + + Provider-reported usage wins; steps without usable reported numbers fall + back to the repo's character-length estimate and mark the whole cell + ``estimated`` (never silently mixed into ``reported``). + """ + usage_by_model: dict[str, dict[str, int]] = {} + any_estimated = False + models_used: list[dict[str, Any]] = [] + for row in trace: + agent_id = row.get("served_agent_id") or row["agent_id"] + model_id = agents_by_id[agent_id] + models_used.append( + {"step_id": row["id"], "role": row["role"], "agent_id": agent_id, "model_id": model_id} + ) + usage = row.get("usage") if isinstance(row.get("usage"), dict) else {} + prompt_tokens = _coerce_token_count(usage.get("prompt_tokens")) + completion_tokens = _coerce_token_count(usage.get("completion_tokens")) + if prompt_tokens is None: + prompt_tokens = estimate_tokens(task_prompt) + any_estimated = True + if completion_tokens is None: + completion_tokens = estimate_tokens(row.get("output") or "") + any_estimated = True + bucket = usage_by_model.setdefault(model_id, {"prompt_tokens": 0, "completion_tokens": 0}) + bucket["prompt_tokens"] += prompt_tokens + bucket["completion_tokens"] += completion_tokens + prompt_total = sum(bucket["prompt_tokens"] for bucket in usage_by_model.values()) + completion_total = sum(bucket["completion_tokens"] for bucket in usage_by_model.values()) + summary = { + "prompt_tokens": prompt_total, + "completion_tokens": completion_total, + "total_tokens": prompt_total + completion_total, + "token_usage_source": "estimated" if any_estimated else "reported", + "models_used": models_used, + } + return usage_by_model, summary + + +def _classify_run_error(exc: Exception) -> str: + """Split a failed policy run into the issue's timeout vs failure classes.""" + causes = {type(exc), type(exc.__cause__)} + if causes & {TimeoutError, socket.timeout}: + return "timeout" + return "failure" + + +def run_policy_cell( + policy_name: str, + task: dict[str, Any], + run_callable: Callable[[], dict[str, Any]], + agents_by_id: dict[str, str], + pricing_scenario: dict[str, Any] | None, + timer: Callable[[], float], +) -> dict[str, Any]: + """Execute one policy on one task and record the full evidence cell.""" + scorer = task["scorer"] + started = timer() + try: + result = run_callable() + except (BenchmarkBudgetError, BenchmarkAuthError): + # Budget exhaustion and credential rejection must abort the whole run + # (fail closed), never degrade into one quietly failed cell. + raise + except Exception as exc: # noqa: BLE001 - classified into the contract outcomes + return { + "policy_name": policy_name, + "task_id": task["task_id"], + "task_split": task["split"], + "scorer_name": scorer["name"], + "scorer_version": scorer["version"], + "task_score": None, + "run_outcome": _classify_run_error(exc), + "outcome_reason": f"{type(exc).__name__}", + "end_to_end_latency_ms": round((timer() - started) * 1000, 3), + "provider_latency_ms": None, + "call_count": 0, + "workflow_depth": 0, + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "token_usage_source": "unavailable", + "actual_cost_usd": 0.0, + "hypothetical_cost_usd": "unknown", + "models_used": [], + "response_sha256": None, + } + elapsed_ms = round((timer() - started) * 1000, 3) + answer = result.get("answer") or "" + scorer_fn = SCORER_REGISTRY[(scorer["name"], scorer["version"])] + trace = result.get("trace") or [] + usage_by_model, usage_summary = _cell_usage(trace, agents_by_id, task["prompt"]) + return { + "policy_name": policy_name, + "task_id": task["task_id"], + "task_split": task["split"], + "scorer_name": scorer["name"], + "scorer_version": scorer["version"], + "task_score": scorer_fn(task["expected"], answer), + "run_outcome": "success", + "outcome_reason": "completed", + "end_to_end_latency_ms": elapsed_ms, + # Provider-side latency is not observable through the OpenAI-compatible + # response body; recorded as None rather than a fabricated number. + "provider_latency_ms": None, + "call_count": len(trace), + "workflow_depth": len(trace), + "prompt_tokens": usage_summary["prompt_tokens"], + "completion_tokens": usage_summary["completion_tokens"], + "total_tokens": usage_summary["total_tokens"], + "token_usage_source": usage_summary["token_usage_source"], + # Actual cost of the hosted NIM catalog to the caller is zero today; + # hypothetical paid cost comes only from the explicit scenario. + "actual_cost_usd": 0.0, + "hypothetical_cost_usd": hypothetical_cost_usd(pricing_scenario, usage_by_model), + "models_used": usage_summary["models_used"], + "response_sha256": hashlib.sha256(answer.encode("utf-8")).hexdigest(), + } + + +def _combined_rate(pricing_scenario: dict[str, Any], model_id: str) -> float | None: + """Combined input+output USD/1M rate for cheapest-worker selection, or ``None``.""" + rate = pricing_scenario["usd_per_million_tokens"].get(model_id) + if rate is None: + return None + return float(rate["input"]) + float(rate["output"]) + + +def cheapest_priced_agent( + agents: list[ModelAgent], pricing_scenario: dict[str, Any] | None +) -> ModelAgent | None: + """The cheapest scenario-priced worker (deterministic tiebreak by model id).""" + if pricing_scenario is None: + return None + priced = [ + (rate, agent.model, agent) + for agent in agents + for rate in [_combined_rate(pricing_scenario, agent.model)] + if rate is not None + ] + if not priced: + return None + return min(priced, key=lambda row: (row[0], row[1]))[2] + + +def planned_evaluation_requests(worker_count: int, locked_task_count: int) -> int: + """Upper bound on evaluation calls, checked pre-flight so the run fails closed. + + Direct baselines: one call per worker per task; ``route_once``: one call + per task; ``conduct``: at most :data:`MAX_WORKFLOW_DEPTH` calls per task; + cheapest-eligible: one call per task. + """ + return locked_task_count * (worker_count + 1 + MAX_WORKFLOW_DEPTH + 1) + + +def plan_complete_request_budget( + discovered_model_count: int, + max_eval_models: int, + locked_task_count: int, +) -> dict[str, int]: + """Return the complete conservative request plan for one catalog snapshot. + + The plan reserves one catalog request, every model-capability probe, and + the worst-case equal-budget evaluation envelope for every worker that may + enter the capped evaluation pool. It is intentionally conservative: fewer + chat-eligible or scenario-priced workers may leave requests unused, but a + live run never starts a biased partial probe phase. + + Args: + discovered_model_count: Usable model ids returned by ``/v1/models``. + max_eval_models: Maximum workers allowed into policy evaluation. + locked_task_count: Number of locked benchmark tasks. + + Returns: + Named request counts including the complete run total. + + Raises: + BenchmarkContractError: If any count is boolean or not positive. + """ + counts = { + "discovered_model_count": discovered_model_count, + "max_eval_models": max_eval_models, + "locked_task_count": locked_task_count, + } + for label, value in counts.items(): + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise BenchmarkContractError( + f"{label} must be a positive integer" + ) + planned_worker_count = min(discovered_model_count, max_eval_models) + capability_probe_request_count = ( + discovered_model_count * len(CAPABILITY_PROBE_ORDER) + ) + evaluation_reserve_request_count = planned_evaluation_requests( + planned_worker_count, + locked_task_count, + ) + return { + "catalog_request_count": 1, + "capability_probe_request_count": capability_probe_request_count, + "evaluation_reserve_request_count": evaluation_reserve_request_count, + "planned_worker_count": planned_worker_count, + "total_required_request_count": ( + 1 + + capability_probe_request_count + + evaluation_reserve_request_count + ), + } + + +def planned_complete_run_requests( + model_count: int, + locked_task_count: int, + max_eval_models: int, +) -> dict[str, int]: + """Return buyer-facing request counts for a complete benchmark run. + + This stable planning view translates the internal conservative preflight + into terminology used by release acceptance, operator documentation, and + acquisition evidence. Validation remains centralized in + :func:`plan_complete_request_budget`, so both views fail closed identically. + + Args: + model_count: Usable model identifiers discovered from ``/v1/models``. + locked_task_count: Number of locked evaluation tasks. + max_eval_models: Maximum workers admitted to policy comparison. + + Returns: + Catalog, capability, evaluation, post-catalog, and total request counts. + """ + plan = plan_complete_request_budget( + discovered_model_count=model_count, + max_eval_models=max_eval_models, + locked_task_count=locked_task_count, + ) + requests_after_catalog = ( + plan["capability_probe_request_count"] + + plan["evaluation_reserve_request_count"] + ) + return { + "catalog_discovery_requests": plan["catalog_request_count"], + "capability_probe_requests": plan["capability_probe_request_count"], + "evaluation_worker_ceiling": plan["planned_worker_count"], + "evaluation_requests": plan["evaluation_reserve_request_count"], + "requests_after_catalog": requests_after_catalog, + "total_requests": plan["total_required_request_count"], + } + + +def evaluate_policies( + agents: list[ModelAgent], + manifest: dict[str, Any], + pricing_scenario: dict[str, Any] | None, + client: ModelClient, + request_budget: RequestBudget, + timer: Callable[[], float] = time.perf_counter, + total_token_budget: int = 256, + maximum_calls: int = MAX_WORKFLOW_DEPTH, +) -> dict[str, Any]: + """Run every compared policy with equal cell-level token and call budgets. + + Every policy/task cell receives a fresh orchestrator and limiter so traces, + usage, call counts, and allowances never bleed across tasks or policy arms. + + Args: + agents: Chat-eligible workers selected from capability probes. + manifest: Validated task manifest. + pricing_scenario: Optional explicit hypothetical price assumptions. + client: Shared request-budgeted model client. + request_budget: Complete-run provider request cap. + timer: Monotonic latency source. + total_token_budget: Equal prompt-plus-completion allowance per cell. + maximum_calls: Equal declared provider-call envelope per cell. + + Returns: + Evaluation cells and pool/task metadata. + + Raises: + BenchmarkContractError: If no workers or locked tasks are available. + BenchmarkBudgetError: If the complete evaluation cannot fit the run cap. + """ + if not agents: + raise BenchmarkContractError( + "policy evaluation requires at least one chat-eligible worker" + ) + tasks = locked_evaluation_tasks(manifest) + if not tasks: + raise BenchmarkContractError("task manifest has no locked evaluation tasks") + planned = planned_evaluation_requests(len(agents), len(tasks)) + if planned > request_budget.remaining_requests: + raise BenchmarkBudgetError( + f"planned evaluation needs up to {planned} requests but only " + f"{request_budget.remaining_requests} remain in the budget" + ) + + agents_by_id = {agent.id: agent.model for agent in agents} + depth_policy = dataclasses.replace( + OrchestrationPolicy(), + max_workflow_steps=MAX_WORKFLOW_DEPTH, + ) + + def run_cell( + policy_name: str, + task: dict[str, Any], + pool: list[ModelAgent], + mode: str, + ) -> dict[str, Any]: + """Run one independent policy/task cell and append budget evidence.""" + cell_client = EqualBudgetModelClient( + client, + total_token_budget, + maximum_calls, + ) + orchestrator = TaskOrchestrator(pool, client=cell_client) + orchestrator.policy = depth_policy + cell = run_policy_cell( + policy_name, + task, + lambda: orchestrator.complete( + [{"role": "user", "content": task["prompt"]}], + mode=mode, + ), + agents_by_id, + pricing_scenario, + timer, + ) + cell.update( + { + "configured_total_token_budget": total_token_budget, + "configured_maximum_calls": maximum_calls, + "observed_budget_tokens": cell_client.observed_tokens, + "observed_budget_calls": cell_client.observed_calls, + "remaining_budget_tokens": cell_client.remaining_tokens, + } + ) + if cell_client.exceeded: + cell["run_outcome"] = "failure" + cell["outcome_reason"] = "observed_usage_exceeded_equal_token_budget" + cell["task_score"] = None + return cell + + cells: list[dict[str, Any]] = [] + for agent in agents: + for task in tasks: + cells.append( + run_cell( + f"direct_single_worker:{agent.model}", + task, + [agent], + "route", + ) + ) + for task in tasks: + cells.append(run_cell("route_once", task, agents, "route")) + cells.append(run_cell("conduct_bounded", task, agents, "conduct")) + + cheapest_skip_reason = None + cheapest = cheapest_priced_agent(agents, pricing_scenario) + if cheapest is None: + cheapest_skip_reason = ( + "no_pricing_scenario_supplied" + if pricing_scenario is None + else "no_worker_priced_by_scenario" + ) + else: + for task in tasks: + cells.append( + run_cell( + "cheapest_eligible_worker", + task, + [cheapest], + "route", + ) + ) + cells.sort(key=lambda cell: (cell["policy_name"], cell["task_id"])) + return { + "evaluation_cells": cells, + "cheapest_worker_skip_reason": cheapest_skip_reason, + "locked_task_count": len(tasks), + "worker_count": len(agents), + } + + +# -------------------------------------------------------------------------- +# Statistics: paired bootstrap + Pareto frontiers +# -------------------------------------------------------------------------- + + +def paired_bootstrap_mean_difference( + paired_scores: list[tuple[float, float]], + iterations: int = 2000, + seed: int = 7, +) -> dict[str, Any]: + """Paired bootstrap CI for mean(score_a - score_b) over shared tasks.""" + if not paired_scores: + raise BenchmarkContractError("paired bootstrap requires at least one score pair") + differences = [a - b for a, b in paired_scores] + rng = random.Random(seed) + resampled_means = sorted( + sum(rng.choice(differences) for _ in differences) / len(differences) for _ in range(iterations) + ) + lower_index = int(0.025 * (iterations - 1)) + upper_index = int(0.975 * (iterations - 1)) + return { + "mean_difference": round(sum(differences) / len(differences), 6), + "ci_low": round(resampled_means[lower_index], 6), + "ci_high": round(resampled_means[upper_index], 6), + "iterations": iterations, + "seed": seed, + "pair_count": len(differences), + "method": "paired_bootstrap_percentile_95", + } + + +def pareto_frontier( + rows: list[dict[str, Any]], quality_key: str, cost_key: str +) -> list[dict[str, Any]]: + """Rows not dominated on (``quality_key`` up, ``cost_key`` down).""" + frontier = [ + a + for a in rows + if not any( + b is not a + and b[quality_key] >= a[quality_key] + and b[cost_key] <= a[cost_key] + and (b[quality_key] > a[quality_key] or b[cost_key] < a[cost_key]) + for b in rows + ) + ] + return sorted(frontier, key=lambda row: (-row[quality_key], row[cost_key])) + + +def summarize_policies(cells: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Aggregate evaluation cells per policy with honest unknown-cost labeling.""" + grouped: dict[str, list[dict[str, Any]]] = {} + for cell in cells: + grouped.setdefault(cell["policy_name"], []).append(cell) + summaries = [] + for policy_name in sorted(grouped): + policy_cells = grouped[policy_name] + scored = [cell for cell in policy_cells if cell["run_outcome"] == "success"] + priced = [cell for cell in scored if isinstance(cell["hypothetical_cost_usd"], float)] + mean_score = round(sum(cell["task_score"] for cell in scored) / len(scored), 6) if scored else 0.0 + summaries.append( + { + "policy_name": policy_name, + "cell_count": len(policy_cells), + "success_count": len(scored), + "mean_task_score": mean_score, + "mean_latency_ms": round( + sum(cell["end_to_end_latency_ms"] for cell in policy_cells) / len(policy_cells), 3 + ), + "total_call_count": sum(cell["call_count"] for cell in policy_cells), + "max_workflow_depth": max(cell["workflow_depth"] for cell in policy_cells), + "total_tokens": sum(cell["total_tokens"] for cell in policy_cells), + "actual_cost_usd": 0.0, + "mean_hypothetical_cost_usd": ( + round(sum(cell["hypothetical_cost_usd"] for cell in priced) / len(priced), 10) + if priced + else "unknown" + ), + "unknown_hypothetical_cost_cells": len(scored) - len(priced), + } + ) + return summaries + + +def best_single_worker_hindsight(summaries: list[dict[str, Any]]) -> dict[str, Any] | None: + """The best direct single worker selected in hindsight on the locked split.""" + direct = [row for row in summaries if row["policy_name"].startswith("direct_single_worker:")] + if not direct: + return None + best = max(direct, key=lambda row: (row["mean_task_score"], row["policy_name"])) + return { + "policy_name": best["policy_name"], + "model_id": best["policy_name"].split(":", 1)[1], + "mean_task_score": best["mean_task_score"], + "selection_basis": "hindsight_argmax_mean_locked_score", + } + + +def paired_policy_comparisons(cells: list[dict[str, Any]], seed: int) -> list[dict[str, Any]]: + """Paired task-level bootstrap comparisons between the headline policies.""" + scores: dict[str, dict[str, float]] = {} + for cell in cells: + if cell["run_outcome"] == "success": + scores.setdefault(cell["policy_name"], {})[cell["task_id"]] = cell["task_score"] + summaries = summarize_policies(cells) + hindsight = best_single_worker_hindsight(summaries) + comparison_pairs = [("conduct_bounded", "route_once"), ("cheapest_eligible_worker", "route_once")] + if hindsight is not None: + comparison_pairs.append(("route_once", hindsight["policy_name"])) + comparison_pairs.append(("conduct_bounded", hindsight["policy_name"])) + comparisons = [] + for policy_a, policy_b in comparison_pairs: + tasks_a, tasks_b = scores.get(policy_a), scores.get(policy_b) + if not tasks_a or not tasks_b: + continue + shared_tasks = sorted(set(tasks_a) & set(tasks_b)) + if not shared_tasks: + continue + pairs = [(tasks_a[task_id], tasks_b[task_id]) for task_id in shared_tasks] + comparisons.append( + { + "policy_a": policy_a, + "policy_b": policy_b, + **paired_bootstrap_mean_difference(pairs, seed=seed), + } + ) + return comparisons + + +def _numeric_cost_rows(summaries: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Summaries whose mean hypothetical cost is numeric (unknowns excluded, labeled).""" + return [row for row in summaries if isinstance(row["mean_hypothetical_cost_usd"], float)] + + +def build_pareto_frontiers(summaries: list[dict[str, Any]]) -> dict[str, Any]: + """Quality-latency and quality-hypothetical-cost Pareto frontiers.""" + successful = [row for row in summaries if row["success_count"] > 0] + return { + "quality_vs_latency": pareto_frontier( + successful, + "mean_task_score", + "mean_latency_ms", + ), + "quality_vs_hypothetical_cost": pareto_frontier( + _numeric_cost_rows(successful), + "mean_task_score", + "mean_hypothetical_cost_usd", + ), + "excluded_unknown_cost_policies": sorted( + row["policy_name"] + for row in successful + if not isinstance(row["mean_hypothetical_cost_usd"], float) + ), + "excluded_zero_success_policies": sorted( + row["policy_name"] for row in summaries if row["success_count"] == 0 + ), + } + + +def _validate_actual_cost_evidence(report: dict[str, Any]) -> None: + """Require complete official provenance for the zero access-cost claim.""" + evidence = report.get("actual_cost_evidence") + if not isinstance(evidence, dict): + raise BenchmarkContractError( + "benchmark report is missing actual_cost_evidence" + ) + required_fields = ( + "evidence_schema_version", + "source_title", + "source_url", + "reviewed_at_date", + "valid_until_date", + "access_program", + "access_scope", + "production_access_note", + "actual_cost_usd", + "uncertainty", + ) + missing = [field for field in required_fields if field not in evidence] + if missing: + raise BenchmarkContractError( + f"actual cost evidence is missing fields: {missing}" + ) + if evidence["actual_cost_usd"] != 0.0: + raise BenchmarkContractError( + "actual cost evidence must preserve the reviewed zero-cost value" + ) + if evidence["source_url"] != "https://docs.api.nvidia.com/nim/docs/product": + raise BenchmarkContractError( + "actual cost evidence must cite the reviewed NVIDIA NIM General FAQ" + ) + reviewed_at = _parse_evidence_date(evidence["reviewed_at_date"], "reviewed_at_date") + valid_until = _parse_evidence_date(evidence["valid_until_date"], "valid_until_date") + if valid_until < reviewed_at: + raise BenchmarkContractError( + "actual cost evidence validity precedes its review date" + ) + + +def _require_current_actual_cost_evidence( + today: datetime_module.date | None = None, +) -> None: + """Fail closed after the reviewed hosted-access validity horizon.""" + observed_date = today or datetime_module.date.today() + reviewed_at = _parse_evidence_date( + ACTUAL_COST_EVIDENCE["reviewed_at_date"], + "reviewed_at_date", + ) + valid_until = _parse_evidence_date( + ACTUAL_COST_EVIDENCE["valid_until_date"], + "valid_until_date", + ) + if observed_date < reviewed_at: + raise BenchmarkContractError( + "reviewed NVIDIA hosted-endpoint cost evidence is dated in the future" + ) + if observed_date > valid_until: + raise BenchmarkContractError( + "reviewed NVIDIA hosted-endpoint cost evidence expired; " + "re-review official terms" + ) + + +def _evaluation_evidence_summary( + cells: list[dict[str, Any]], + locked_task_count: int, +) -> dict[str, Any]: + """Classify whether benchmark evidence can inform production review.""" + successful_cells = [cell for cell in cells if cell["run_outcome"] == "success"] + completion_fraction = ( + round(len(successful_cells) / len(cells), 6) if cells else 0.0 + ) + successful_tasks_by_policy: dict[str, set[str]] = {} + for cell in successful_cells: + successful_tasks_by_policy.setdefault(cell["policy_name"], set()).add( + cell["task_id"] + ) + paired_task_ids = successful_tasks_by_policy.get("route_once", set()) & ( + successful_tasks_by_policy.get("conduct_bounded", set()) + ) + sufficient = ( + locked_task_count >= MINIMUM_PAIRED_TASK_COUNT + and len(paired_task_ids) >= MINIMUM_PAIRED_TASK_COUNT + and completion_fraction >= REQUIRED_COMPLETION_FRACTION + ) + return { + "evidence_status": ( + "evidence_review_required" if sufficient else "insufficient_evidence" + ), + "decision_use": ( + "production_candidate_review" if sufficient else "benchmark_smoke_only" + ), + "minimum_paired_task_count": MINIMUM_PAIRED_TASK_COUNT, + "required_completion_fraction": REQUIRED_COMPLETION_FRACTION, + "observed_locked_task_count": locked_task_count, + "observed_paired_task_count": len(paired_task_ids), + "observed_completion_fraction": completion_fraction, + "routing_recommendation": None, + } + + +# -------------------------------------------------------------------------- +# Provenance, report schema, artifacts +# -------------------------------------------------------------------------- + + +def sha256_of_file(path: str) -> str: + """Hex SHA-256 of a file's bytes (manifest/pricing provenance hashes).""" + with open(path, "rb") as handle: + return hashlib.sha256(handle.read()).hexdigest() + + +def sha256_of_json(value: Any) -> str: + """Hex SHA-256 of a canonical JSON serialization (catalog snapshot hash).""" + return hashlib.sha256(json.dumps(value, sort_keys=True, ensure_ascii=False).encode("utf-8")).hexdigest() + + +def build_provenance( + run_mode: str, + git_sha: str, + workflow_run_id: str, + catalog_snapshot: dict[str, Any], + task_manifest_path: str, + pricing_scenario_path: str | None, + benchmark_parameters: dict[str, Any], +) -> dict[str, Any]: + """Assemble the provenance block; live runs fail closed on missing identity.""" + if run_mode == "live" and (not git_sha or not workflow_run_id): + raise BenchmarkContractError("live runs require --git-sha and --workflow-run-id provenance") + return { + "run_mode": run_mode, + "git_sha": git_sha or DRY_RUN_PROVENANCE_PLACEHOLDER, + "workflow_run_id": workflow_run_id or DRY_RUN_PROVENANCE_PLACEHOLDER, + "catalog_snapshot_sha256": sha256_of_json(catalog_snapshot), + "task_manifest_sha256": sha256_of_file(task_manifest_path), + "pricing_scenario_sha256": ( + sha256_of_file(pricing_scenario_path) if pricing_scenario_path else None + ), + "benchmark_parameters": benchmark_parameters, + } + + +_REPORT_REQUIRED_PATHS = ( + "benchmark_schema_version", + "provenance.run_mode", + "provenance.git_sha", + "provenance.workflow_run_id", + "provenance.catalog_snapshot_sha256", + "provenance.task_manifest_sha256", + "provenance.benchmark_parameters", + "catalog_snapshot.endpoint", + "catalog_snapshot.discovered_model_count", + "catalog_snapshot.duplicate_model_ids", + "catalog_snapshot.invalid_entries", + "catalog_snapshot.probed_models", + "capability_summary", + "evaluation.evaluation_cells", + "evaluation.policy_summaries", + "evaluation.paired_comparisons", + "evaluation.pareto_frontiers", + "evaluation.evidence_status", + "evaluation.decision_use", + "evaluation.minimum_paired_task_count", + "evaluation.required_completion_fraction", + "evaluation.observed_paired_task_count", + "evaluation.observed_completion_fraction", + "evaluation.routing_recommendation", + "request_budget.max_total_requests", + "request_budget.requests_spent", + "request_budget.planned_total_requests", + "request_budget.catalog_requests", + "request_budget.capability_probe_requests", + "request_budget.evaluation_reserve_requests", + "request_budget.planned_worker_count", + "actual_cost_evidence", + "honesty_labels.actual_cost_basis", + "honesty_labels.provider_latency_source", + "honesty_labels.hypothetical_cost_source", +) + + +def validate_report_schema(report: dict[str, Any]) -> None: + """Fail closed when any required report path is absent.""" + missing = [] + for path in _REPORT_REQUIRED_PATHS: + node: Any = report + for key in path.split("."): + if not isinstance(node, dict) or key not in node: + missing.append(path) + break + node = node[key] + if missing: + raise BenchmarkContractError(f"benchmark report is missing required paths: {missing}") + + +_CSV_CELL_COLUMNS = ( + "policy_name", + "task_id", + "task_split", + "scorer_name", + "scorer_version", + "task_score", + "run_outcome", + "outcome_reason", + "end_to_end_latency_ms", + "provider_latency_ms", + "call_count", + "workflow_depth", + "prompt_tokens", + "completion_tokens", + "total_tokens", + "token_usage_source", + "configured_total_token_budget", + "configured_maximum_calls", + "observed_budget_tokens", + "observed_budget_calls", + "remaining_budget_tokens", + "actual_cost_usd", + "hypothetical_cost_usd", + "response_sha256", +) + + +def _ensure_secret_absent(serialized: str) -> None: + """Refuse to write any artifact that contains the resolved provider secret.""" + secret = get_credential(NIM_CREDENTIAL_NAME) + if secret and secret in serialized: + raise SecretLeakError("benchmark artifact would contain the provider credential; refusing to write") + + +def render_markdown_summary(report: dict[str, Any]) -> str: + """Render a buyer-readable summary with evidence and cost caveats.""" + lines = [ + "# NIM cost-quality benchmark summary", + "", + f"- run mode: `{report['provenance']['run_mode']}`", + f"- git sha: `{report['provenance']['git_sha']}`", + f"- workflow run id: `{report['provenance']['workflow_run_id']}`", + f"- catalog snapshot sha256: `{report['provenance']['catalog_snapshot_sha256']}`", + f"- discovered models: {report['catalog_snapshot']['discovered_model_count']}", + f"- requests spent: {report['request_budget']['requests_spent']}" + f" / {report['request_budget']['max_total_requests']}", + f"- complete request plan: {report['request_budget']['planned_total_requests']} " + "(catalog + all capability probes + evaluation reserve)", + f"- evidence status: `{report['evaluation']['evidence_status']}`", + f"- decision use: `{report['evaluation']['decision_use']}`", + "", + "## Capability classifications", + "", + "| classification | models |", + "| --- | --- |", + ] + for classification, count in sorted(report["capability_summary"].items()): + lines.append(f"| {classification} | {count} |") + lines += [ + "", + "## Policy summaries (locked split)", + "", + "| policy | mean score | mean latency ms | mean hypothetical cost USD | actual cost USD |", + "| --- | --- | --- | --- | --- |", + ] + for row in report["evaluation"]["policy_summaries"]: + lines.append( + f"| {row['policy_name']} | {row['mean_task_score']} | " + f"{row['mean_latency_ms']} | {row['mean_hypothetical_cost_usd']} " + f"| {row['actual_cost_usd']} |" + ) + lines += ["", "## Paired comparisons (95% bootstrap CI)", ""] + for comparison in report["evaluation"]["paired_comparisons"]: + lines.append( + f"- `{comparison['policy_a']}` vs `{comparison['policy_b']}`: " + f"mean diff {comparison['mean_difference']} " + f"[{comparison['ci_low']}, {comparison['ci_high']}]" + ) + evidence = report["actual_cost_evidence"] + lines += [ + "", + "## Evidence sufficiency", + "", + f"- paired tasks: {report['evaluation']['observed_paired_task_count']} " + f"/ {report['evaluation']['minimum_paired_task_count']} required", + f"- completion fraction: {report['evaluation']['observed_completion_fraction']} " + f"/ {report['evaluation']['required_completion_fraction']} required", + "- production routing recommendation: none" + if report["evaluation"]["routing_recommendation"] is None + else f"- production routing recommendation: {report['evaluation']['routing_recommendation']}", + "", + "## Actual API access-cost evidence", + "", + f"- source: {evidence['source_title']}", + f"- reviewed: {evidence['reviewed_at_date']}", + f"- valid until: {evidence['valid_until_date']}", + f"- access context: {evidence['access_program']} — {evidence['access_scope']}", + f"- production distinction: {evidence['production_access_note']}", + f"- uncertainty: {evidence['uncertainty']}", + "", + "## Honesty labels", + "", + f"- actual cost basis: {report['honesty_labels']['actual_cost_basis']}", + f"- provider latency: {report['honesty_labels']['provider_latency_source']}", + f"- hypothetical cost source: {report['honesty_labels']['hypothetical_cost_source']}", + "", + ] + return "\n".join(lines) + + +def write_benchmark_artifacts( + report: dict[str, Any], + output_dir: str, +) -> dict[str, str]: + """Validate cost evidence and schema, then write JSON, CSV, and Markdown.""" + _validate_actual_cost_evidence(report) + validate_report_schema(report) + os.makedirs(output_dir, exist_ok=True) + json_text = json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + _ensure_secret_absent(json_text) + json_path = os.path.join(output_dir, "benchmark_report.json") + with open(json_path, "w", encoding="utf-8", newline="\n") as handle: + handle.write(json_text + "\n") + + csv_path = os.path.join(output_dir, "benchmark_cells.csv") + csv_buffer = io.StringIO() + writer = csv.DictWriter( + csv_buffer, + fieldnames=_CSV_CELL_COLUMNS, + extrasaction="ignore", + ) + writer.writeheader() + for cell in report["evaluation"]["evaluation_cells"]: + writer.writerow(cell) + csv_text = csv_buffer.getvalue() + _ensure_secret_absent(csv_text) + with open(csv_path, "w", encoding="utf-8", newline="") as handle: + handle.write(csv_text) + + markdown_text = render_markdown_summary(report) + _ensure_secret_absent(markdown_text) + markdown_path = os.path.join(output_dir, "benchmark_summary.md") + with open(markdown_path, "w", encoding="utf-8", newline="\n") as handle: + handle.write(markdown_text) + return { + "json_path": json_path, + "csv_path": csv_path, + "markdown_path": markdown_path, + } + + +# -------------------------------------------------------------------------- +# Benchmark assembly (shared by dry and live runs) +# -------------------------------------------------------------------------- + + +def assemble_benchmark_report( + run_mode: str, + endpoint: str, + catalog: dict[str, Any], + probed_models: list[dict[str, Any]], + evaluation: dict[str, Any], + request_budget: RequestBudget, + provenance_inputs: dict[str, Any], + seed: int, +) -> dict[str, Any]: + """Assemble and validate the complete evidence-grade benchmark report.""" + cells = evaluation["evaluation_cells"] + summaries = summarize_policies(cells) + capability_summary: dict[str, int] = {} + for row in probed_models: + capability_summary[row["model_classification"]] = ( + capability_summary.get(row["model_classification"], 0) + 1 + ) + catalog_snapshot = { + "endpoint": endpoint, + "discovered_model_count": len(catalog["models"]), + "duplicate_model_ids": catalog["duplicate_model_ids"], + "invalid_entries": catalog["invalid_entries"], + "probed_models": probed_models, + } + evidence_summary = _evaluation_evidence_summary( + cells, + evaluation["locked_task_count"], + ) + report = { + "benchmark_schema_version": BENCHMARK_SCHEMA_VERSION, + "provenance": build_provenance( + run_mode, + provenance_inputs["git_sha"], + provenance_inputs["workflow_run_id"], + catalog_snapshot, + provenance_inputs["task_manifest_path"], + provenance_inputs["pricing_scenario_path"], + provenance_inputs["benchmark_parameters"], + ), + "catalog_snapshot": catalog_snapshot, + "capability_summary": capability_summary, + "evaluation": { + "evaluation_cells": cells, + "policy_summaries": summaries, + "best_single_worker_hindsight": best_single_worker_hindsight(summaries), + "paired_comparisons": paired_policy_comparisons(cells, seed=seed), + "pareto_frontiers": build_pareto_frontiers(summaries), + "cheapest_worker_skip_reason": evaluation[ + "cheapest_worker_skip_reason" + ], + "locked_task_count": evaluation["locked_task_count"], + "worker_count": evaluation["worker_count"], + **evidence_summary, + }, + "request_budget": { + "max_total_requests": request_budget.max_total_requests, + "requests_spent": request_budget.requests_spent, + "planned_total_requests": provenance_inputs["request_plan"][ + "total_required_request_count" + ], + "catalog_requests": provenance_inputs["request_plan"][ + "catalog_request_count" + ], + "capability_probe_requests": provenance_inputs["request_plan"][ + "capability_probe_request_count" + ], + "evaluation_reserve_requests": provenance_inputs["request_plan"][ + "evaluation_reserve_request_count" + ], + "planned_worker_count": provenance_inputs["request_plan"][ + "planned_worker_count" + ], + }, + "actual_cost_evidence": dict(ACTUAL_COST_EVIDENCE), + "honesty_labels": { + "actual_cost_basis": ( + "deterministic_dry_run_no_provider_egress" + if run_mode == "dry_run" + else "reviewed_nvidia_developer_program_hosted_endpoint_access" + ), + "provider_latency_source": ( + "not_observable_via_openai_compatible_body" + ), + "hypothetical_cost_source": ( + "explicit_versioned_pricing_scenario_or_unknown" + ), + "dry_run_scores_note": ( + "dry-run scores reflect deterministic mock echoes, not model quality" + ), + }, + } + _validate_actual_cost_evidence(report) + validate_report_schema(report) + return report + + +# -------------------------------------------------------------------------- +# Deterministic dry-run provider (all modality classes, no network) +# -------------------------------------------------------------------------- + +# Synthetic catalog covering every capability class the harness can emit, +# plus one duplicate id and one invalid entry to exercise catalog hygiene. +_DRY_RUN_MODEL_BEHAVIOR = { + "dryrun/chat-basic": {"chat_completion"}, + "dryrun/chat-vision": {"chat_completion", "image_understanding"}, + "dryrun/chat-omni": { + "chat_completion", + "image_understanding", + "video_understanding", + "audio_understanding", + }, + "dryrun/chat-video": {"chat_completion", "video_understanding"}, + "dryrun/embed-basic": {"text_embedding"}, + "dryrun/completion-legacy": {"text_completion"}, + "dryrun/responses-native": {"response_generation"}, + "dryrun/audio-transcribe": {"audio_transcription"}, + "dryrun/audio-speech": {"audio_speech"}, + "dryrun/throttled-model": "rate_limited", + "dryrun/outage-model": "unavailable", + "dryrun/legacy-unsupported": "unsupported", +} + + +def _dry_run_catalog_body() -> bytes: + """Serialized synthetic /v1/models body, including hygiene edge cases.""" + data = [{"id": model_id, "owned_by": "dryrun"} for model_id in _DRY_RUN_MODEL_BEHAVIOR] + data.append({"id": "dryrun/chat-basic", "owned_by": "dryrun"}) # duplicate id + data.append({"owned_by": "dryrun"}) # missing model id + return json.dumps({"object": "list", "data": data}).encode("utf-8") + + +def _dry_run_success_body(path: str) -> bytes: + """Minimal valid success body for each probed endpoint contract.""" + if path.endswith("/embeddings"): + return json.dumps({"data": [{"embedding": [0.0, 0.1]}]}).encode("utf-8") + if path.endswith("/responses"): + return json.dumps({"output_text": "OK"}).encode("utf-8") + if path.endswith("/audio/transcriptions"): + return json.dumps({"text": "ok"}).encode("utf-8") + if path.endswith("/audio/speech"): + return b"RIFFdryrunaudio" + return json.dumps({"choices": [{"message": {"content": "OK"}}]}).encode("utf-8") + + +def _dry_run_probe_capability(path: str, body: bytes | None) -> str: + """Infer which capability a dry-run probe request represents.""" + if path.endswith("/chat/completions"): + text = (body or b"").decode("utf-8") + if "image_url" in text: + return "image_understanding" + if "video_url" in text: + return "video_understanding" + if "input_audio" in text: + return "audio_understanding" + return "chat_completion" + for capability_name, spec in CAPABILITY_PROBES.items(): + if path.endswith(spec["path"]) and capability_name != "chat_completion": + return capability_name + raise CatalogDiscoveryError(f"dry-run transport received an unexpected path: {path}") + + +def build_dry_run_transport() -> ProviderTransport: + """In-process provider fake serving the synthetic all-modality catalog.""" + + def transport(method: str, url: str, headers: dict[str, str], body: bytes | None) -> tuple[int, bytes]: + """Serve catalog and probe requests deterministically without network.""" + path = urllib.parse.urlparse(url).path + if method == "GET" and path.endswith("/models"): + return 200, _dry_run_catalog_body() + model_match = re.search(rb'"model"\s*:\s*"([^"]+)"', body or b"") + if model_match is None: + model_match = re.search(rb'name="model"\r\n\r\n([^\r]+)', body or b"") + model_id = model_match.group(1).decode("utf-8") if model_match else "" + behavior = _DRY_RUN_MODEL_BEHAVIOR.get(model_id) + if behavior is None: + return 404, json.dumps({"error": "unknown dry-run model"}).encode("utf-8") + if behavior == "rate_limited": + return 429, b"{}" + if behavior == "unavailable": + return 503, b"{}" + if behavior == "unsupported": + return 404, b"{}" + capability_name = _dry_run_probe_capability(path, body) + if capability_name in behavior: + return 200, _dry_run_success_body(path) + return 400, json.dumps({"error": "capability not supported"}).encode("utf-8") + + return transport + + +def _deterministic_timer() -> Callable[[], float]: + """Monotonic fake timer for reproducible dry-run latency fields.""" + state = {"now": 0.0} + + def timer() -> float: + """Advance one millisecond per observation.""" + state["now"] += 0.001 + return state["now"] + + return timer + + +# -------------------------------------------------------------------------- +# Run orchestration + CLI +# -------------------------------------------------------------------------- + + +def run_benchmark( + run_mode: str, + task_manifest_path: str, + pricing_scenario_path: str | None, + output_dir: str, + endpoint: str = NIM_DEFAULT_ENDPOINT, + max_total_requests: int = 2000, + probe_concurrency: int = 4, + timeout_seconds: float = 60.0, + max_output_tokens: int = 256, + max_eval_models: int = 7, + seed: int = 7, + git_sha: str = "", + workflow_run_id: str = "", + transport: ProviderTransport | None = None, +) -> dict[str, Any]: + """Run the complete benchmark in deterministic dry or evidence-gated live mode. + + Live evidence, optional paid-price provenance, and run identity are validated + before any provider transport can execute. Dry runs use an in-process provider + and never need or read the NVIDIA credential. + + Args: + run_mode: ``dry_run`` or ``live``. + task_manifest_path: Versioned locked/exploratory task manifest. + pricing_scenario_path: Optional explicit hypothetical pricing scenario. + output_dir: Destination for JSON, CSV, and Markdown artifacts. + endpoint: OpenAI-compatible provider endpoint. + max_total_requests: Complete-run provider request cap. + probe_concurrency: Maximum concurrent model probe workers. + timeout_seconds: Per-address network timeout. + max_output_tokens: Equal per-cell prompt-plus-completion token budget. + max_eval_models: Maximum chat-eligible workers in policy evaluation. + seed: Deterministic bootstrap seed. + git_sha: Exact source revision, required live. + workflow_run_id: Workflow provenance identifier, required live. + transport: Optional injected provider transport for deterministic tests. + + Returns: + Complete report including written artifact paths. + + Raises: + BenchmarkContractError: If mode, evidence, or parameters are invalid. + NotConfigured: If a live run cannot resolve its KV credential. + """ + if run_mode not in ("dry_run", "live"): + raise BenchmarkContractError( + f"run_mode must be 'dry_run' or 'live', not {run_mode!r}" + ) + manifest = load_task_manifest(task_manifest_path) + pricing_scenario = load_pricing_scenario(pricing_scenario_path) + if run_mode == "live": + if not git_sha or not workflow_run_id: + raise BenchmarkContractError( + "live runs require --git-sha and --workflow-run-id provenance" + ) + _require_current_actual_cost_evidence() + validate_live_pricing_scenario(pricing_scenario) + request_budget = RequestBudget(max_total_requests) + + if run_mode == "dry_run": + api_key = "dry-run-placeholder-not-a-secret" + active_transport = transport or build_dry_run_transport() + clock: Callable[[], float] = lambda: DRY_RUN_FIXED_UNIX_TIME + probe_timer: Callable[[], float] = lambda: 0.0 + timer = _deterministic_timer() + eval_base_url = "mock://nim-dry-run" + eval_client: ModelClient = _BudgetedModelClient( + request_budget, + max_output_tokens=max_output_tokens, + ) + else: + api_key = get_credential(NIM_CREDENTIAL_NAME) or "" + if not api_key: + raise NotConfigured( + f"live benchmark requires the '{NIM_CREDENTIAL_NAME}' credential " + "in the KV; seed it via register-credential bootstrap (never argv)" + ) + active_transport = transport or build_default_transport(timeout_seconds) + clock = time.time + probe_timer = time.perf_counter + timer = time.perf_counter + eval_base_url = endpoint + eval_client = _BudgetedModelClient( + request_budget, + timeout=float(timeout_seconds), + max_output_tokens=max_output_tokens, + ) + + benchmark_parameters = { + "endpoint": endpoint, + "max_total_requests": max_total_requests, + "probe_concurrency": probe_concurrency, + "timeout_seconds": timeout_seconds, + "max_output_tokens": max_output_tokens, + "max_eval_models": max_eval_models, + "max_workflow_depth": MAX_WORKFLOW_DEPTH, + "policy_total_token_budget": max_output_tokens, + "policy_maximum_calls": MAX_WORKFLOW_DEPTH, + "minimum_paired_task_count": MINIMUM_PAIRED_TASK_COUNT, + "required_completion_fraction": REQUIRED_COMPLETION_FRACTION, + "seed": seed, + "task_manifest_version": manifest["manifest_version"], + "pricing_scenario_version": ( + pricing_scenario["scenario_version"] if pricing_scenario else None + ), + "pricing_scenario_status": ( + pricing_scenario["scenario_status"] if pricing_scenario else None + ), + } + + catalog = discover_model_catalog( + active_transport, + endpoint, + api_key, + request_budget, + ) + request_plan = plan_complete_request_budget( + discovered_model_count=len(catalog["models"]), + max_eval_models=max_eval_models, + locked_task_count=len(locked_evaluation_tasks(manifest)), + ) + if ( + request_plan["total_required_request_count"] + > request_budget.max_total_requests + ): + raise BenchmarkBudgetError( + "complete benchmark needs " + f"{request_plan['total_required_request_count']} requests but " + f"configured cap is {request_budget.max_total_requests}; " + "no capability probes were started" + ) + benchmark_parameters.update( + { + "catalog_request_count": request_plan["catalog_request_count"], + "capability_probe_request_count": request_plan[ + "capability_probe_request_count" + ], + "evaluation_reserve_request_count": request_plan[ + "evaluation_reserve_request_count" + ], + "planned_worker_count": request_plan["planned_worker_count"], + "total_required_request_count": request_plan[ + "total_required_request_count" + ], + } + ) + probed_models = probe_discovered_models( + catalog["models"], + active_transport, + endpoint, + api_key, + request_budget, + probe_concurrency, + clock, + probe_timer, + ) + agents = build_worker_agents(probed_models, eval_base_url, max_eval_models) + evaluation = evaluate_policies( + agents, + manifest, + pricing_scenario, + eval_client, + request_budget, + timer, + total_token_budget=max_output_tokens, + maximum_calls=MAX_WORKFLOW_DEPTH, + ) + report = assemble_benchmark_report( + run_mode, + endpoint, + catalog, + probed_models, + evaluation, + request_budget, + { + "git_sha": git_sha, + "workflow_run_id": workflow_run_id, + "task_manifest_path": task_manifest_path, + "pricing_scenario_path": pricing_scenario_path, + "benchmark_parameters": benchmark_parameters, + "request_plan": request_plan, + }, + seed, + ) + report["artifact_paths"] = write_benchmark_artifacts(report, output_dir) + return report + + +def _bootstrap_live_credential() -> None: + """One-shot bootstrap: move the job-environment secret into the KV. + + Environment is used strictly as bootstrap transport (the same contract as + ``register-credential --from-env``); runtime reads then resolve the key + through :func:`get_credential` only. + """ + if get_credential(NIM_CREDENTIAL_NAME) is None and os.environ.get(NIM_CREDENTIAL_NAME): + register_credential(NIM_CREDENTIAL_NAME, os.environ[NIM_CREDENTIAL_NAME]) + + +def run_benchmark_cli(argv: list[str]) -> int: + """CLI entry for ``python -m contextual_orchestrator nim-benchmark``. + + The provider secret is never accepted via argv: live runs resolve it from + the KV, seeded from the job environment by the one-shot bootstrap step. + """ + parser = argparse.ArgumentParser( + prog="python -m contextual_orchestrator nim-benchmark", + description="Evidence-grade NVIDIA NIM model discovery and cost-quality benchmark.", + ) + parser.add_argument("--dry-run", action="store_true", help="Validate everything without contacting NVIDIA.") + parser.add_argument("--task-manifest", default="examples/nim_task_manifest.json") + parser.add_argument("--pricing-scenario", default=None, + help="Versioned hypothetical price-assumption JSON (omit => costs stay 'unknown').") + parser.add_argument("--output-dir", default="benchmark_artifacts") + parser.add_argument("--endpoint", default=NIM_DEFAULT_ENDPOINT) + parser.add_argument("--max-total-requests", type=int, default=2000) + parser.add_argument("--probe-concurrency", type=int, default=4) + parser.add_argument("--timeout-seconds", type=float, default=60.0) + parser.add_argument("--max-output-tokens", type=int, default=256) + parser.add_argument("--max-eval-models", type=int, default=7) + parser.add_argument("--seed", type=int, default=7) + parser.add_argument("--git-sha", default="", help="Provenance: the exact commit under benchmark (required live).") + parser.add_argument("--workflow-run-id", default="", help="Provenance: the CI run id (required live).") + args = parser.parse_args(argv) + + run_mode = "dry_run" if args.dry_run else "live" + if run_mode == "live": + _bootstrap_live_credential() + try: + report = run_benchmark( + run_mode, + args.task_manifest, + args.pricing_scenario, + args.output_dir, + endpoint=args.endpoint, + max_total_requests=args.max_total_requests, + probe_concurrency=args.probe_concurrency, + timeout_seconds=args.timeout_seconds, + max_output_tokens=args.max_output_tokens, + max_eval_models=args.max_eval_models, + seed=args.seed, + git_sha=args.git_sha, + workflow_run_id=args.workflow_run_id, + ) + except (BenchmarkContractError, CatalogDiscoveryError, BenchmarkAuthError, + BenchmarkBudgetError, SecretLeakError, NotConfigured, OSError) as exc: + print(json.dumps({"benchmark_failed_closed": True, "error_class": type(exc).__name__, + "error_message": str(exc)}, ensure_ascii=False)) + return 1 + print(json.dumps( + { + "run_mode": report["provenance"]["run_mode"], + "discovered_model_count": report["catalog_snapshot"]["discovered_model_count"], + "capability_summary": report["capability_summary"], + "requests_spent": report["request_budget"]["requests_spent"], + "artifact_paths": report["artifact_paths"], + }, + ensure_ascii=False, + indent=2, + )) + return 0 diff --git a/contextual_orchestrator/nim_csv_evidence.py b/contextual_orchestrator/nim_csv_evidence.py new file mode 100644 index 00000000..2da63a02 --- /dev/null +++ b/contextual_orchestrator/nim_csv_evidence.py @@ -0,0 +1,518 @@ +"""Complete and transactionally publish NIM benchmark evidence artifacts. + +The benchmark report records ``models_used`` for every policy/task cell in +JSON. This optional, standard-library-only adapter copies that evidence into the +uploaded CSV as deterministic JSON so spreadsheet consumers retain the exact +step, role, agent, and model identity required for audit and replay. + +Runtime orchestration traces use non-negative integer step identifiers, while +route-only evidence can also carry a present-but-empty string sentinel. The +adapter preserves integer identities as decimal strings and assigns empty +sentinels deterministic positional IDs without changing real non-empty string +IDs. Missing, boolean, negative-integer, unsupported-type, or duplicate explicit +IDs fail closed. + +The adapter is intentionally lazy: importing :mod:`contextual_orchestrator` +does not import this module or mutate the benchmark implementation. The NIM CLI +composition root invokes it only for the benchmark command. The wrapper writes, +validates, enriches, and secret-checks the complete artifact set in a hidden +sibling staging directory before publishing the directory as one unit. +""" + +from __future__ import annotations + +import contextlib +import csv +import io +import json +import os +import shutil +import stat +import sys +import tempfile +import uuid +from pathlib import Path +from typing import Callable, TextIO + +from .nim_benchmark import run_benchmark_cli + +DEFAULT_BENCHMARK_OUTPUT_DIRECTORY = Path("benchmark_artifacts") +_ASSIGNMENT_FIELDS = ("step_id", "role", "agent_id", "model_id") +_CELL_IDENTITY_FIELDS = ("policy_name", "task_id") +_ARTIFACT_FILENAMES = ( + "benchmark_report.json", + "benchmark_cells.csv", + "benchmark_summary.md", +) +_ARTIFACT_PATH_KEYS = { + "json_path": "benchmark_report.json", + "csv_path": "benchmark_cells.csv", + "markdown_path": "benchmark_summary.md", +} + + +class CsvEvidenceError(RuntimeError): + """The JSON, CSV, and Markdown files cannot form one complete evidence set.""" + + +def _cell_identity(cell: object, source_label: str) -> tuple[str, str]: + """Return one non-empty policy/task identity from a report or CSV row.""" + if not isinstance(cell, dict): + raise CsvEvidenceError(f"{source_label} cell must be an object") + values: list[str] = [] + for field_name in _CELL_IDENTITY_FIELDS: + value = cell.get(field_name) + if not isinstance(value, str) or not value: + raise CsvEvidenceError( + f"{source_label} cell requires non-empty {field_name}" + ) + values.append(value) + return values[0], values[1] + + +def _canonical_step_id( + raw_step_id: object, + position: int, + used_step_ids: set[str], +) -> str: + """Return one unique string trace ID from supported runtime identifiers.""" + if isinstance(raw_step_id, bool): + raise CsvEvidenceError("model assignment requires string or integer step_id") + if isinstance(raw_step_id, int): + if raw_step_id < 0: + raise CsvEvidenceError("model assignment step_id integer must be non-negative") + candidate = str(raw_step_id) + elif isinstance(raw_step_id, str): + candidate = raw_step_id.strip() + else: + raise CsvEvidenceError("model assignment requires string or integer step_id") + if not candidate: + candidate = f"trace_step_{position:04d}" + suffix = 2 + while candidate in used_step_ids: + candidate = f"trace_step_{position:04d}_{suffix}" + suffix += 1 + elif candidate in used_step_ids: + raise CsvEvidenceError(f"duplicate step_id: {candidate}") + used_step_ids.add(candidate) + return candidate + + +def _models_used_json(value: object) -> str: + """Validate, canonicalize, and deterministically serialize model assignments.""" + if not isinstance(value, list): + raise CsvEvidenceError("report cell models_used must be a list") + normalized: list[dict[str, str]] = [] + used_step_ids: set[str] = set() + for position, assignment in enumerate(value, start=1): + if not isinstance(assignment, dict): + raise CsvEvidenceError("each model assignment must be an object") + normalized_assignment = { + "step_id": _canonical_step_id( + assignment.get("step_id"), + position, + used_step_ids, + ) + } + for field_name in _ASSIGNMENT_FIELDS[1:]: + field_value = assignment.get(field_name) + if not isinstance(field_value, str) or not field_value: + raise CsvEvidenceError( + f"model assignment requires non-empty {field_name}" + ) + normalized_assignment[field_name] = field_value + normalized.append(normalized_assignment) + return json.dumps( + normalized, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + + +def _report_assignment_map(report_path: Path) -> dict[tuple[str, str], str]: + """Load the report and index validated assignment evidence by cell identity.""" + try: + report = json.loads(report_path.read_text(encoding="utf-8")) + except (UnicodeError, json.JSONDecodeError) as exc: + raise CsvEvidenceError("benchmark report must contain valid JSON") from exc + evaluation = report.get("evaluation") if isinstance(report, dict) else None + cells = evaluation.get("evaluation_cells") if isinstance(evaluation, dict) else None + if not isinstance(cells, list): + raise CsvEvidenceError( + "benchmark report requires evaluation.evaluation_cells as a list" + ) + + assignments: dict[tuple[str, str], str] = {} + for cell in cells: + identity = _cell_identity(cell, "report") + if identity in assignments: + raise CsvEvidenceError( + "duplicate report cell identity: " + "/".join(identity) + ) + assignments[identity] = _models_used_json(cell.get("models_used")) + return assignments + + +def _csv_rows(csv_path: Path) -> tuple[list[str], list[dict[str, str]]]: + """Read CSV rows and validate that each cell identity occurs exactly once.""" + try: + with csv_path.open(encoding="utf-8", newline="") as handle: + reader = csv.DictReader(handle) + fieldnames = list(reader.fieldnames or []) + rows = list(reader) + except (UnicodeError, csv.Error) as exc: + raise CsvEvidenceError("benchmark cell CSV is not readable") from exc + missing_fields = [ + field_name + for field_name in _CELL_IDENTITY_FIELDS + if field_name not in fieldnames + ] + if missing_fields: + raise CsvEvidenceError( + "benchmark cell CSV is missing identity columns: " + + ", ".join(missing_fields) + ) + + seen: set[tuple[str, str]] = set() + for row in rows: + identity = _cell_identity(row, "CSV") + if identity in seen: + raise CsvEvidenceError( + "duplicate CSV cell identity: " + "/".join(identity) + ) + seen.add(identity) + return fieldnames, rows + + +def enrich_benchmark_cell_csv( + report_path: str | os.PathLike[str], + csv_path: str | os.PathLike[str], +) -> None: + """Atomically add deterministic ``models_used_json`` to every CSV cell. + + Args: + report_path: Path to the benchmark's authoritative JSON report. + csv_path: Path to the benchmark cell CSV written by the same run. + + Raises: + CsvEvidenceError: If either artifact is malformed, duplicated, or does + not describe exactly the same policy/task cells. + OSError: If an artifact cannot be read or the atomic replacement fails. + """ + report_file = Path(report_path) + csv_file = Path(csv_path) + assignments = _report_assignment_map(report_file) + fieldnames, rows = _csv_rows(csv_file) + csv_identities = {_cell_identity(row, "CSV") for row in rows} + if csv_identities != set(assignments): + missing_from_csv = sorted(set(assignments) - csv_identities) + missing_from_report = sorted(csv_identities - set(assignments)) + raise CsvEvidenceError( + "benchmark JSON/CSV cell identity mismatch; " + f"missing_from_csv={missing_from_csv}; " + f"missing_from_report={missing_from_report}" + ) + + output_fields = [ + field_name for field_name in fieldnames if field_name != "models_used_json" + ] + output_fields.append("models_used_json") + buffer = io.StringIO(newline="") + writer = csv.DictWriter(buffer, fieldnames=output_fields, extrasaction="ignore") + writer.writeheader() + for row in rows: + enriched_row = dict(row) + enriched_row["models_used_json"] = assignments[_cell_identity(row, "CSV")] + writer.writerow(enriched_row) + + csv_mode = stat.S_IMODE(csv_file.stat().st_mode) + csv_file.parent.mkdir(parents=True, exist_ok=True) + file_descriptor, temporary_name = tempfile.mkstemp( + dir=csv_file.parent, + prefix=f".{csv_file.name}.", + suffix=".tmp", + ) + temporary_path = Path(temporary_name) + try: + with os.fdopen( + file_descriptor, + "w", + encoding="utf-8", + newline="", + ) as temporary: + temporary.write(buffer.getvalue()) + temporary.flush() + os.fsync(temporary.fileno()) + os.chmod(temporary_path, csv_mode) + os.replace(temporary_path, csv_file) + finally: + temporary_path.unlink(missing_ok=True) + + +def output_directory_from_argv(argv: list[str]) -> Path: + """Return the NIM CLI output directory without accepting any secret input.""" + for index, argument in enumerate(argv): + if argument == "--output-dir": + if index + 1 >= len(argv): + raise CsvEvidenceError("--output-dir requires a value") + return Path(argv[index + 1]) + if argument.startswith("--output-dir="): + value = argument.split("=", 1)[1] + if not value: + raise CsvEvidenceError("--output-dir requires a non-empty value") + return Path(value) + return DEFAULT_BENCHMARK_OUTPUT_DIRECTORY + + +def _normalized_output_directory(output_directory: Path) -> Path: + """Return a safe absolute final directory rooted at its resolved parent.""" + expanded = output_directory.expanduser() + if expanded.name in {"", ".", ".."}: + raise CsvEvidenceError( + "output directory must name a dedicated artifact directory" + ) + parent = expanded.parent.resolve() + final_directory = parent / expanded.name + if final_directory.is_symlink(): + raise CsvEvidenceError("output directory must not be a symbolic link") + if final_directory.exists() and not final_directory.is_dir(): + raise CsvEvidenceError("output directory path must be a directory") + return final_directory + + +def _remove_path(path: Path) -> None: + """Remove one private publication path without following symbolic links.""" + if path.is_symlink() or path.is_file(): + path.unlink(missing_ok=True) + elif path.exists(): + shutil.rmtree(path) + + +def _publication_residue(final_directory: Path, residue_kind: str) -> list[Path]: + """Return sorted hidden staging or backup paths for one final directory.""" + return sorted( + final_directory.parent.glob( + f".{final_directory.name}.{residue_kind}-*" + ) + ) + + +def _recover_interrupted_publication(final_directory: Path) -> None: + """Recover one crash backup and remove abandoned staging directories. + + A portable replacement of an existing non-empty directory requires two + same-filesystem renames: final to backup, then staging to final. A process or + host crash between those renames can leave the final name absent and one + hidden backup present. The next run restores that sole backup before doing + any benchmark work. Multiple backups are ambiguous and therefore fail + closed rather than guessing which evidence set is authoritative. + """ + for staging_path in _publication_residue(final_directory, "staging"): + _remove_path(staging_path) + + backups = _publication_residue(final_directory, "backup") + if len(backups) > 1: + raise CsvEvidenceError( + "multiple benchmark publication backups require operator review" + ) + if not backups: + return + backup_path = backups[0] + if final_directory.exists(): + _remove_path(backup_path) + else: + os.replace(backup_path, final_directory) + + +def _argv_with_output_directory(argv: list[str], output_directory: Path) -> list[str]: + """Return CLI arguments with exactly one controlled staging output path.""" + rewritten: list[str] = [] + found = False + index = 0 + while index < len(argv): + argument = argv[index] + if argument == "--output-dir": + if found: + raise CsvEvidenceError("--output-dir may be supplied only once") + if index + 1 >= len(argv): + raise CsvEvidenceError("--output-dir requires a value") + rewritten.extend(("--output-dir", str(output_directory))) + found = True + index += 2 + continue + if argument.startswith("--output-dir="): + if found: + raise CsvEvidenceError("--output-dir may be supplied only once") + rewritten.append(f"--output-dir={output_directory}") + found = True + index += 1 + continue + rewritten.append(argument) + index += 1 + if not found: + rewritten.extend(("--output-dir", str(output_directory))) + return rewritten + + +def _validate_complete_artifact_directory(staging_directory: Path) -> None: + """Require exactly three regular, non-empty, fully enriched artifacts.""" + actual_names = sorted(path.name for path in staging_directory.iterdir()) + if actual_names != sorted(_ARTIFACT_FILENAMES): + raise CsvEvidenceError( + "staged benchmark directory must contain exactly JSON, CSV, and Markdown artifacts" + ) + for artifact_name in _ARTIFACT_FILENAMES: + artifact_path = staging_directory / artifact_name + if artifact_path.is_symlink() or not artifact_path.is_file(): + raise CsvEvidenceError("staged benchmark artifacts must be regular files") + if artifact_path.stat().st_size == 0: + raise CsvEvidenceError("staged benchmark artifacts must not be empty") + with (staging_directory / "benchmark_cells.csv").open( + encoding="utf-8", + newline="", + ) as handle: + fieldnames = list(csv.DictReader(handle).fieldnames or []) + if "models_used_json" not in fieldnames: + raise CsvEvidenceError( + "staged benchmark CSV lacks model-assignment evidence" + ) + + +def _restore_backup_after_failure( + final_directory: Path, + backup_directory: Path | None, +) -> None: + """Restore the prior complete set after an ordinary publication failure.""" + if backup_directory is None: + _remove_path(final_directory) + return + if final_directory.exists(): + _remove_path(final_directory) + if backup_directory.exists(): + os.replace(backup_directory, final_directory) + + +def _publish_staged_directory( + staging_directory: Path, + final_directory: Path, +) -> None: + """Publish a complete staged directory with rollback for ordinary failures.""" + backup_directory: Path | None = None + if final_directory.exists(): + backup_directory = final_directory.parent / ( + f".{final_directory.name}.backup-{uuid.uuid4().hex}" + ) + if backup_directory.exists(): + raise CsvEvidenceError("generated benchmark backup path already exists") + os.replace(final_directory, backup_directory) + try: + os.replace(staging_directory, final_directory) + if backup_directory is not None: + shutil.rmtree(backup_directory) + except BaseException as publication_error: + try: + _restore_backup_after_failure(final_directory, backup_directory) + except BaseException as restoration_error: + raise CsvEvidenceError( + "benchmark publication failed and the prior artifact set could not be restored" + ) from restoration_error + raise publication_error + + +def _rewrite_success_output( + buffered_output: str, + final_directory: Path, +) -> str: + """Replace private staging paths in the success payload with final paths.""" + try: + payload = json.loads(buffered_output) + except json.JSONDecodeError as exc: + raise CsvEvidenceError("benchmark success output must be valid JSON") from exc + if not isinstance(payload, dict): + raise CsvEvidenceError("benchmark success output must be a JSON object") + artifact_paths = payload.get("artifact_paths") + if not isinstance(artifact_paths, dict): + raise CsvEvidenceError("benchmark success output requires artifact_paths") + payload["artifact_paths"] = { + key: str(final_directory / filename) + for key, filename in _ARTIFACT_PATH_KEYS.items() + } + return json.dumps(payload, ensure_ascii=False, indent=2) + "\n" + + +def _write_fail_closed_result(stdout: TextIO, error: BaseException) -> None: + """Write one bounded fail-closed result without exposing staged contents.""" + stdout.write( + json.dumps( + { + "benchmark_failed_closed": True, + "error_class": type(error).__name__, + "error_message": str(error)[:500], + }, + ensure_ascii=False, + ) + + "\n" + ) + + +def run_benchmark_cli_with_complete_csv( + argv: list[str], + *, + benchmark_cli: Callable[[list[str]], int] = run_benchmark_cli, + stdout: TextIO = sys.stdout, +) -> int: + """Run, enrich, and transactionally publish one complete evidence set. + + The benchmark runs in a hidden sibling staging directory. A successful + result is emitted only after JSON/CSV identity validation, deterministic CSV + assignment enrichment, complete-set validation, and directory publication. + Fresh-target failures expose no final directory; replacement failures restore + the prior complete set; temporary staging and ordinary rollback residue are + removed. Benchmark failures are passed through unchanged. + + The portable crash contract is narrower than the ordinary rollback contract: + replacing an existing non-empty directory takes two atomic renames, leaving + an unavoidable crash window between backup creation and final publication. + A later invocation restores a sole hidden backup before running. Ambiguous + multiple backups fail closed for operator review. + """ + staging_directory: Path | None = None + benchmark_output = io.StringIO() + try: + requested_directory = output_directory_from_argv(argv) + final_directory = _normalized_output_directory(requested_directory) + final_directory.parent.mkdir(parents=True, exist_ok=True) + _recover_interrupted_publication(final_directory) + staging_directory = Path( + tempfile.mkdtemp( + dir=final_directory.parent, + prefix=f".{final_directory.name}.staging-", + ) + ) + staged_argv = _argv_with_output_directory(argv, staging_directory) + with contextlib.redirect_stdout(benchmark_output): + exit_code = benchmark_cli(staged_argv) + if exit_code != 0: + stdout.write(benchmark_output.getvalue()) + return exit_code + + enrich_benchmark_cell_csv( + staging_directory / "benchmark_report.json", + staging_directory / "benchmark_cells.csv", + ) + _validate_complete_artifact_directory(staging_directory) + success_output = _rewrite_success_output( + benchmark_output.getvalue(), + final_directory, + ) + _publish_staged_directory(staging_directory, final_directory) + staging_directory = None + stdout.write(success_output) + return 0 + except (CsvEvidenceError, OSError, UnicodeError, csv.Error) as error: + _write_fail_closed_result(stdout, error) + return 1 + finally: + if staging_directory is not None: + _remove_path(staging_directory) diff --git a/contextual_orchestrator/nim_strict_scoring.py b/contextual_orchestrator/nim_strict_scoring.py new file mode 100644 index 00000000..772b2ea9 --- /dev/null +++ b/contextual_orchestrator/nim_strict_scoring.py @@ -0,0 +1,415 @@ +"""Strict, explicitly activated scoring for evidence-grade NIM benchmarks. + +The historical benchmark scorers remain available for compatibility with old +manifests and tests. The supported ``nim-benchmark`` composition root activates +this module and derives a temporary, versioned strict manifest before any +provider request. Ordinary ``import contextual_orchestrator`` remains free of +benchmark imports and global mutation. +""" + +from __future__ import annotations + +import copy +import decimal +import json +import os +import re +import tempfile +import unicodedata +from pathlib import Path +from typing import Any, Callable + +from . import nim_benchmark as benchmark + + +STRICT_SCORING_POLICY_VERSION = "2026-08-07.4" +MAX_STRICT_ANSWER_CHARACTERS = 4096 +_DEFAULT_TASK_MANIFEST = Path("examples/nim_task_manifest.json") +_STRICT_NUMBER_KEY = ("exact_number_match", "2") +_STRICT_TEXT_KEY = ("exact_text_match", "1") +_LEGACY_NUMBER_KEY = ("exact_number_match", "1") +_LEGACY_TEXT_KEY = ("substring_match", "1") +_ASCII_DECIMAL_LITERAL = re.compile( + r"[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?" +) +_NUMERIC_PROMPT_TOKEN = re.compile( + r"(? None: + """Reject an answer-key string that exceeds the strict-scoring input cap.""" + if len(value) > MAX_STRICT_ANSWER_CHARACTERS: + raise benchmark.BenchmarkContractError( + f"{label} exceeds the strict-scoring character budget" + ) + + +def _normalized_exact_text(value: str, *, case_sensitive: bool) -> str: + """Return NFC text with normalized whitespace and the declared case policy.""" + normalized = unicodedata.normalize("NFC", value) + compact = " ".join(normalized.split()) + return compact if case_sensitive else compact.casefold() + + +def _expected_decimal(expected: dict[str, Any]) -> decimal.Decimal: + """Return one finite, reproducible expected numeric literal. + + Raises: + benchmark.BenchmarkContractError: If ``expected.number`` is not one + bounded finite ASCII decimal literal represented as a JSON string. + """ + value = expected.get("number") + if not isinstance(value, str): + raise benchmark.BenchmarkContractError( + "exact-number expected.number must be a finite numeric literal string" + ) + _require_expected_character_budget(value, "exact-number expected.number") + literal = unicodedata.normalize("NFC", value).strip() + if _ASCII_DECIMAL_LITERAL.fullmatch(literal) is None: + raise benchmark.BenchmarkContractError( + "exact-number expected.number must be a finite numeric literal string" + ) + try: + return decimal.Decimal(literal) + except decimal.InvalidOperation as exc: + raise benchmark.BenchmarkContractError( + "exact-number expected.number must be a finite numeric literal string" + ) from exc + + +def _answer_decimal(answer_text: str) -> decimal.Decimal | None: + """Parse one bounded complete numeric answer, returning ``None`` when unusable.""" + if len(answer_text) > MAX_STRICT_ANSWER_CHARACTERS: + return None + literal = unicodedata.normalize("NFC", answer_text).strip() + if _ASCII_DECIMAL_LITERAL.fullmatch(literal) is None: + return None + try: + return decimal.Decimal(literal) + except decimal.InvalidOperation: + return None + + +def score_exact_number_match_v2( + expected: dict[str, Any], + answer_text: str, +) -> float: + """Score one full numeric response by exact finite decimal value. + + Containment is deliberately rejected: prose, negation, units, and multiple + numbers cannot earn credit merely because they include the expected token. + Oversized or unrepresentable model output scores zero instead of consuming + unbounded normalization resources or aborting the benchmark. + """ + expected_value = _expected_decimal(expected) + answer_value = _answer_decimal(answer_text) + return 1.0 if answer_value is not None and answer_value == expected_value else 0.0 + + +def _text_case_sensitive(expected: dict[str, Any]) -> bool: + """Return an explicit Boolean case policy, defaulting legacy text to folded.""" + value = expected.get("case_sensitive", False) + if not isinstance(value, bool): + raise benchmark.BenchmarkContractError( + "exact-text expected.case_sensitive must be boolean" + ) + return value + + +def _expected_texts(expected: dict[str, Any]) -> tuple[tuple[str, ...], bool]: + """Return unique normalized alternatives and their declared case policy. + + Raises: + benchmark.BenchmarkContractError: If alternatives or case policy are + absent, malformed, empty, oversized, or duplicate after normalization. + """ + values = expected.get("texts") + if not isinstance(values, list) or not values: + raise benchmark.BenchmarkContractError( + "exact-text expected must contain a non-empty texts list" + ) + case_sensitive = _text_case_sensitive(expected) + normalized: list[str] = [] + for value in values: + if not isinstance(value, str): + raise benchmark.BenchmarkContractError( + "exact-text expected must contain a non-empty texts list of strings" + ) + _require_expected_character_budget(value, "exact-text expected alias") + candidate = _normalized_exact_text( + value, + case_sensitive=case_sensitive, + ) + if not candidate: + raise benchmark.BenchmarkContractError( + "exact-text expected must contain a non-empty texts list of strings" + ) + if candidate in normalized: + raise benchmark.BenchmarkContractError( + "exact-text expected contains a duplicate normalized answer" + ) + normalized.append(candidate) + return tuple(normalized), case_sensitive + + +def score_exact_text_match(expected: dict[str, Any], answer_text: str) -> float: + """Score one bounded complete text response against explicit alternatives.""" + expected_values, case_sensitive = _expected_texts(expected) + if len(answer_text) > MAX_STRICT_ANSWER_CHARACTERS: + return 0.0 + answer_value = _normalized_exact_text( + answer_text, + case_sensitive=case_sensitive, + ) + return 1.0 if answer_value in expected_values else 0.0 + + +def enable_strict_evidence_scoring() -> None: + """Idempotently register strict scorer versions at an explicit composition root. + + Raises: + benchmark.BenchmarkContractError: If another implementation already + owns either versioned scorer identity. + """ + registrations: dict[ + tuple[str, str], + Callable[[dict[str, Any], str], float], + ] = { + _STRICT_NUMBER_KEY: score_exact_number_match_v2, + _STRICT_TEXT_KEY: score_exact_text_match, + } + for scorer_key, scorer_function in registrations.items(): + existing = benchmark.SCORER_REGISTRY.get(scorer_key) + if existing is not None and existing is not scorer_function: + raise benchmark.BenchmarkContractError( + f"strict scorer identity collision: {scorer_key}" + ) + benchmark.SCORER_REGISTRY[scorer_key] = scorer_function + + +def _legacy_text_expectation(expected: dict[str, Any]) -> dict[str, Any]: + """Translate one legacy substring key into explicit strict text semantics.""" + substring = expected.get("substring") + if not isinstance(substring, str) or not substring.strip(): + raise benchmark.BenchmarkContractError( + "legacy locked substring expectation must be a non-empty string" + ) + texts = expected.get("strict_texts", [substring]) + strict_expected = { + "texts": texts, + "case_sensitive": expected.get("strict_case_sensitive", False), + } + _expected_texts(strict_expected) + return strict_expected + + +def _prompt_leaks_decimal(prompt: str, expected_value: decimal.Decimal) -> bool: + """Return whether a complete prompt token equals the numeric answer key.""" + _require_expected_character_budget(prompt, "locked task prompt") + for match in _NUMERIC_PROMPT_TOKEN.finditer(prompt): + try: + observed = decimal.Decimal(match.group(1)) + except decimal.InvalidOperation: + continue + if observed == expected_value: + return True + return False + + +def _prompt_leaks_text(prompt: str, expected: dict[str, Any]) -> bool: + """Return whether a declared complete text alias appears in the prompt.""" + _require_expected_character_budget(prompt, "locked task prompt") + expected_values, case_sensitive = _expected_texts(expected) + normalized_prompt = _normalized_exact_text( + prompt, + case_sensitive=case_sensitive, + ) + for expected_value in expected_values: + prefix = r"(? bool: + """Return whether one strict locked task reveals any accepted answer token.""" + prompt = task.get("prompt") + if not isinstance(prompt, str) or not prompt.strip(): + raise benchmark.BenchmarkContractError( + "strict scoring requires a non-empty locked task prompt" + ) + scorer = task["scorer"] + expected = task["expected"] + scorer_key = (str(scorer.get("name")), str(scorer.get("version"))) + if scorer_key == _STRICT_NUMBER_KEY: + return _prompt_leaks_decimal(prompt, _expected_decimal(expected)) + if scorer_key == _STRICT_TEXT_KEY: + return _prompt_leaks_text(prompt, expected) + raise benchmark.BenchmarkContractError( + f"locked task names unsupported strict scorer: {scorer_key}" + ) + + +def strict_task_manifest_payload(manifest: object) -> dict[str, Any]: + """Derive a strict locked-split manifest while preserving exploratory tasks. + + Historical authoring manifests may use the legacy containment scorers. This + function upgrades only locked tasks to versioned complete-answer scorers. + Already-strict locked tasks are preserved. Unknown locked scorer contracts + and prompt leakage fail closed before provider egress. + + Args: + manifest: Parsed task-manifest JSON value. + + Returns: + Deep-copied, strict, versioned manifest payload. + + Raises: + benchmark.BenchmarkContractError: If the manifest or a locked scorer + cannot be converted without ambiguity or leaks an accepted answer. + """ + if not isinstance(manifest, dict): + raise benchmark.BenchmarkContractError( + "strict scoring requires a task manifest object" + ) + tasks = manifest.get("tasks") + if not isinstance(tasks, list): + raise benchmark.BenchmarkContractError( + "strict scoring requires a task manifest tasks list" + ) + strict_manifest = copy.deepcopy(manifest) + strict_tasks = strict_manifest["tasks"] + for task in strict_tasks: + if not isinstance(task, dict): + raise benchmark.BenchmarkContractError( + "strict scoring requires every task to be an object" + ) + if task.get("split") != "locked": + continue + scorer = task.get("scorer") + expected = task.get("expected") + if not isinstance(scorer, dict) or not isinstance(expected, dict): + raise benchmark.BenchmarkContractError( + "strict scoring requires locked scorer and expected objects" + ) + scorer_key = (str(scorer.get("name")), str(scorer.get("version"))) + if scorer_key == _LEGACY_NUMBER_KEY: + _expected_decimal(expected) + task["scorer"] = { + "name": _STRICT_NUMBER_KEY[0], + "version": _STRICT_NUMBER_KEY[1], + } + elif scorer_key == _LEGACY_TEXT_KEY: + task["scorer"] = { + "name": _STRICT_TEXT_KEY[0], + "version": _STRICT_TEXT_KEY[1], + } + task["expected"] = _legacy_text_expectation(expected) + elif scorer_key == _STRICT_NUMBER_KEY: + _expected_decimal(expected) + elif scorer_key == _STRICT_TEXT_KEY: + _expected_texts(expected) + else: + raise benchmark.BenchmarkContractError( + f"locked task names unsupported strict scorer: {scorer_key}" + ) + if _strict_task_leaks_expected(task): + raise benchmark.BenchmarkContractError( + f"task {task.get('task_id')!r} leaks its expected answer into the prompt" + ) + source_version = strict_manifest.get("manifest_version") + if not isinstance(source_version, str) or not source_version: + raise benchmark.BenchmarkContractError( + "strict scoring requires a non-empty manifest_version" + ) + strict_manifest["manifest_version"] = ( + f"{source_version}+strict.{STRICT_SCORING_POLICY_VERSION}" + ) + strict_manifest["scoring_policy_version"] = STRICT_SCORING_POLICY_VERSION + return strict_manifest + + +def _task_manifest_argument(argv: list[str]) -> tuple[Path, list[str]]: + """Return the source manifest path and arguments without its selector.""" + source_path = _DEFAULT_TASK_MANIFEST + remaining: list[str] = [] + found = False + index = 0 + while index < len(argv): + argument = argv[index] + if argument == "--task-manifest": + if found or index + 1 >= len(argv): + raise benchmark.BenchmarkContractError( + "--task-manifest must be supplied exactly once with a value" + ) + source_path = Path(argv[index + 1]) + found = True + index += 2 + continue + if argument.startswith("--task-manifest="): + if found: + raise benchmark.BenchmarkContractError( + "--task-manifest may be supplied only once" + ) + value = argument.split("=", 1)[1] + if not value: + raise benchmark.BenchmarkContractError( + "--task-manifest requires a non-empty value" + ) + source_path = Path(value) + found = True + index += 1 + continue + remaining.append(argument) + index += 1 + return source_path, remaining + + +def _write_strict_manifest(source_path: Path, destination_path: Path) -> None: + """Validate, transform, and privately write one deterministic strict manifest.""" + try: + source_payload = json.loads(source_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise benchmark.BenchmarkContractError( + "strict scoring could not read a valid task manifest" + ) from exc + strict_payload = strict_task_manifest_payload(source_payload) + serialized = json.dumps( + strict_payload, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + "\n" + file_descriptor = os.open( + destination_path, + os.O_WRONLY | os.O_CREAT | os.O_EXCL, + 0o600, + ) + with os.fdopen(file_descriptor, "w", encoding="utf-8", newline="\n") as handle: + handle.write(serialized) + + +def run_strict_benchmark_cli( + argv: list[str], + *, + benchmark_cli: Callable[[list[str]], int] = benchmark.run_benchmark_cli, +) -> int: + """Run the benchmark CLI with a private strict locked-task manifest. + + The scorer registry is activated explicitly, the selected source manifest + is converted before provider egress, and the derived manifest is deleted + when the one-shot CLI call ends. Its deterministic contents remain bound to + the artifact through the existing manifest SHA-256 and version provenance. + """ + enable_strict_evidence_scoring() + source_path, remaining_argv = _task_manifest_argument(argv) + with tempfile.TemporaryDirectory(prefix="nim-strict-scoring-") as directory: + strict_path = Path(directory) / "task_manifest.strict.json" + _write_strict_manifest(source_path, strict_path) + return benchmark_cli( + [*remaining_argv, "--task-manifest", str(strict_path)] + ) diff --git a/docs/architecture.md b/docs/architecture.md index c0f63a81..e1c19fc7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -41,6 +41,7 @@ This repository implements the interface and control plane, not the trained coor The deliberate simplification is the policy. The paper systems learn routing and topology from rewards; this lab uses deterministic keyword scoring so the repo runs without training data, GPUs, or vendor credentials. Add learned routing only when there is an evaluation set and logs proving the heuristic policy is the bottleneck. +The [NIM cost-quality benchmark](nim_benchmark.md) is that evaluation set's supplier: it discovers the hosted catalog dynamically, probes every modality contract, and compares route/conduct/single-worker policies with paired uncertainty — evidence first, learned policy later. ## Product Planning Interpretation diff --git a/docs/doctoring/nim-benchmark-csv-assignment-evidence.md b/docs/doctoring/nim-benchmark-csv-assignment-evidence.md new file mode 100644 index 00000000..50d70e30 --- /dev/null +++ b/docs/doctoring/nim-benchmark-csv-assignment-evidence.md @@ -0,0 +1,137 @@ +# NIM benchmark CSV assignment-evidence and publication boundary + +## Decision + +The benchmark JSON report is the authoritative structured record for each +policy/task cell. The supported CLI composition root enriches +`benchmark_cells.csv` with one additional column, `models_used_json`, before it +publishes a success result. The column is deterministic compact JSON containing +only these required fields for every observed call: + +- `step_id` +- `role` +- `agent_id` +- `model_id` + +This closes an acquisition-evidence gap: the JSON report already retained exact +role and worker assignments, but the spreadsheet-oriented CSV silently omitted +them. A buyer reviewing latency, score, tokens, and cost in CSV could therefore +not reconstruct which model served each orchestration step without joining the +separate JSON artifact manually. + +The supported CLI now treats JSON, CSV, and Markdown as one publication unit. +It asks the benchmark implementation to render and secret-scan all three files +inside a hidden sibling staging directory, enriches and validates the CSV there, +validates the complete directory, and only then moves the staged directory to +the requested public path. The success payload is withheld until publication +succeeds and reports only final, public artifact paths. + +## Integrity contract + +### Cell and assignment integrity + +1. The JSON report must contain `evaluation.evaluation_cells` as a list. +2. Every JSON and CSV cell must have one non-empty `(policy_name, task_id)` + identity, with no duplicates. +3. The JSON and CSV identity sets must match exactly. +4. Every `models_used` entry must contain non-empty step, role, agent, and model + identifiers. +5. Assignment arrays are serialized with UTF-8, sorted object keys, and compact + separators. Array order is preserved because it carries workflow-step order. +6. CSV enrichment uses a synchronized temporary file followed by same-directory + replacement. Validation failure leaves the staged CSV untouched. + +### Complete-set publication integrity + +1. The final path must name a dedicated directory. A symbolic link, regular + file, `.` path, or otherwise ambiguous target is rejected. +2. Benchmark rendering, report-schema validation, cost-evidence validation, + secret scanning, Markdown rendering, and CSV assignment enrichment happen + outside the final path. +3. The staged directory must contain exactly three non-empty regular files: + `benchmark_report.json`, `benchmark_cells.csv`, and + `benchmark_summary.md`. Symbolic-link artifacts and extra files are rejected. +4. The enriched CSV must contain `models_used_json` before publication. +5. A fresh-target failure leaves no visible final directory and removes hidden + staging residue. +6. When replacing an existing complete set, the prior directory is first moved + to a hidden same-filesystem backup. If ordinary publication fails, the prior + set is restored byte-for-byte and staging/backup residue is removed. +7. On success, the new complete directory becomes the final path, the prior + backup is removed, and the emitted JSON result contains only final paths. +8. Benchmark-process failures pass through unchanged. Enrichment, validation, + or publication failures emit a bounded machine-readable + `benchmark_failed_closed` result rather than the buffered success payload. + +## Portable rollback and crash-window contract + +Portable filesystems do not provide one atomic operation that replaces an +existing non-empty directory with another non-empty directory. Replacement +therefore requires two same-filesystem renames: + +1. final directory to hidden backup; +2. hidden staging directory to final directory. + +Ordinary exceptions between or after these operations are rolled back in the +same process. A process or host crash between the two renames can nevertheless +leave the final name absent and one hidden backup present. The next supported +CLI invocation removes abandoned staging directories and restores that sole +backup before starting benchmark work. Multiple backups are ambiguous evidence; +the CLI fails closed for operator review rather than guessing which set is +authoritative. This contract provides recoverable transactional publication, +not an unsupported claim of crash-atomic replacement on every filesystem. + +## Security and optional-adapter boundary + +The adapter does not accept credentials, open sockets, change routing, infer +prices, or mutate benchmark globals. The benchmark renderer performs its normal +credential-leak checks while writing the private staged files. Assignment +enrichment copies only already validated assignment fields from that staged, +secret-scanned JSON report. The adapter is imported only by the explicit +`nim-benchmark` CLI branch, preserving side-effect-free ordinary package import +and the optional-adapter boundary. + +## Deliberate limits + +- The JSON report remains authoritative. CSV is a loss-minimized projection for + spreadsheet and data-warehouse consumers, not a replacement for nested JSON. +- The compact assignment value is deterministic JSON, but this implementation + does not claim full JSON Canonicalization Scheme conformance. +- Programmatic callers that invoke `run_benchmark` directly receive the original + renderer semantics and may call `enrich_benchmark_cell_csv` explicitly. The + supported CLI and scheduled workflow provide complete-set transactional + publication automatically. +- Crash recovery is invocation-driven; no background daemon is introduced. +- No release or routing recommendation is created from this transformation. + +## Verification + +`tests/test_nim_csv_evidence.py` and +`tests/test_nim_csv_evidence_edges.py` cover deterministic serialization, +idempotence, malformed report shapes, invalid assignment values, duplicate and +mismatched identities, missing CSV columns, invalid JSON, output-directory +parsing, benchmark-failure passthrough, success withholding, and fail-closed +enrichment. + +`tests/test_nim_artifact_publication.py` and +`tests/test_nim_artifact_publication_edges.py` cover fresh-target failure, prior +set preservation, ordinary mid-publication rollback, crash-backup recovery, +ambiguous-backup rejection, staging and backup cleanup, target-path safety, +complete-directory shape validation, final-path result rewriting, and malformed +success-payload rejection. The dedicated NIM quality job includes all four test +modules in the 100% production statement and branch coverage and public-docstring +gates, then builds, installs, and imports the wheel. + +## References + +Bray, T. (2017). *The JavaScript Object Notation (JSON) data interchange format* +(RFC 8259). Internet Engineering Task Force. https://doi.org/10.17487/RFC8259 + +National Institute of Standards and Technology. (2022). *Secure software +development framework (SSDF) version 1.1: Recommendations for mitigating the +risk of software vulnerabilities* (NIST Special Publication 800-218). +https://doi.org/10.6028/NIST.SP.800-218 + +Shafranovich, Y. (2005). *Common format and MIME type for comma-separated values +(CSV) files* (RFC 4180). Internet Engineering Task Force. +https://doi.org/10.17487/RFC4180 diff --git a/docs/doctoring/nim-benchmark-evidence-grade.md b/docs/doctoring/nim-benchmark-evidence-grade.md new file mode 100644 index 00000000..8046ccdc --- /dev/null +++ b/docs/doctoring/nim-benchmark-evidence-grade.md @@ -0,0 +1,257 @@ +# Evidence-grade NVIDIA NIM benchmark: engineering decision record + +## Decision + +The NVIDIA NIM benchmark is an optional, provider-neutral evaluation adapter. +It is not imported by the normal package initializer and it never modifies the +runtime gateway as an import side effect. Live execution uses the same +validation-time-address-pinned HTTPS boundary as the gateway, while dry +execution remains deterministic, network-free, and credential-free. + +The benchmark is evidence-generating rather than policy-authorizing. It records +what was discovered, planned, attempted, completed, failed, measured, estimated, +and unknown. It never changes production routing automatically. A report below the +explicit evidence floor is labeled `insufficient_evidence`, and every report +keeps `routing_recommendation` null so a responsible human review remains +necessary. + +## Architecture and MSA boundary + +`contextual_orchestrator/nim_benchmark.py` owns catalog discovery, capability +probing, equal-budget policy comparison, evidence validity, uncertainty, +Pareto analysis, and artifact serialization. The ordinary gateway remains +standalone and provider-neutral. Other ContextualWisdomLab services may invoke +the benchmark as a module or CLI without taking ownership of its transport, +credential, pricing, or evidence rules. + +The boundary preserves the following responsibilities: + +- the host workflow owns GitHub Secret delivery and immutable run provenance; +- the benchmark moves the secret into the process-local credential registry and + resolves it by the `NVIDIA_NIM_API_KEY` credential name; +- the benchmark owns bounded provider calls and secret-redacted artifacts; +- the central `.github` repository owns independent review and protected-branch + policy; and +- consumers such as naruon may read artifacts but do not receive authority to + reinterpret unknown prices or incomplete evidence as production facts. + +## Provider-egress security contract + +A conventional URL opener is not used for live NIM requests. Validation and +connection are one security boundary: + +1. Parse an HTTPS URL and reject missing hostnames. +2. Resolve the hostname once for that request. +3. Reject any answer that is not globally routable, including private, + loopback, link-local, multicast, reserved, unspecified, IPv6 unique-local, + and RFC 6598 shared address space. +4. Dial only an address from that exact validation result. +5. Preserve the original hostname for HTTP authority, TLS SNI, and certificate + hostname verification. +6. Do not consult environment proxy settings. +7. Reject every redirect before a bearer credential can reach another origin. +8. Close responses and connections deterministically and use only another + address from the same validation result for fallback. +9. Read at most 8 MiB plus one sentinel byte from a provider response and fail + closed before an oversized body can be materialized into benchmark evidence. + +RFC 6598 defines `100.64.0.0/10` as shared, non-globally-routable address space. +RFC 4193 defines IPv6 unique-local addresses as local rather than globally +routable. RFC 9110 defines redirects as new target-URI actions and identifies +`Authorization` as resource-specific credential material that merits removal +when redirecting. The implementation chooses the narrower fail-closed policy of +not following redirects at all. + +## Dynamic discovery and deterministic probe allocation + +`GET /v1/models` is the run-time source of the model inventory. Model identifiers +are not hard-coded as authoritative catalog entries. The parser records invalid +and duplicate entries and sorts the usable inventory. + +Every discovered model must receive a completed outcome row for every supported +probe contract in a successful live run. Immediately after the catalog request, +the benchmark computes one complete request plan containing the catalog request, +all `(model_id, capability_name)` probes, and the worst-case equal-budget policy +evaluation reserve. If the configured hard cap is even one request short, the +run fails closed before the first capability probe; a lexicographic model prefix +can never be emitted as routing-readiness evidence. + +The acceptance fixture uses 127 discovered models, nine capability contracts, +seven evaluation workers, and thirty locked tasks. Its complete upper bound is +`1 + (127 × 9) + (30 × (7 + 1 + 5 + 1)) = 1,564` requests. The monthly workflow +uses a reviewed hard ceiling of 2,000 requests, leaving bounded room for catalog +growth while retaining a deterministic cap. If a later catalog no longer fits, +the same preflight reports required and configured counts and makes zero partial +probe calls. Once admitted, all probe cells execute under bounded concurrency; +thread scheduling changes only completion order, never inventory coverage or +evaluation capacity. + +The video-understanding probe contains a deterministic, decodable one-frame H.264 +MP4. Its embedded bytes have SHA-256 +`777dda43b5a15162b68a39aa486d5c70c9994d7fe761742fd00d4e13508983c0`. +Startup validation confirms the ISO Base Media File Format structure, a video +handler, AVC sample entry, 16 × 16 dimensions, one sample, and media data. This +prevents a malformed `ftyp`-only stub from turning a capable video model's valid +rejection into a false unsupported classification. + +## Fair policy comparison + +Direct single-worker, `route_once`, bounded `conduct`, and reviewed +cheapest-worker cells receive the same per-task contract: + +- one locked task and scorer version; +- one total prompt-plus-completion token allowance; +- one five-call maximum envelope; +- one timeout policy; and +- one workflow-depth ceiling. + +The token allowance is cell-wide rather than per request. Prompt estimates are +charged before a call, the output cap is reduced to the remaining allowance, +and provider-reported usage replaces the latest estimate when valid. Booleans, +negative values, NaN, and infinities are not accepted as token counts. A deep +policy cannot obtain five times a single-call arm's total token budget merely by +issuing five calls. + +## Cost evidence and price honesty + +Actual access cost and hypothetical production cost are separate fields and +separate evidence classes. + +As reviewed on 2026-08-05, NVIDIA's NIM General FAQ states that NVIDIA Developer +Program members have free access to hosted NIM API endpoints for prototyping. +The same source distinguishes development, testing, research, and evaluation +from production and states that production requires NVIDIA AI Enterprise. The +report therefore records `actual_cost_usd = 0.0` only for the reviewed hosted +endpoint access context, includes the exact source, review date, validity +horizon, program scope, production distinction, and uncertainty, and refuses a +live run after 2026-09-04 until the source is reviewed again. + +No NVIDIA model price is embedded or inferred. A live hypothetical pricing +scenario is optional; absence means `unknown`. If supplied, it must be marked +`reviewed` and include an HTTPS source, reviewer, review date, validity horizon, +rate basis, uncertainty, and explicit input/output rates. Unreviewed, future, +incomplete, or expired evidence fails before network egress. The included +example remains deliberately `example_unreviewed` and is valid only for dry-run +schema testing. + +NVIDIA's offering documentation further distinguishes exploratory/free NIM +availability from NIM Certified, which requires NVIDIA AI Enterprise for +enterprise lifecycle, CVE, support, and compliance expectations. The benchmark +does not convert free prototype access into a claim about production licensing, +support, or per-model production price. + +## Evidence sufficiency and uncertainty + +The bundled locked split now contains thirty original, objectively scored tasks, +so the scheduled benchmark can reach the repository's paired-task count floor. +This repairs a product-validity gap in the former ten-task smoke fixture, which +could exercise the pipeline but could never produce `evidence_review_required` +under the benchmark's own minimum-pair contract. + +The governance floor is: + +- at least 30 locked paired tasks shared by compared policies; and +- at least 90% successful cells across the requested comparison matrix. + +The first threshold prevents a scheduled run from being structurally condemned +to smoke-only evidence. It does not establish domain representativeness, +statistical power for every effect size, or production readiness. The tasks use +objective exact-number or substring scorers and include arithmetic, logic, +conversion, factual recall, language, and sequence cases, but buyers must still +review whether this coverage matches their workloads. HELM's emphasis on +multi-metric, scenario-explicit evaluation supports exposing such coverage +limits rather than hiding them behind a single score. + +These values are explicit conservative release-governance thresholds, not a +claim of universal statistical sufficiency. The artifact reports the observed +paired-task count, requested thresholds, completion fraction, and whether the +run is `insufficient_evidence` or `evidence_review_required`. Even when the floor +is met, production routing remains a human decision and +`routing_recommendation` stays null. + +Paired bootstrap intervals preserve task pairing and expose uncertainty in mean +score differences. Pareto frontiers show quality against latency and reviewed +hypothetical cost; policies with unknown cost are excluded from that cost +frontier and named explicitly. HELM motivates standardized multi-metric +conditions and visible incompleteness. FrugalGPT and RouteLLM motivate measuring +cost-quality routing trade-offs, but their results are not treated as evidence +for this repository's models or tasks. + +## Workflow and credential separation + +`.github/workflows/nim-benchmark.yml` has separate dry and live jobs. The dry job +never receives `NVIDIA_NIM_API_KEY`. Only the live benchmark step receives the +GitHub Secret, and the credential is never passed through argv or written to an +artifact. Both jobs use immutable action revisions, bounded execution, and +single-flight concurrency. The workflow cannot merge, release, approve its own +changes, or modify routing configuration. + +The ordinary test workflow separately proves: + +- complete focused production statement and branch coverage; +- 100% public docstrings; +- wheel build and clean-environment import; +- no eager optional benchmark import; +- no compatibility monkeypatch module; +- no temporary branch-writing or source-export repair job; and +- no retained one-use transformation payload. + +## Verification contract + +An exact pull-request head is eligible for review only after all of the following +succeed: + +- deterministic unit and adversarial security tests; +- transport tests for DNS rebinding, proxy isolation, redirects, SNI/authority, + address fallback, bounded response bodies, and cleanup; +- complete-plan preflight tests for 127 models, the exact boundary, one request short, zero partial egress, deterministic concurrency, and a valid media fixture; +- a permanent acceptance test proving thirty unique locked task IDs and the + 1,564-request representative complete-plan total; +- live pricing and access-evidence expiry tests that prove failure before egress; +- equal token/call budget tests for every comparison arm; +- evidence-sufficiency, Pareto, provenance, and secret-redaction tests; +- 100% statement and branch coverage for the production benchmark module; +- 100% public docstrings; +- package build, install, and import smoke tests; +- repository Tests, Fuzz, Security, Security Scan, and SAST; and +- independent exact-head review with no unresolved actionable thread. + +No earlier head, local-only result, queued check, or stale approval is accepted as +release evidence. + +## References + +Autio, C., Schwartz, R., Dunietz, J., Jain, S., Stanley, M., Tabassi, E., Hall, +P., & Roberts, K. (2024). *Artificial intelligence risk management framework: +Generative artificial intelligence profile* (NIST AI 600-1). National Institute +of Standards and Technology. https://doi.org/10.6028/NIST.AI.600-1 + +Chen, L., Zaharia, M., & Zou, J. (2023). FrugalGPT: How to use large language +models while reducing cost and improving performance. *arXiv*. +https://doi.org/10.48550/arXiv.2305.05176 + +Fielding, R., Nottingham, M., & Reschke, J. (2022). *HTTP semantics* (RFC 9110; +STD 97). RFC Editor. https://doi.org/10.17487/RFC9110 + +Hinden, R., & Haberman, B. (2005). *Unique local IPv6 unicast addresses* +(RFC 4193). RFC Editor. https://doi.org/10.17487/RFC4193 + +Liang, P., Bommasani, R., Lee, T., Tsipras, D., Soylu, D., Yasunaga, M., Zhang, +Y., Narayanan, D., Wu, Y., Kumar, A., Newman, B., Yuan, B., Yan, B., Zhang, C., +Cosgrove, C., Manning, C. D., Ré, C., Acosta-Navas, D., Hudson, D. A., … Koreeda, +Y. (2023). Holistic evaluation of language models. *Transactions on Machine +Learning Research*. https://doi.org/10.48550/arXiv.2211.09110 + +NVIDIA Corporation. (n.d.). *General FAQ*. NVIDIA NIM Documentation. Retrieved +August 5, 2026, from https://docs.api.nvidia.com/nim/docs/product + +NVIDIA Corporation. (2026, June 4). *NIM offerings*. NVIDIA NIM for Large +Language Models. https://docs.nvidia.com/nim/large-language-models/2.0.5/about-nim-llm/nim-offerings.html + +Ong, I., Almahairi, A., Wu, V., Chiang, W.-L., Wu, T., Gonzalez, J. E., Kadous, +M. W., & Stoica, I. (2024). RouteLLM: Learning to route LLMs with preference +data. *arXiv*. https://doi.org/10.48550/arXiv.2406.18665 + +Weil, J., Kuarsingh, V., Donley, C., Liljenstolpe, C., & Azinger, M. (2012). +*IANA-reserved IPv4 prefix for shared address space* (RFC 6598; BCP 153). RFC +Editor. https://doi.org/10.17487/RFC6598 diff --git a/docs/doctoring/nim-benchmark-security-integration-receipt.md b/docs/doctoring/nim-benchmark-security-integration-receipt.md new file mode 100644 index 00000000..044a9843 --- /dev/null +++ b/docs/doctoring/nim-benchmark-security-integration-receipt.md @@ -0,0 +1,101 @@ +# NIM benchmark security-integration receipt + +## Status + +This record binds the evidence-grade NVIDIA NIM benchmark to the provider-egress +security base without treating an older pull-request head as current approval +evidence. It is an architecture decision record and verification receipt, not a +release declaration or production routing recommendation. + +## Current review identity + +The current review target is the exact head and base reported by GitHub for +`ContextualWisdomLab/contextual-orchestrator#90` at review time. This tracked +file intentionally does not hard-code its own head SHA: committing such a value +would immediately create a new head and make the value stale. The pull-request +body and GitHub Checks are the current identity and evidence index. + +No workflow run or approval is recorded here as current-head acceptance evidence. +Every head or base change requires fresh exact-head checks and independent +review before readiness changes. + +## Historical integration evidence + +The following identifiers are retained only as historical integration evidence. +They must not be used as current approval, branch-protection, or merge evidence: + +- benchmark source head `9b72816bb61fa540b8eb20fdc559e740a3c3c6ec`; +- provider-egress security head `03124cf97b7bf02e30a48a13acfd78b6ef08d1ef`; +- resulting historical integration commit + `a2af3634d67a361cac18ba11ff9b8db24417b646`; and +- historical integration workflow run `31005510483`. + +The benchmark remains stacked on the provider-egress security branch until that +branch reaches protected `main`. Every later head or base change invalidates +prior checks and approvals automatically. + +## Preserved security invariants + +The integrated tree must preserve all of these properties: + +1. HTTPS provider sockets dial only validation-time globally routable addresses. +2. The original provider hostname remains the HTTP authority, TLS SNI value, and + certificate-verification identity. +3. Redirects and ambient proxy routing cannot forward provider credentials. +4. Plain HTTP remains restricted to literal loopback integration targets. +5. Provider responses are bounded before materialization. +6. Importing `contextual_orchestrator` does not eagerly load the optional NIM + adapter or monkey-patch runtime classes. +7. Dry benchmark execution receives no `NVIDIA_NIM_API_KEY`; the bounded live + job alone owns the GitHub Secret binding. +8. `COPILOT_GITHUB_TOKEN` is not a supported benchmark or review credential. + +## Preserved evaluation invariants + +The integrated benchmark must also preserve these evidence properties: + +1. `GET /v1/models` is the run-time inventory authority. +2. The complete request plan is calculated after discovery and before the first + capability probe. +3. An undersized hard cap fails before partial model probing. +4. Every discovered model receives every required capability probe when the run + proceeds. +5. Direct, route-once, conduct, and reviewed cheapest-worker cells receive the + same total prompt-plus-completion token allowance and maximum-call envelope. +6. Actual free-to-caller access evidence and hypothetical paid pricing remain + distinct, versioned evidence classes. +7. Unknown model prices remain `unknown`; no rate is inferred or invented. +8. Evidence below the configured task and completion floors reports + `insufficient_evidence` and cannot authorize a production routing change. + +## Historical verification evidence + +Historical integration commit `a2af3634d67a361cac18ba11ff9b8db24417b646` +was recorded as passing: + +- 449 repository tests; +- 111 focused NIM tests; +- 982 production statements at 100% coverage; +- 374 production branches at 100% coverage; +- 100% public-docstring coverage; +- Python compilation; +- wheel build, isolated installation, and import; and +- `git diff --check`. + +That historical run had one reviewed conflict, in `CHANGELOG.md`. Its +deterministic resolution retained both the NIM complete-request-plan evidence +and the security base's APA 7 environment-marker doctoring entry. These results +do not establish the status of any later pull-request head. + +## Current-head acceptance rule + +This receipt does not satisfy branch protection by itself. After every head or +base change, repository Tests, Fuzz, Security, Security Scan, SAST Semgrep, +central coverage, OpenCode, Noema, Strix, CodeRabbit, packaging, and all other +required checks must rerun on the exact current head and base. A non-author +independent approval is mandatory. Queued, skipped, action-required, stale-head, +or failed results are not success. + +After the security prerequisite merges, this branch must be retargeted to the +resulting protected `main` and revalidated without modifying the accepted source +behavior. Only then may the pull request become Ready for review. diff --git a/docs/doctoring/nim-benchmark-strict-answer-scoring.md b/docs/doctoring/nim-benchmark-strict-answer-scoring.md new file mode 100644 index 00000000..d57bc5c4 --- /dev/null +++ b/docs/doctoring/nim-benchmark-strict-answer-scoring.md @@ -0,0 +1,216 @@ +# NIM benchmark strict complete-answer scoring + +## Decision + +The supported `python -m contextual_orchestrator nim-benchmark` composition root +uses versioned complete-answer scorers for every locked evaluation task. Legacy +containment scorers remain registered only for historical compatibility and the +exploratory tuning split. They are not used for headline policy comparison. + +The strict scoring policy is activated explicitly by the benchmark command. It +is not imported or installed by ordinary `import contextual_orchestrator`, so +the optional benchmark remains outside the standalone gateway's import path and +does not mutate runtime globals eagerly. + +## Validity gap + +The original numeric scorer awarded credit when an expected number appeared +anywhere in a response. The original text scorer awarded credit when an expected +string appeared as a case-insensitive substring. Those contracts were useful as +simple smoke-test fixtures, but they were not defensible quality evidence: + +- `not 21` could receive the same numeric score as `21`; +- `Australia` could satisfy an expected chemical symbol of `Au`; +- explanatory prose, contradictory alternatives, units, and multiple answers + could receive credit despite prompts requiring an answer only; and +- one global case-folding rule would either reject harmless capitalization in + names or incorrectly accept a case-sensitive symbol such as `au` for `Au`. + +The Korean word `사과` also has a fruit sense and an apology-related sense. A +locked translation prompt that omitted the fruit context could reward or punish +a model for resolving an ambiguity rather than for translation quality. The +authoring prompt now names the fruit context explicitly. + +A second validity gap appeared after complete-answer scorers were introduced. +The original no-leakage check asked the scorer to grade the whole prompt. A +complete-answer scorer correctly gives a prompt sentence a zero, even when that +sentence embeds the answer token. Thus `Hint: the result is 21.0` could evade a +numeric answer key of `21`, and a declared alias such as `Pacific Ocean` could +be present in the prompt without being detected. + +These are construct-irrelevant score effects. They can change policy means, +bootstrap differences, Pareto membership, and the apparent advantage of direct, +route-once, or conduct policies without any real improvement in task accuracy. + +## Versioned scoring contracts + +### Exact finite number, version 2 + +`exact_number_match` version `2` requires the entire trimmed response to be one +finite ASCII decimal literal. The literal is parsed with decimal arithmetic, so +numerically equivalent forms such as `21`, `21.0`, and `2.1e1` compare equally +without binary floating-point rounding. Prose, units, negation, multiple values, +`NaN`, and infinities score zero. The expected value itself must be a string +containing one finite decimal literal; malformed answer keys fail before +provider egress. + +Version `1` remains unchanged for backward compatibility and is excluded from +the derived locked evidence manifest. + +### Exact normalized text, version 1 + +`exact_text_match` version `1` compares the complete response against an +explicit non-empty list of accepted answers. Both sides use Unicode NFC plus +whitespace trimming and collapse. Each task declares whether comparison is +case-sensitive. Case-insensitive tasks additionally use Unicode case folding; +case-sensitive tasks retain case after NFC and whitespace normalization. + +This treats canonically equivalent text consistently while avoiding Unicode +compatibility normalization that could erase meaningful distinctions. It also +lets capital-city and ordinary-name tasks accept harmless capitalization while +requiring exact case for a chemical symbol. Substrings, explanations, negations, +and undeclared aliases do not match. Empty, non-string, or duplicate normalized +answer keys and non-Boolean case policies fail before provider egress. + +Accepted alternatives must be declared in the answer key. For example, the +largest-ocean task explicitly accepts `Pacific` and `Pacific Ocean`; the scorer +does not invent synonyms, translations, abbreviations, prices, or semantic +equivalence. + +### Prompt no-leakage contract + +The derived locked manifest is reviewed independently of response scoring before +any credential lookup or provider request. + +- Numeric prompts are scanned for complete ASCII decimal tokens. Each token is + parsed with decimal arithmetic and compared by value, so `21.0` and `21` are + equivalent leakage while `121` is not. +- Text prompts are normalized with the same NFC, whitespace, and task-specific + case policy as the answer key. Every declared alias is searched at Unicode + word boundaries, so `Pacific Ocean` is caught, `Au` is not inferred from + `Australia`, and lower-case `au` does not leak the case-sensitive symbol + `Au`. +- Invalid numeric tokens that Python cannot represent do not abort review unless + they are the expected answer key. +- Missing, empty, oversized, or non-string locked prompts fail before egress. +- Unknown scorer identities cannot bypass the leakage dispatcher. + +This check is intentionally lexical and answer-key-driven. It does not claim to +detect paraphrased or semantically encoded leakage, and it never asks an LLM to +judge its own benchmark prompt. + +### Strict-scoring resource boundary + +Every expected alias, expected numeric literal, locked prompt, and model answer +is limited to 4,096 Unicode code points before normalization or decimal parsing. +The limit is a conservative implementation guard for answer-only tasks, not a +statistical or linguistic sufficiency claim. + +- An oversized expected value or locked prompt is an invalid manifest and fails + before provider egress. +- An oversized model answer scores zero rather than allocating unbounded + normalization or decimal resources. +- A syntactically matched decimal exponent that Python cannot represent is + classified as an unusable model answer and scores zero instead of aborting the + benchmark. +- Expected numeric conversion errors remain manifest errors, preserving the + distinction between invalid scoring keys and failed model responses. + +The provider-response 8 MiB boundary remains independent and authoritative for +network body consumption. The smaller scoring cap prevents a bounded but still +large response from becoming a CPU or memory amplification input at the scoring +layer. + +## Authoring and evidence provenance + +The repository keeps legacy scorer fields in the reviewed authoring manifest +for historical compatibility, while adding scoring-side strict metadata where +material: + +- `strict_texts` declares accepted complete-answer aliases; +- `strict_case_sensitive` declares case-sensitive matching; and +- disambiguating prompt context is author-visible but no expected answer is + injected into the model request. + +Immediately before the supported benchmark CLI starts, the strict composition +root: + +1. reads the selected manifest without resolving a provider credential; +2. deep-copies it; +3. validates and upgrades locked numeric scorer `1` to numeric scorer `2`; +4. converts locked substring expectations into explicit exact-text answer lists + and case policies; +5. validates already-strict numeric and text answer keys; +6. rejects equivalent numeric tokens and declared text aliases embedded in each + locked prompt; +7. preserves exploratory tasks and already-strict locked tasks; +8. rejects unknown locked scorer contracts; +9. adds `scoring_policy_version = 2026-08-07.4` and a derived manifest version; +10. writes the deterministic derived manifest to an owner-only temporary + directory; and +11. invokes the existing benchmark with that path. + +The existing task-manifest SHA-256 and manifest-version fields therefore bind +artifacts to the exact derived scoring contract that produced them. The private +file is deleted after the one-shot command. No credential, model response, +price, or routing decision enters the transformation. + +## Security and authority boundaries + +- Importing the normal package does not import the benchmark or strict scorer. +- Activation adds only previously unowned versioned scorer identities and fails + closed on a registry collision. +- The transformation opens no socket and reads no provider credential. +- Ambiguous manifest selectors, malformed JSON, unsupported locked scorers, + leaked answers, invalid aliases, invalid case policies, oversized expected + values or prompts, and invalid numeric keys fail before provider egress. +- Oversized or unrepresentable model answers cannot abort the benchmark or + consume the full provider-response allowance inside the scorer. +- The benchmark remains evidence-generating. Strict scoring does not authorize + a production route, price, merge, or release. + +## Verification + +`tests/test_nim_strict_scorer_validity.py` proves full-response numeric and text +validity, task-declared case semantics, explicit aliases, Korean fruit-context +authoring, invalid answer-key rejection, explicit activation, scorer ownership, +deterministic manifest conversion, and private temporary-manifest lifecycle. + +`tests/test_nim_strict_scoring_bounds.py` proves the 4,096-character answer +contract, zero-score handling for oversized and unrepresentable model answers, +and fail-before-egress rejection of oversized expected values. + +`tests/test_nim_strict_scoring_leakage.py` proves decimal-equivalent leakage, +complete numeric-token boundaries, declared multi-word aliases, task-specific +case behavior, larger-word false-positive prevention, punctuation aliases, +unrepresentable unrelated numeric tokens, prompt type and size boundaries, and +fail-closed unknown scorer dispatch. + +`tests/test_nim_strict_scoring_integration.py` proves ordinary package-import +isolation and runs the supported transactional publication path end to end, +requiring every locked evaluation cell to carry only the strict scorer versions +and leaving `routing_recommendation` null. + +The permanent NIM quality workflow includes all four strict-scoring test modules +and this production module in 100% statement and branch coverage, 100% public +docstrings, wheel packaging, and installed-package import checks. + +## References + +Autio, C., Schwartz, R., Dunietz, J., Jain, S., Stanley, M., Tabassi, E., Hall, +P., & Roberts, K. (2024). *Artificial intelligence risk management framework: +Generative artificial intelligence profile* (NIST AI 600-1). National Institute +of Standards and Technology. https://doi.org/10.6028/NIST.AI.600-1 + +Liang, P., Bommasani, R., Lee, T., Tsipras, D., Soylu, D., Yasunaga, M., Zhang, +Y., Narayanan, D., Wu, Y., Kumar, A., Newman, B., Yuan, B., Yan, B., Zhang, C., +Cosgrove, C., Manning, C. D., Ré, C., Acosta-Navas, D., Hudson, D. A., … +Koreeda, Y. (2023). Holistic evaluation of language models. *Transactions on +Machine Learning Research*. https://doi.org/10.48550/arXiv.2211.09110 + +Python Software Foundation. (2026). *decimal—Decimal fixed-point and +floating-point arithmetic*. Python 3 documentation. Retrieved August 7, 2026, +from https://docs.python.org/3/library/decimal.html + +Unicode Consortium. (2025). *Unicode normalization forms* (Unicode Standard +Annex #15, Revision 57, Unicode 17.0.0). https://www.unicode.org/reports/tr15/ diff --git a/docs/doctoring/pull-request-exact-head-verification.md b/docs/doctoring/pull-request-exact-head-verification.md new file mode 100644 index 00000000..617667a4 --- /dev/null +++ b/docs/doctoring/pull-request-exact-head-verification.md @@ -0,0 +1,63 @@ +# Pull-request exact-head verification boundary + +## Decision + +Repository-local Tests, Fuzz, and Security jobs explicitly check out +`github.event.pull_request.head.sha` for `pull_request` events. Push, schedule, +and manual runs continue to use `github.sha`. Checkout credentials remain +non-persistent. + +GitHub documents that an open, mergeable pull request ordinarily sets +`GITHUB_REF` to `refs/pull//merge` and `GITHUB_SHA` to the synthetic +merge commit. The official `actions/checkout` documentation therefore gives an +explicit `ref: ${{ github.event.pull_request.head.sha }}` example for checking +the contributor head instead of that merge commit. + +This repository needs both evidence classes, but it must not confuse them: + +- **Exact-head evidence** proves the immutable contributor SHA that review and + approval refer to. +- **Integration evidence** proves a clearly identified merge result against a + clearly identified base. + +A successful workflow associated with a branch SHA is not automatically +exact-head evidence. The checked-out commit in the job log is authoritative. +Queued, pending, cancelled, predecessor-head, stale-base, or synthetic-merge +results cannot be relabeled as current-head success. + +## Threat and permission boundary + +These workflows use the `pull_request` event, read-only repository permissions, +and no provider or deployment secret. Selecting the pull-request head therefore +does not convert a privileged `pull_request_target` or secret-bearing workflow +into an untrusted-code execution path. The explicit ref is intentionally limited +to the repository-local verification workflows covered by the regression +contract. + +Organization-central workflows retain their own authority and threat model. +Their results must be classified from their actual checkout behavior. A central +workflow that intentionally evaluates a merge commit remains useful integration +evidence, but it does not satisfy an exact-head gate unless it separately proves +the contributor SHA under its documented policy. + +## Regression contract + +`tests/test_pr_exact_head_workflows.py` counts every `actions/checkout` use in: + +- `.github/workflows/tests.yml` +- `.github/workflows/fuzz.yml` +- `.github/workflows/security.yml` + +For every checkout, the test requires the event-aware exact-head expression and +`persist-credentials: false`. Adding another checkout without the same boundary +fails the full unit and contract suite. + +## References + +GitHub. (2026). *Events that trigger workflows: How the merge branch affects +your workflow*. GitHub Docs. +https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows + +GitHub. (2026). *Checkout pull request HEAD commit instead of merge commit*. +In *actions/checkout*. GitHub. +https://github.com/actions/checkout#checkout-pull-request-head-commit-instead-of-merge-commit diff --git a/docs/nim_benchmark.md b/docs/nim_benchmark.md new file mode 100644 index 00000000..f4f6bed4 --- /dev/null +++ b/docs/nim_benchmark.md @@ -0,0 +1,221 @@ +# NIM model discovery + cost-quality benchmark + +The optional benchmark harness (`contextual_orchestrator/nim_benchmark.py`) +addresses issue #86: generate reproducible evidence about how the repository's +routing policies behave on a **real, dynamically discovered** model pool. +NVIDIA NIM is an evaluation provider, not a runtime dependency. Importing the +normal `contextual_orchestrator` package does not import or mutate the optional +benchmark module. + +The detailed engineering and evidence record is +[`docs/doctoring/nim-benchmark-evidence-grade.md`](doctoring/nim-benchmark-evidence-grade.md). + +## Run it + +```bash +# Deterministic dry run: validates manifests, scorers, budgets, evidence +# sufficiency, and artifact schemas against an in-process provider. It performs +# no network calls and never receives NVIDIA_NIM_API_KEY. +python -m contextual_orchestrator nim-benchmark --dry-run \ + --pricing-scenario examples/nim_pricing_scenario.json \ + --output-dir benchmark_artifacts + +# Live CI run: the workflow injects NVIDIA_NIM_API_KEY only into the live step. +# The process bootstraps it into the credential registry and runtime access +# resolves the credential by name. +python -m contextual_orchestrator nim-benchmark \ + --max-total-requests 2000 \ + --max-output-tokens 256 \ + --git-sha "$GITHUB_SHA" \ + --workflow-run-id "$GITHUB_RUN_ID" +``` + +The provider secret is never accepted through argv, printed, or serialized. +Artifact writing fails closed if the resolved secret appears in any output. + +## Provider-egress security boundary + +Catalog discovery, probes, and live policy evaluation use validation-time +address pinning: + +- every provider URL must use HTTPS; +- each request resolves its hostname exactly once; +- every answer must be globally routable, so RFC 6598 shared space, private, + loopback, link-local, multicast, reserved, unspecified, and IPv6 unique-local + addresses are rejected; +- the socket dials only an address from that exact DNS answer; +- HTTP authority, TLS SNI, and certificate hostname verification retain the + original hostname; +- environment proxy settings are not consulted; +- redirects are rejected rather than followed; and +- address fallback is limited to the same validation result; and +- every provider response is read through an 8 MiB hard cap before it can be + materialized in memory. + +This closes the DNS time-of-check/time-of-use gap created by validating a +hostname and then letting a generic URL opener resolve it again. + +## Dynamic catalog and all-modality probes + +The live inventory comes from the OpenAI-compatible `GET /v1/models`; no +hard-coded list is treated as authoritative. The parser deduplicates and sorts +usable identifiers while retaining invalid-entry and duplicate evidence. Zero +usable models fails the run closed. + +Every discovered model receives a row for each contract: + +| Probe | Endpoint or request contract | +| --- | --- | +| `chat_completion` | `POST /chat/completions` | +| `text_completion` | `POST /completions` | +| `response_generation` | `POST /responses` | +| `text_embedding` | `POST /embeddings` | +| `image_understanding` | chat with a tiny PNG `image_url` part | +| `video_understanding` | chat with a validated one-frame MP4 `video_url` part | +| `audio_understanding` | chat with a tiny WAV `input_audio` part | +| `audio_transcription` | `POST /audio/transcriptions` | +| `audio_speech` | `POST /audio/speech` | + +After catalog discovery and before the first capability request, the harness +constructs the complete request plan: one discovery request, every sorted +`(model_id, capability_name)` probe, and the conservative evaluation reserve for +the maximum eligible worker pool. If the configured cap is even one request +short, the run fails closed before capability egress and reports the required +and configured counts. Partial model-major prefixes cannot produce routing +evidence. Once preflight passes, all fixed cells execute under bounded +concurrency; thread scheduling can change completion order but not coverage. + +The monthly schedule uses a hard ceiling of 2,000 requests. At the representative +127-model catalog scale, the thirty-task, seven-worker configuration requires +1,564 requests: one catalog request, 1,143 capability probes, and a 420-request +worst-case evaluation reserve. Catalog growth beyond the ceiling causes a +zero-partial-egress preflight failure rather than silent truncation. + +The embedded video fixture is a deterministic, decodable 16 × 16, one-frame +H.264 MP4. Its bytes are verified against SHA-256 +`777dda43b5a15162b68a39aa486d5c70c9994d7fe761742fd00d4e13508983c0`, and its +container structure, video handler, AVC sample entry, dimensions, sample count, +and media data are validated before use. + +Probe outcomes are `supported`, `unsupported`, `rate_limited`, `timeout`, +`unavailable`, `failed`, `malformed_response`, or `skipped`. HTTP 401 fails the +whole run closed. Model-level classes are derived from observations rather than +model names or marketing metadata. + +## Fair comparison contract + +Every policy × task cell receives the same: + +- locked task and scorer version; +- total prompt-plus-completion token allowance configured by + `--max-output-tokens`; +- five-call maximum envelope; +- timeout policy; and +- five-step workflow-depth ceiling. + +The token allowance is cell-wide, not per call. Prompt tokens are charged before +a request, the output cap is reduced to the remaining allowance, and valid +provider-reported usage replaces the latest estimate. A deep `conduct` path +cannot receive five times a direct arm's total token budget merely because it +uses more calls. + +Compared policies are: + +1. one direct baseline per chat-eligible worker; +2. deterministic `route_once`; +3. bounded `conduct_bounded`; and +4. a cheapest eligible worker only when an explicit reviewed pricing scenario + supports that comparison. + +Each cell records configured and observed budgets, score and scorer version, +outcome and reason, latency, depth, usage source, model/role/step assignments, +actual and hypothetical cost fields, and a response SHA-256. + +## Cost honesty and evidence validity + +Actual endpoint access and hypothetical paid cost remain separate evidence +classes. + +As reviewed on 2026-08-05, NVIDIA's current General FAQ states that NVIDIA +Developer Program members have free access to hosted NIM API endpoints for +prototyping. The report records that exact source, review date, validity horizon, +program context, production distinction, and uncertainty. A live run fails +closed after 2026-09-04 until the official source is reviewed again. Production +support and licensing are not inferred from prototype access and require +NVIDIA AI Enterprise under the reviewed documentation. + +Hypothetical paid cost is computed only from an explicit pricing scenario. +Omitting a scenario is valid and leaves cost `"unknown"`. A live scenario must +be marked `reviewed` and contain an HTTPS source, reviewer, review date, validity +horizon, rate basis, uncertainty, and explicit input/output rates. Unreviewed, +future, incomplete, or expired scenarios fail before provider egress. The +included example is intentionally `example_unreviewed`; it exists only to test +dry-run schemas and must never be presented as real model pricing. + +## Evidence sufficiency and uncertainty + +The bundled manifest contains thirty original, objectively scored locked tasks +and therefore reaches the repository's paired-task count floor. That count is a +governance prerequisite, not proof that the task sample represents every buyer +domain or that any routing policy is ready for production. A report reaches +`evidence_review_required` only when it also completes at least 90% of the +requested comparison cells; otherwise it reports `insufficient_evidence` and +names the shortfall. + +These thresholds are explicit conservative governance floors, not universal +statistical guarantees. Every report keeps `routing_recommendation` null even +when both thresholds are met, so a human must review domain coverage, +uncertainty, failure patterns, and operational constraints before changing +production routing. + +- Seeded paired bootstrap intervals preserve task pairing. +- Pareto frontiers cover quality versus latency and quality versus reviewed + hypothetical cost. +- Unknown-cost policies are excluded from the cost frontier and named. +- The manifest rejects expected-answer leakage according to each task's scorer. +- Only locked tasks enter reported comparisons; exploratory tasks remain outside + the decision evidence. + +## Fail-closed contract + +A run aborts without artifacts when live provenance is absent, the credential is +missing, endpoint validation fails, discovery is incomplete, authentication is +rejected, a request/call/token budget is exhausted, cost evidence is invalid or +expired, a provider response exceeds 8 MiB, report validation fails, or output +would contain the provider secret. + +## Provenance and artifacts + +Each run writes: + +- `benchmark_report.json`; +- `benchmark_cells.csv`; and +- `benchmark_summary.md`. + +Provenance records the exact Git SHA and workflow run, catalog/manifest/pricing +hashes, benchmark parameters, capability failures and skips, equal-budget +configuration and observations, access-cost evidence and validity, evidence +sufficiency, uncertainty, and Pareto results. Dry-run artifacts are deterministic +so schema and evidence regressions are reviewable as diffs. + +## Workflow + +`.github/workflows/nim-benchmark.yml` uses separate dry and live jobs. The dry +job has no NVIDIA secret. Only the live benchmark step receives +`NVIDIA_NIM_API_KEY`. Both paths use immutable action revisions, hard request and +execution bounds, single-flight concurrency, and retained artifacts. The +workflow cannot merge, release, approve its own changes, or rewrite production +routing. + +The normal Tests workflow separately proves 100% production statement and +branch coverage, 100% public docstrings, wheel build/install/import behavior, +optional-import isolation, and absence of temporary repair/export mechanisms. + +## Method grounding + +HELM supports standardized multi-metric evaluation and explicit reporting of +coverage gaps. FrugalGPT, RouteLLM, and Hybrid LLM motivate measuring routing +cost-quality trade-offs. NIST AI 600-1 supports documented, risk-aware testing, +evaluation, verification, and validation. These sources shape the measurement +and governance design; they do not substitute for exact-head evidence from the +models, tasks, and policies actually under review. diff --git a/docs/papers/README.md b/docs/papers/README.md index 65a89d2a..e8cde9e7 100644 --- a/docs/papers/README.md +++ b/docs/papers/README.md @@ -35,6 +35,19 @@ redistribution; each is cited below with its arXiv identifier. the responsive path. Distributed under the arXiv non-exclusive license / CC BY as marked on arXiv. +## Evaluation methodology (NIM cost-quality benchmark) + +- **Holistic Evaluation of Language Models (HELM)** — Percy Liang, Rishi + Bommasani, Tony Lee, et al. arXiv:2211.09110, 2022 (TMLR 2023). + `helm-holistic-evaluation-2211.09110.pdf` + Grounds the **NIM benchmark harness** (`docs/nim_benchmark.md`): evaluate a + broad, explicitly enumerated model pool on multiple metrics at once (quality, + latency, cost) instead of a single leaderboard number; report incompleteness + honestly (skipped/unsupported/rate-limited cells stay machine-readable rather + than silently dropped); and standardize conditions across compared systems + (same tasks, scorers, caps, and budgets). Distributed under the arXiv + non-exclusive license / CC BY as marked on arXiv. + ## Batch execution / load balancing The external `pg-llm-batch` service carries its own grounding papers, including @@ -46,3 +59,24 @@ but not vendored here so this repository remains one deployable control plane. > Citations are provided for scholarly attribution. Redistribution here relies > on the arXiv non-exclusive distribution license each author granted; no > GPL/AGPL-licensed material is vendored anywhere in this repository. + +## APA 7th edition references + +Chen, L., Zaharia, M., & Zou, J. (2023). FrugalGPT: How to use large language +models while reducing cost and improving performance. *arXiv*. +https://doi.org/10.48550/arXiv.2305.05176 + +Ding, D., Mallick, A., Wang, C., Sim, R., Mukherjee, S., Rühle, V., Lakshmanan, +L. V. S., & Awadallah, A. H. (2024). Hybrid LLM: Cost-efficient and +quality-aware query routing. *arXiv*. +https://doi.org/10.48550/arXiv.2404.14618 + +Liang, P., Bommasani, R., Lee, T., Tsipras, D., Soylu, D., Yasunaga, M., Zhang, +Y., Narayanan, D., Wu, Y., Kumar, A., Newman, B., Yuan, B., Yan, B., Zhang, C., +Cosgrove, C., Manning, C. D., Ré, C., Acosta-Navas, D., Hudson, D. A., … Koreeda, +Y. (2023). Holistic evaluation of language models. *Transactions on Machine +Learning Research*. https://doi.org/10.48550/arXiv.2211.09110 + +Ong, I., Almahairi, A., Wu, V., Chiang, W.-L., Wu, T., Gonzalez, J. E., Kadous, +M. W., & Stoica, I. (2024). RouteLLM: Learning to route LLMs with preference +data. *arXiv*. https://doi.org/10.48550/arXiv.2406.18665 diff --git a/docs/papers/helm-holistic-evaluation-2211.09110.pdf b/docs/papers/helm-holistic-evaluation-2211.09110.pdf new file mode 100644 index 00000000..5a6770de Binary files /dev/null and b/docs/papers/helm-holistic-evaluation-2211.09110.pdf differ diff --git a/examples/nim_pricing_scenario.json b/examples/nim_pricing_scenario.json new file mode 100644 index 00000000..b9b1b4a5 --- /dev/null +++ b/examples/nim_pricing_scenario.json @@ -0,0 +1,10 @@ +{ + "scenario_version": "2026-08-04.1", + "scenario_status": "example_unreviewed", + "scenario_notes": "Schema-demonstration scenario for dry runs and tests. These are HYPOTHETICAL USD-per-million-token assumptions, not authoritative NVIDIA rates: the hosted NIM catalog is currently free to the caller (actual cost 0). A live paid-cost analysis requires a reviewed scenario ('scenario_status': 'reviewed') supplied by the operator; models absent from this table are honestly reported as 'unknown'. Rates below deliberately price only some dry-run models so the 'unknown' path stays exercised.", + "usd_per_million_tokens": { + "dryrun/chat-basic": {"input": 0.2, "output": 0.6}, + "dryrun/chat-vision": {"input": 0.35, "output": 1.1}, + "dryrun/chat-omni": {"input": 0.5, "output": 1.6} + } +} diff --git a/examples/nim_task_manifest.json b/examples/nim_task_manifest.json new file mode 100644 index 00000000..04755822 --- /dev/null +++ b/examples/nim_task_manifest.json @@ -0,0 +1,233 @@ +{ + "manifest_version": "2026-08-07.1", + "manifest_notes": "Immutable task ids; expected answers and strict aliases are scoring-side only and are never injected into model prompts (no test-set leakage). The locked split contains thirty original, objectively scored tasks so the scheduled paired comparison can reach the repository's declared non-smoke evidence floor. Legacy scorer fields preserve authoring compatibility; the supported benchmark derives versioned complete-answer semantics, including explicit aliases and task-specific case sensitivity. The exploratory split remains tuning-only and never enters headline comparisons.", + "tasks": [ + { + "task_id": "trick_arithmetic_bat_ball", + "split": "locked", + "prompt": "A bat and a ball cost $1.10 in total. The bat costs $1.00 more than the ball. How much does the ball cost, in dollars? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "0.05"} + }, + { + "task_id": "constant_speed_distance", + "split": "locked", + "prompt": "A car travels 60 kilometers in 40 minutes. At that same speed, how many kilometers does it travel in 100 minutes? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "150"} + }, + { + "task_id": "trick_arithmetic_lily_pads", + "split": "locked", + "prompt": "A patch of lily pads doubles in size every day. It covers a whole lake in 48 days. After how many days did it cover half the lake? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "47"} + }, + { + "task_id": "letter_counting_strawberry", + "split": "locked", + "prompt": "How many times does the letter r appear in the word strawberry? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "3"} + }, + { + "task_id": "unit_conversion_km_miles", + "split": "locked", + "prompt": "A road is exactly 160.9344 kilometers long. How long is it in miles? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "100"} + }, + { + "task_id": "logic_trap_month_days", + "split": "locked", + "prompt": "Some months have 30 days and some have 31. How many months of the year have 28 days? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "12"} + }, + { + "task_id": "capital_recall_france", + "split": "locked", + "prompt": "Name the capital city of France. Answer with the city name only.", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "Paris"} + }, + { + "task_id": "capital_recall_australia", + "split": "locked", + "prompt": "Name the capital city of Australia. Answer with the city name only.", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "Canberra"} + }, + { + "task_id": "sequence_next_fibonacci", + "split": "locked", + "prompt": "What number comes next in this sequence: 1, 1, 2, 3, 5, 8, 13? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "21"} + }, + { + "task_id": "digit_sum_reasoning", + "split": "locked", + "prompt": "Multiply 12 by 12, then add 56 to the result. Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "200"} + }, + { + "task_id": "linear_equation_solution", + "split": "locked", + "prompt": "Solve 3x + 5 = 26 for x. Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "7"} + }, + { + "task_id": "combination_pair_count", + "split": "locked", + "prompt": "Five students each shake hands with every other student exactly once. How many handshakes occur? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "10"} + }, + { + "task_id": "fair_coin_probability", + "split": "locked", + "prompt": "A fair coin is flipped twice. What is the probability that both flips are heads? Answer with a decimal number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "0.25"} + }, + { + "task_id": "leap_year_day_count", + "split": "locked", + "prompt": "How many days are in the year 2024? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "366"} + }, + { + "task_id": "temperature_conversion_celsius", + "split": "locked", + "prompt": "Convert 68 degrees Fahrenheit to degrees Celsius. Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "20"} + }, + { + "task_id": "next_prime_number", + "split": "locked", + "prompt": "What is the smallest prime number greater than 29? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "31"} + }, + { + "task_id": "arithmetic_mean_value", + "split": "locked", + "prompt": "What is the arithmetic mean of 2, 4, and 9? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "5"} + }, + { + "task_id": "percentage_discount_price", + "split": "locked", + "prompt": "An item costs $80 before a 25 percent discount. What is the discounted price in dollars? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "60"} + }, + { + "task_id": "rectangle_perimeter_value", + "split": "locked", + "prompt": "A rectangle has side lengths 7 centimeters and 4 centimeters. What is its perimeter in centimeters? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "22"} + }, + { + "task_id": "constant_rate_travel", + "split": "locked", + "prompt": "A train travels at 72 kilometers per hour for 2.5 hours. How many kilometers does it travel? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "180"} + }, + { + "task_id": "binary_decimal_conversion", + "split": "locked", + "prompt": "Convert the binary numeral 101101 to base ten. Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "45"} + }, + { + "task_id": "roman_numeral_conversion", + "split": "locked", + "prompt": "Convert the Roman numeral XLII to an Arabic numeral. Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "42"} + }, + { + "task_id": "elapsed_clock_time", + "split": "locked", + "prompt": "A meeting starts at 14:35 and lasts 95 minutes. At what time does it end? Answer in HH:MM format only.", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "16:10"} + }, + { + "task_id": "syllogism_validity_classification", + "split": "locked", + "prompt": "All poets are readers. Some readers are cyclists. Therefore some poets are cyclists. Classify the conclusion in one word.", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "invalid"} + }, + { + "task_id": "string_reversal_result", + "split": "locked", + "prompt": "Reverse the letters of the English word stressed. Answer with the resulting word only.", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "desserts"} + }, + { + "task_id": "korean_word_translation", + "split": "locked", + "prompt": "Translate the Korean fruit word 사과 into English. Answer with one word only.", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "apple"} + }, + { + "task_id": "chemical_symbol_gold", + "split": "locked", + "prompt": "What is the chemical symbol for gold? Answer with the symbol only.", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "Au", "strict_case_sensitive": true} + }, + { + "task_id": "largest_ocean_name", + "split": "locked", + "prompt": "Name the largest ocean on Earth. Answer with its name only.", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": { + "substring": "Pacific", + "strict_texts": ["Pacific", "Pacific Ocean"] + } + }, + { + "task_id": "capital_recall_canada", + "split": "locked", + "prompt": "Name the capital city of Canada. Answer with the city name only.", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "Ottawa"} + }, + { + "task_id": "square_root_integer", + "split": "locked", + "prompt": "What is the positive integer square root of 144? Answer with the number only.", + "scorer": {"name": "exact_number_match", "version": "1"}, + "expected": {"number": "12"} + }, + { + "task_id": "exploratory_summary_probe", + "split": "exploratory", + "prompt": "Summarize in one word the mood of a calm sunny morning by the sea.", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "peace"} + }, + { + "task_id": "exploratory_translation_probe", + "split": "exploratory", + "prompt": "Translate the English word hello into French. Answer with one word only.", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "bonjour"} + } + ] +} diff --git a/fuzz/corpus/nim_catalog/hostile_entries.json b/fuzz/corpus/nim_catalog/hostile_entries.json new file mode 100644 index 00000000..12e2f390 --- /dev/null +++ b/fuzz/corpus/nim_catalog/hostile_entries.json @@ -0,0 +1 @@ +{"data": [{"id": "dup/model"}, {"id": "dup/model"}, {"owned_by": "no-id"}, "not-an-object", {"id": " "}, {"id": 42}, {"id": "ok/model", "owned_by": 99}]} diff --git a/fuzz/corpus/nim_catalog/valid_catalog.json b/fuzz/corpus/nim_catalog/valid_catalog.json new file mode 100644 index 00000000..0bffa69d --- /dev/null +++ b/fuzz/corpus/nim_catalog/valid_catalog.json @@ -0,0 +1 @@ +{"object": "list", "data": [{"id": "meta/llama-3.1-8b-instruct", "owned_by": "meta"}, {"id": "nvidia/nv-embed-v1", "owned_by": "nvidia"}]} diff --git a/fuzz/fuzz_nim_catalog.py b/fuzz/fuzz_nim_catalog.py new file mode 100644 index 00000000..a7e09c32 --- /dev/null +++ b/fuzz/fuzz_nim_catalog.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Atheris coverage-guided harness: NIM benchmark model-catalog parser. + +Surface: ``nim_benchmark.parse_model_catalog_body`` -- the untrusted-input +parser for the provider's ``GET /v1/models`` response body. + +Run locally (needs a permissive-licensed build of Atheris, Apache-2.0):: + + python fuzz/fuzz_nim_catalog.py -atomic_step -max_total_time=60 fuzz/corpus/nim_catalog +""" + +import sys +from pathlib import Path + +import atheris + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +with atheris.instrument_imports(): + import contextual_orchestrator.nim_benchmark + from fuzz.targets import exercise_nim_catalog + + +def one_input(data: bytes) -> None: + """Feed one fuzzer-generated body to the catalog parser invariants.""" + exercise_nim_catalog(data) + + +def main() -> None: + """Set up and run the Atheris fuzzing loop.""" + atheris.Setup(sys.argv, one_input) + atheris.Fuzz() + + +if __name__ == "__main__": + main() diff --git a/fuzz/targets.py b/fuzz/targets.py index d0c34446..319eba5e 100644 --- a/fuzz/targets.py +++ b/fuzz/targets.py @@ -188,3 +188,42 @@ def exercise_orchestration(prompt: str, mode: str) -> None: continue assert frame.startswith("data: ") json.loads(frame[len("data: "):]) + + +def exercise_nim_catalog(raw: bytes) -> None: + """Drive the NIM benchmark model-catalog parser over arbitrary bytes. + + ``parse_model_catalog_body`` consumes an untrusted provider response + (``GET /v1/models``). Structural failures must surface only as + ``CatalogDiscoveryError``; successful parses are deduplicated, sorted + (immune to provider response-order drift), machine-readably annotated, and + stable under reparse. + """ + from contextual_orchestrator.nim_benchmark import ( + CatalogDiscoveryError, + parse_model_catalog_body, + ) + + try: + catalog = parse_model_catalog_body(raw) + except CatalogDiscoveryError: + return + + assert set(catalog) == {"models", "duplicate_model_ids", "invalid_entries"} + model_ids = [row["model_id"] for row in catalog["models"]] + assert model_ids == sorted(model_ids), "catalog must be order-drift immune" + assert len(model_ids) == len(set(model_ids)), "catalog must be deduplicated" + for row in catalog["models"]: + assert isinstance(row["model_id"], str) and row["model_id"].strip() + assert isinstance(row["owned_by"], str) + for entry in catalog["invalid_entries"]: + assert entry["invalid_reason"] in {"entry_not_an_object", "missing_model_id"} + assert catalog["duplicate_model_ids"] == sorted(catalog["duplicate_model_ids"]) + + # The whole result must be JSON-serialisable, and reparsing the surviving + # models must be a fixed point (parse . serialize . parse == parse). + reserialized = json.dumps( + {"data": [{"id": row["model_id"], "owned_by": row["owned_by"]} for row in catalog["models"]]} + ).encode("utf-8") + if catalog["models"]: + assert parse_model_catalog_body(reserialized)["models"] == catalog["models"] diff --git a/tests/fuzz/test_fuzz_properties.py b/tests/fuzz/test_fuzz_properties.py index 7e7b3f34..3b7b79d8 100644 --- a/tests/fuzz/test_fuzz_properties.py +++ b/tests/fuzz/test_fuzz_properties.py @@ -18,6 +18,7 @@ from fuzz.targets import ( exercise_agent_config, + exercise_nim_catalog, exercise_orchestration, exercise_redaction, exercise_request_body, @@ -108,3 +109,36 @@ def test_redaction_never_crashes_and_is_idempotent(text: str) -> None: ) def test_orchestration_on_arbitrary_prompt(prompt: str, mode: str) -> None: exercise_orchestration(prompt, mode) + + +@_SETTINGS +@given(st.binary(max_size=4096)) +def test_nim_catalog_never_crashes_on_raw_bytes(raw: bytes) -> None: + exercise_nim_catalog(raw) + + +# Catalog-shaped adversarial entries: wrong types, missing ids, duplicates. +_catalog_entry = ( + st.none() + | st.text(max_size=16) + | st.integers() + | st.fixed_dictionaries( + {}, + optional={ + "id": st.text(max_size=20) | st.integers() | st.none() | st.just("dup/model"), + "owned_by": st.text(max_size=12) | st.integers() | st.none(), + }, + ) +) + + +@_SETTINGS +@given(st.lists(_catalog_entry, max_size=8).map(lambda entries: json.dumps({"data": entries}).encode("utf-8"))) +def test_nim_catalog_on_structured_entries(raw: bytes) -> None: + exercise_nim_catalog(raw) + + +@_SETTINGS +@given(_json_values.map(lambda v: json.dumps(v).encode("utf-8"))) +def test_nim_catalog_on_arbitrary_json(raw: bytes) -> None: + exercise_nim_catalog(raw) diff --git a/tests/test_nim_artifact_publication.py b/tests/test_nim_artifact_publication.py new file mode 100644 index 00000000..705f869c --- /dev/null +++ b/tests/test_nim_artifact_publication.py @@ -0,0 +1,254 @@ +"""Regression tests for transactional NIM benchmark artifact publication.""" + +from __future__ import annotations + +import io +import json +import os +from pathlib import Path + +import pytest + +from contextual_orchestrator import nim_csv_evidence as csv_evidence + +_ARTIFACT_NAMES = ( + "benchmark_report.json", + "benchmark_cells.csv", + "benchmark_summary.md", +) + + +def _model_use() -> dict[str, str]: + """Return one valid deterministic model-assignment record.""" + return { + "step_id": "step_one", + "role": "worker", + "agent_id": "agent_one", + "model_id": "vendor/model-one", + } + + +def _write_complete_artifacts(output_directory: Path, *, valid_report: bool = True) -> None: + """Write one complete benchmark artifact set for wrapper-level tests.""" + output_directory.mkdir(parents=True, exist_ok=True) + report: object + if valid_report: + report = { + "evaluation": { + "evaluation_cells": [ + { + "policy_name": "route_once", + "task_id": "task_one", + "models_used": [_model_use()], + } + ] + } + } + else: + report = {"evaluation": {"evaluation_cells": "invalid"}} + (output_directory / "benchmark_report.json").write_text( + json.dumps(report), + encoding="utf-8", + ) + (output_directory / "benchmark_cells.csv").write_text( + "policy_name,task_id,task_score\r\nroute_once,task_one,1.0\r\n", + encoding="utf-8", + ) + (output_directory / "benchmark_summary.md").write_text( + "# staged benchmark summary\n", + encoding="utf-8", + ) + + +def _successful_benchmark_cli(argv: list[str]) -> int: + """Write valid artifacts to the output directory supplied by the wrapper.""" + output_directory = csv_evidence.output_directory_from_argv(argv) + _write_complete_artifacts(output_directory) + print( + json.dumps( + { + "run_mode": "dry_run", + "artifact_paths": { + "json_path": str(output_directory / "benchmark_report.json"), + "csv_path": str(output_directory / "benchmark_cells.csv"), + "markdown_path": str(output_directory / "benchmark_summary.md"), + }, + } + ) + ) + return 0 + + +def _publication_residue(parent: Path, final_name: str) -> list[Path]: + """Return staging or backup directories left beside a final artifact set.""" + return sorted( + [ + *parent.glob(f".{final_name}.staging-*"), + *parent.glob(f".{final_name}.backup-*"), + ] + ) + + +def test_success_publishes_one_complete_set_and_returns_final_paths( + tmp_path: Path, +) -> None: + """A successful run exposes only the enriched final artifact directory.""" + final_directory = tmp_path / "buyer_evidence" + stdout = io.StringIO() + + result = csv_evidence.run_benchmark_cli_with_complete_csv( + ["--dry-run", "--output-dir", str(final_directory)], + benchmark_cli=_successful_benchmark_cli, + stdout=stdout, + ) + + assert result == 0 + assert sorted(path.name for path in final_directory.iterdir()) == sorted( + _ARTIFACT_NAMES + ) + csv_text = (final_directory / "benchmark_cells.csv").read_text(encoding="utf-8") + assert "models_used_json" in csv_text + payload = json.loads(stdout.getvalue()) + assert payload["artifact_paths"] == { + "json_path": str(final_directory / "benchmark_report.json"), + "csv_path": str(final_directory / "benchmark_cells.csv"), + "markdown_path": str(final_directory / "benchmark_summary.md"), + } + assert _publication_residue(tmp_path, final_directory.name) == [] + + +def test_fresh_target_failure_leaves_no_visible_or_hidden_partial_set( + tmp_path: Path, +) -> None: + """CSV enrichment failure must not expose a new partial artifact set.""" + final_directory = tmp_path / "buyer_evidence" + + def invalid_success(argv: list[str]) -> int: + staged_directory = csv_evidence.output_directory_from_argv(argv) + _write_complete_artifacts(staged_directory, valid_report=False) + print(json.dumps({"run_mode": "dry_run", "artifact_paths": {}})) + return 0 + + stdout = io.StringIO() + result = csv_evidence.run_benchmark_cli_with_complete_csv( + ["--output-dir", str(final_directory)], + benchmark_cli=invalid_success, + stdout=stdout, + ) + + assert result == 1 + assert json.loads(stdout.getvalue())["benchmark_failed_closed"] is True + assert not final_directory.exists() + assert _publication_residue(tmp_path, final_directory.name) == [] + + +def test_failure_preserves_an_existing_complete_artifact_set(tmp_path: Path) -> None: + """Pre-publication failure must leave a prior complete set byte-identical.""" + final_directory = tmp_path / "buyer_evidence" + final_directory.mkdir() + original_bytes: dict[str, bytes] = {} + for artifact_name in _ARTIFACT_NAMES: + artifact_path = final_directory / artifact_name + artifact_path.write_bytes(f"prior:{artifact_name}\n".encode()) + original_bytes[artifact_name] = artifact_path.read_bytes() + + def invalid_success(argv: list[str]) -> int: + staged_directory = csv_evidence.output_directory_from_argv(argv) + _write_complete_artifacts(staged_directory, valid_report=False) + print(json.dumps({"run_mode": "dry_run", "artifact_paths": {}})) + return 0 + + result = csv_evidence.run_benchmark_cli_with_complete_csv( + ["--output-dir", str(final_directory)], + benchmark_cli=invalid_success, + stdout=io.StringIO(), + ) + + assert result == 1 + assert { + artifact_name: (final_directory / artifact_name).read_bytes() + for artifact_name in _ARTIFACT_NAMES + } == original_bytes + assert _publication_residue(tmp_path, final_directory.name) == [] + + +def test_mid_publication_failure_rolls_back_the_prior_complete_set( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Failure after backup creation restores the exact prior directory.""" + final_directory = tmp_path / "buyer_evidence" + final_directory.mkdir() + for artifact_name in _ARTIFACT_NAMES: + (final_directory / artifact_name).write_text( + f"prior:{artifact_name}\n", + encoding="utf-8", + ) + original = { + path.name: path.read_bytes() + for path in final_directory.iterdir() + } + real_replace = os.replace + failed_once = False + + def fail_staging_publish(source: str | os.PathLike[str], destination: str | os.PathLike[str]) -> None: + nonlocal failed_once + source_path = Path(source) + destination_path = Path(destination) + if ( + not failed_once + and source_path.name.startswith(f".{final_directory.name}.staging-") + and destination_path == final_directory + ): + failed_once = True + raise OSError("simulated directory publication failure") + real_replace(source, destination) + + monkeypatch.setattr(csv_evidence.os, "replace", fail_staging_publish) + stdout = io.StringIO() + result = csv_evidence.run_benchmark_cli_with_complete_csv( + ["--output-dir", str(final_directory)], + benchmark_cli=_successful_benchmark_cli, + stdout=stdout, + ) + + assert failed_once is True + assert result == 1 + assert json.loads(stdout.getvalue())["error_class"] == "OSError" + assert { + path.name: path.read_bytes() + for path in final_directory.iterdir() + } == original + assert _publication_residue(tmp_path, final_directory.name) == [] + + +def test_interrupted_backup_is_recovered_before_the_next_run(tmp_path: Path) -> None: + """A sole crash backup is restored when the final directory is absent.""" + final_directory = tmp_path / "buyer_evidence" + backup_directory = tmp_path / f".{final_directory.name}.backup-interrupted" + backup_directory.mkdir() + for artifact_name in _ARTIFACT_NAMES: + (backup_directory / artifact_name).write_text( + f"recovered:{artifact_name}\n", + encoding="utf-8", + ) + + def failed_benchmark(argv: list[str]) -> int: + print(json.dumps({"benchmark_failed_closed": True, "error_class": "TestError"})) + return 1 + + stdout = io.StringIO() + result = csv_evidence.run_benchmark_cli_with_complete_csv( + ["--output-dir", str(final_directory)], + benchmark_cli=failed_benchmark, + stdout=stdout, + ) + + assert result == 1 + assert sorted(path.name for path in final_directory.iterdir()) == sorted( + _ARTIFACT_NAMES + ) + assert (final_directory / "benchmark_report.json").read_text( + encoding="utf-8" + ).startswith("recovered:") + assert _publication_residue(tmp_path, final_directory.name) == [] diff --git a/tests/test_nim_artifact_publication_edges.py b/tests/test_nim_artifact_publication_edges.py new file mode 100644 index 00000000..88408a39 --- /dev/null +++ b/tests/test_nim_artifact_publication_edges.py @@ -0,0 +1,294 @@ +"""Edge coverage for transactional NIM benchmark artifact publication.""" + +from __future__ import annotations + +import json +import os +import shutil +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from contextual_orchestrator import nim_csv_evidence as csv_evidence + +_ARTIFACT_NAMES = ( + "benchmark_report.json", + "benchmark_cells.csv", + "benchmark_summary.md", +) + + +def _write_valid_published_directory(directory: Path, marker: str) -> None: + """Write three non-empty files with enriched CSV evidence.""" + directory.mkdir(parents=True, exist_ok=True) + (directory / "benchmark_report.json").write_text( + json.dumps({"marker": marker}), + encoding="utf-8", + ) + (directory / "benchmark_cells.csv").write_text( + "policy_name,task_id,models_used_json\r\nroute_once,task_one,[]\r\n", + encoding="utf-8", + ) + (directory / "benchmark_summary.md").write_text( + f"# {marker}\n", + encoding="utf-8", + ) + + +def test_output_target_rejects_dot_symbolic_link_and_regular_file( + tmp_path: Path, +) -> None: + """Publication accepts only a dedicated non-symlink directory path.""" + with pytest.raises(csv_evidence.CsvEvidenceError, match="dedicated"): + csv_evidence._normalized_output_directory(Path(".")) + + target = tmp_path / "target_directory" + target.mkdir() + linked = tmp_path / "linked_evidence" + linked.symlink_to(target, target_is_directory=True) + with pytest.raises(csv_evidence.CsvEvidenceError, match="symbolic link"): + csv_evidence._normalized_output_directory(linked) + + regular_file = tmp_path / "evidence_file" + regular_file.write_text("not a directory", encoding="utf-8") + with pytest.raises(csv_evidence.CsvEvidenceError, match="must be a directory"): + csv_evidence._normalized_output_directory(regular_file) + + +def test_remove_path_handles_file_directory_symbolic_link_and_missing( + tmp_path: Path, +) -> None: + """Private cleanup never follows links and treats absence as success.""" + regular_file = tmp_path / "temporary_file" + regular_file.write_text("temporary", encoding="utf-8") + csv_evidence._remove_path(regular_file) + assert not regular_file.exists() + + directory = tmp_path / "temporary_directory" + directory.mkdir() + (directory / "child").write_text("temporary", encoding="utf-8") + csv_evidence._remove_path(directory) + assert not directory.exists() + + target = tmp_path / "target_directory" + target.mkdir() + linked = tmp_path / "temporary_link" + linked.symlink_to(target, target_is_directory=True) + csv_evidence._remove_path(linked) + assert not linked.exists() + assert target.exists() + + csv_evidence._remove_path(tmp_path / "already_missing") + + +def test_recovery_cleans_staging_rejects_ambiguity_and_discards_stale_backup( + tmp_path: Path, +) -> None: + """Recovery is deterministic for staging and backup residue.""" + final_directory = tmp_path / "buyer_evidence" + staging = tmp_path / ".buyer_evidence.staging-abandoned" + staging.mkdir() + csv_evidence._recover_interrupted_publication(final_directory) + assert not staging.exists() + + first_backup = tmp_path / ".buyer_evidence.backup-one" + second_backup = tmp_path / ".buyer_evidence.backup-two" + first_backup.mkdir() + second_backup.mkdir() + with pytest.raises(csv_evidence.CsvEvidenceError, match="multiple"): + csv_evidence._recover_interrupted_publication(final_directory) + first_backup.rmdir() + second_backup.rmdir() + + _write_valid_published_directory(final_directory, "current") + stale_backup = tmp_path / ".buyer_evidence.backup-stale" + _write_valid_published_directory(stale_backup, "stale") + csv_evidence._recover_interrupted_publication(final_directory) + assert final_directory.exists() + assert not stale_backup.exists() + + +def test_staging_argv_rewrite_supports_all_forms_and_rejects_duplicates( + tmp_path: Path, +) -> None: + """Only one user output option is replaced by the private staging path.""" + staging = tmp_path / "private_staging" + assert csv_evidence._argv_with_output_directory(["--dry-run"], staging) == [ + "--dry-run", + "--output-dir", + str(staging), + ] + assert csv_evidence._argv_with_output_directory( + ["--dry-run", "--output-dir=public"], + staging, + ) == ["--dry-run", f"--output-dir={staging}"] + assert csv_evidence._argv_with_output_directory( + ["--output-dir", "public", "--dry-run"], + staging, + ) == ["--output-dir", str(staging), "--dry-run"] + + with pytest.raises(csv_evidence.CsvEvidenceError, match="only once"): + csv_evidence._argv_with_output_directory( + ["--output-dir", "one", "--output-dir=two"], + staging, + ) + with pytest.raises(csv_evidence.CsvEvidenceError, match="only once"): + csv_evidence._argv_with_output_directory( + ["--output-dir=one", "--output-dir", "two"], + staging, + ) + with pytest.raises(csv_evidence.CsvEvidenceError, match="requires a value"): + csv_evidence._argv_with_output_directory(["--output-dir"], staging) + + +def test_complete_directory_validation_rejects_shape_link_empty_and_plain_csv( + tmp_path: Path, +) -> None: + """Every staged set must contain exactly three regular enriched files.""" + wrong_shape = tmp_path / "wrong_shape" + wrong_shape.mkdir() + (wrong_shape / "unexpected").write_text("x", encoding="utf-8") + with pytest.raises(csv_evidence.CsvEvidenceError, match="exactly JSON"): + csv_evidence._validate_complete_artifact_directory(wrong_shape) + + linked_shape = tmp_path / "linked_shape" + linked_shape.mkdir() + (linked_shape / "benchmark_report.json").write_text("{}", encoding="utf-8") + (linked_shape / "benchmark_cells.csv").write_text( + "policy_name,task_id,models_used_json\n", + encoding="utf-8", + ) + link_target = tmp_path / "summary_target" + link_target.write_text("# target\n", encoding="utf-8") + (linked_shape / "benchmark_summary.md").symlink_to(link_target) + with pytest.raises(csv_evidence.CsvEvidenceError, match="regular files"): + csv_evidence._validate_complete_artifact_directory(linked_shape) + + empty_shape = tmp_path / "empty_shape" + _write_valid_published_directory(empty_shape, "valid") + (empty_shape / "benchmark_summary.md").write_text("", encoding="utf-8") + with pytest.raises(csv_evidence.CsvEvidenceError, match="must not be empty"): + csv_evidence._validate_complete_artifact_directory(empty_shape) + + plain_csv = tmp_path / "plain_csv" + _write_valid_published_directory(plain_csv, "valid") + (plain_csv / "benchmark_cells.csv").write_text( + "policy_name,task_id\nroute_once,task_one\n", + encoding="utf-8", + ) + with pytest.raises(csv_evidence.CsvEvidenceError, match="model-assignment"): + csv_evidence._validate_complete_artifact_directory(plain_csv) + + +def test_restore_helper_handles_fresh_partial_missing_backup_and_partial_publish( + tmp_path: Path, +) -> None: + """Rollback removes a fresh partial and restores a present prior backup.""" + fresh_partial = tmp_path / "fresh_partial" + _write_valid_published_directory(fresh_partial, "partial") + csv_evidence._restore_backup_after_failure(fresh_partial, None) + assert not fresh_partial.exists() + + final_directory = tmp_path / "buyer_evidence" + missing_backup = tmp_path / ".buyer_evidence.backup-missing" + csv_evidence._restore_backup_after_failure(final_directory, missing_backup) + assert not final_directory.exists() + + _write_valid_published_directory(final_directory, "partial") + backup_directory = tmp_path / ".buyer_evidence.backup-prior" + _write_valid_published_directory(backup_directory, "prior") + csv_evidence._restore_backup_after_failure(final_directory, backup_directory) + assert "prior" in (final_directory / "benchmark_report.json").read_text( + encoding="utf-8" + ) + assert not backup_directory.exists() + + +def test_publish_replaces_prior_set_and_removes_backup(tmp_path: Path) -> None: + """Successful replacement exposes the new set and deletes its hidden backup.""" + final_directory = tmp_path / "buyer_evidence" + staging_directory = tmp_path / ".buyer_evidence.staging-ready" + _write_valid_published_directory(final_directory, "prior") + _write_valid_published_directory(staging_directory, "new") + + csv_evidence._publish_staged_directory(staging_directory, final_directory) + + assert "new" in (final_directory / "benchmark_report.json").read_text( + encoding="utf-8" + ) + assert not staging_directory.exists() + assert list(tmp_path.glob(".buyer_evidence.backup-*")) == [] + + +def test_publish_rejects_generated_backup_collision( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A pre-existing generated backup name fails closed before mutation.""" + final_directory = tmp_path / "buyer_evidence" + staging_directory = tmp_path / ".buyer_evidence.staging-ready" + _write_valid_published_directory(final_directory, "prior") + _write_valid_published_directory(staging_directory, "new") + monkeypatch.setattr( + csv_evidence.uuid, + "uuid4", + lambda: SimpleNamespace(hex="fixed"), + ) + collision = tmp_path / ".buyer_evidence.backup-fixed" + collision.mkdir() + + with pytest.raises(csv_evidence.CsvEvidenceError, match="already exists"): + csv_evidence._publish_staged_directory(staging_directory, final_directory) + + assert final_directory.exists() + assert staging_directory.exists() + + +def test_publish_reports_irrecoverable_restoration_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Failure of both publication and rollback is surfaced as a domain error.""" + final_directory = tmp_path / "buyer_evidence" + staging_directory = tmp_path / ".buyer_evidence.staging-ready" + _write_valid_published_directory(final_directory, "prior") + _write_valid_published_directory(staging_directory, "new") + monkeypatch.setattr( + csv_evidence.uuid, + "uuid4", + lambda: SimpleNamespace(hex="fixed"), + ) + real_replace = os.replace + call_count = 0 + + def fail_publish_and_restore( + source: str | os.PathLike[str], + destination: str | os.PathLike[str], + ) -> None: + nonlocal call_count + call_count += 1 + if call_count == 1: + real_replace(source, destination) + return + raise OSError("simulated publish or restoration failure") + + monkeypatch.setattr(csv_evidence.os, "replace", fail_publish_and_restore) + with pytest.raises(csv_evidence.CsvEvidenceError, match="could not be restored"): + csv_evidence._publish_staged_directory(staging_directory, final_directory) + + backup_directory = tmp_path / ".buyer_evidence.backup-fixed" + assert backup_directory.exists() + shutil.rmtree(backup_directory) + shutil.rmtree(staging_directory) + + +def test_success_output_rewrite_rejects_malformed_payloads(tmp_path: Path) -> None: + """A published success result must be a JSON object with artifact paths.""" + final_directory = tmp_path / "buyer_evidence" + with pytest.raises(csv_evidence.CsvEvidenceError, match="valid JSON"): + csv_evidence._rewrite_success_output("not-json", final_directory) + with pytest.raises(csv_evidence.CsvEvidenceError, match="JSON object"): + csv_evidence._rewrite_success_output("[]", final_directory) + with pytest.raises(csv_evidence.CsvEvidenceError, match="artifact_paths"): + csv_evidence._rewrite_success_output("{}", final_directory) diff --git a/tests/test_nim_assignment_step_ids.py b/tests/test_nim_assignment_step_ids.py new file mode 100644 index 00000000..5b69fe9a --- /dev/null +++ b/tests/test_nim_assignment_step_ids.py @@ -0,0 +1,85 @@ +"""Contracts for deterministic benchmark model-assignment step identities.""" + +from __future__ import annotations + +import json + +import pytest + +from contextual_orchestrator import nim_csv_evidence as csv_evidence + + +def _assignment(step_id: object, model_id: str = "vendor/model-a") -> dict[str, object]: + """Build one assignment fixture with a configurable raw trace identifier.""" + return { + "step_id": step_id, + "role": "worker", + "agent_id": "agent_one", + "model_id": model_id, + } + + +def test_empty_trace_step_ids_receive_stable_positional_identifiers() -> None: + """Route traces without plan IDs must remain complete, unique CSV evidence.""" + serialized = csv_evidence._models_used_json( + [_assignment(""), _assignment(" ", "vendor/model-b")] + ) + + assert json.loads(serialized) == [ + { + "step_id": "trace_step_0001", + "role": "worker", + "agent_id": "agent_one", + "model_id": "vendor/model-a", + }, + { + "step_id": "trace_step_0002", + "role": "worker", + "agent_id": "agent_one", + "model_id": "vendor/model-b", + }, + ] + + +def test_empty_step_id_avoids_an_existing_positional_identifier() -> None: + """A canonical fallback must remain unique when a real trace used its base ID.""" + serialized = csv_evidence._models_used_json( + [ + _assignment("trace_step_0002"), + _assignment("", "vendor/model-b"), + ] + ) + + assert [row["step_id"] for row in json.loads(serialized)] == [ + "trace_step_0002", + "trace_step_0002_2", + ] + + +def test_existing_trace_step_id_is_preserved_exactly() -> None: + """A real non-empty trace identifier must not be rewritten by publication.""" + serialized = csv_evidence._models_used_json([_assignment("planner_step")]) + assert json.loads(serialized)[0]["step_id"] == "planner_step" + + +def test_runtime_integer_trace_step_ids_are_preserved_as_decimal_strings() -> None: + """Runtime workflow integers must preserve exact identity in CSV-safe strings.""" + serialized = csv_evidence._models_used_json( + [_assignment(0), _assignment(7, "vendor/model-b")] + ) + assert [row["step_id"] for row in json.loads(serialized)] == ["0", "7"] + + +def test_duplicate_non_empty_trace_step_ids_fail_closed() -> None: + """Two assignments may not claim the same explicit workflow-step identity.""" + with pytest.raises(csv_evidence.CsvEvidenceError, match="duplicate step_id"): + csv_evidence._models_used_json( + [_assignment("same_step"), _assignment("same_step", "vendor/model-b")] + ) + + +@pytest.mark.parametrize("step_id", [None, True, -1, 1.5]) +def test_missing_or_unsupported_trace_step_id_is_not_synthesized(step_id: object) -> None: + """Unsupported or invalid runtime identities must fail instead of being invented.""" + with pytest.raises(csv_evidence.CsvEvidenceError, match="step_id"): + csv_evidence._models_used_json([_assignment(step_id)]) diff --git a/tests/test_nim_benchmark.py b/tests/test_nim_benchmark.py new file mode 100644 index 00000000..1b3bb325 --- /dev/null +++ b/tests/test_nim_benchmark.py @@ -0,0 +1,1336 @@ +"""NIM benchmark harness contracts — discovery, all-modality probes, fair eval. + +Everything here runs fully offline: provider behavior is injected through the +transport seam, evaluation workers ride the mock:// path, and the credential +registry is a fresh in-memory KV per test. Adversarial coverage follows the +issue contract: malformed catalogs, duplicate ids, unsupported capabilities, +partial results, non-finite token/cost values, rate limits, timeouts, +response-order drift, and secret redaction. +""" + +from __future__ import annotations + +import contextlib +import inspect +import io +import json +import math +import os +import socket +import tempfile +import urllib.error +from pathlib import Path +import sys + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import nim_benchmark as nb # noqa: E402 +from contextual_orchestrator.credentials import ( # noqa: E402 + InMemoryCredentialBackend, + NotConfigured, + register_credential, + set_backend, +) +from contextual_orchestrator.orchestrator import ModelAgent, ModelClient # noqa: E402 + +REPO_ROOT = Path(__file__).resolve().parents[1] +TASK_MANIFEST_PATH = str(REPO_ROOT / "examples" / "nim_task_manifest.json") +PRICING_SCENARIO_PATH = str(REPO_ROOT / "examples" / "nim_pricing_scenario.json") +FAKE_ENDPOINT = "https://nim.example.test/v1" + + +@pytest.fixture(autouse=True) +def _fresh_backend(): + """Isolated in-memory KV and a clean benchmark env var for every test.""" + set_backend(InMemoryCredentialBackend()) + saved_env = os.environ.pop(nb.NIM_CREDENTIAL_NAME, None) + try: + yield + finally: + set_backend(None) + if saved_env is not None: + os.environ[nb.NIM_CREDENTIAL_NAME] = saved_env + + +def _ok_json(payload: object) -> tuple[int, bytes]: + return 200, json.dumps(payload).encode("utf-8") + + +def _fixed_transport(status: int, body: bytes): + def transport(method, url, headers, body_bytes): + return status, body + + return transport + + +def _mini_manifest(task_count: int = 2) -> dict: + tasks = [ + { + "task_id": f"locked_task_{index}", + "split": "locked", + "prompt": f"Question number {index}?", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "zebra"}, + } + for index in range(task_count) + ] + return {"manifest_version": "test.1", "tasks": tasks} + + +def _mock_agents(*model_ids: str) -> list[ModelAgent]: + taken: set[str] = set() + return [ + ModelAgent( + id=nb.sanitize_worker_agent_id(model_id, taken), + model=model_id, + base_url="mock://nim-test", + credential_key=nb.NIM_CREDENTIAL_NAME, + tags=("reasoning", "writing"), + ) + for model_id in model_ids + ] + + +# -------------------------------------------------------------------------- +# Egress guard + default transport +# -------------------------------------------------------------------------- + + +def test_endpoint_guard_rejects_http() -> None: + with pytest.raises(nb.BenchmarkContractError): + nb.require_public_https_endpoint("http://nim.example.test/v1") + + +def test_endpoint_guard_rejects_missing_host() -> None: + with pytest.raises(nb.BenchmarkContractError): + nb.require_public_https_endpoint("https:///v1") + + +def _patched_getaddrinfo(ip_address: str): + return lambda *args, **kwargs: [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (ip_address, 443))] + + +def test_endpoint_guard_rejects_private_address() -> None: + original = socket.getaddrinfo + socket.getaddrinfo = _patched_getaddrinfo("10.0.0.8") + try: + with pytest.raises(nb.BenchmarkContractError): + nb.require_public_https_endpoint(FAKE_ENDPOINT) + finally: + socket.getaddrinfo = original + + +def test_endpoint_guard_accepts_public_address() -> None: + original = socket.getaddrinfo + socket.getaddrinfo = _patched_getaddrinfo("93.184.216.34") + try: + nb.require_public_https_endpoint(FAKE_ENDPOINT) + finally: + socket.getaddrinfo = original + + +class _FakeDirectResponse: + """Minimal response returned by the pinned HTTPS connection test seam.""" + + def __init__(self, status: int, body: bytes) -> None: + self.status = status + self._body = body + self.closed = False + + def read(self, maximum_bytes: int = -1) -> bytes: + if maximum_bytes < 0: + return self._body + return self._body[:maximum_bytes] + + def close(self) -> None: + self.closed = True + + +class _FakeDirectConnection: + """Scripted pinned connection that records address and authority evidence.""" + + plans: list[object] = [] + instances: list["_FakeDirectConnection"] = [] + + def __init__(self, server_hostname, pinned_ip, port, timeout, context) -> None: + self.server_hostname = server_hostname + self.pinned_ip = pinned_ip + self.port = port + self.timeout = timeout + self.context = context + self.method = "" + self.target = "" + self.body = None + self.headers = {} + self.closed = False + self._plan = type(self).plans.pop(0) + type(self).instances.append(self) + + def request(self, method, target, body, headers) -> None: + self.method = method + self.target = target + self.body = body + self.headers = headers + if isinstance(self._plan, BaseException): + raise self._plan + + def getresponse(self): + return self._plan + + def close(self) -> None: + self.closed = True + + +def _install_direct_transport_fakes(monkeypatch, plans, addresses=("93.184.216.34",)): + """Install deterministic DNS and connection seams for one transport test.""" + resolution_calls = [] + + def resolve(host, port, label): + resolution_calls.append((host, port, label)) + return addresses + + _FakeDirectConnection.plans = list(plans) + _FakeDirectConnection.instances = [] + monkeypatch.setattr(nb, "_validated_public_addresses", resolve) + monkeypatch.setattr(nb, "_PinnedHTTPSConnection", _FakeDirectConnection) + return resolution_calls + + +def test_default_transport_returns_status_and_revalidates_each_request(monkeypatch) -> None: + first_response = _FakeDirectResponse(200, b"body-one") + second_response = _FakeDirectResponse(200, b"body-two") + resolution_calls = _install_direct_transport_fakes( + monkeypatch, + [first_response, second_response], + ) + + transport = nb.build_default_transport(timeout_seconds=5.0) + first = transport("GET", f"{FAKE_ENDPOINT}/models", {}, None) + second = transport("GET", f"{FAKE_ENDPOINT}/models;format=json?second=1", {}, None) + + assert first == (200, b"body-one") + assert second == (200, b"body-two") + assert resolution_calls == [ + ("nim.example.test", 443, "NIM benchmark"), + ("nim.example.test", 443, "NIM benchmark"), + ] + assert all(response.closed for response in (first_response, second_response)) + assert all(connection.closed for connection in _FakeDirectConnection.instances) + assert _FakeDirectConnection.instances[0].server_hostname == "nim.example.test" + assert _FakeDirectConnection.instances[0].pinned_ip == "93.184.216.34" + assert _FakeDirectConnection.instances[1].target == "/v1/models;format=json?second=1" + + +def test_default_transport_returns_http_error_status_with_body(monkeypatch) -> None: + response = _FakeDirectResponse(429, b"slow down") + _install_direct_transport_fakes(monkeypatch, [response]) + assert nb.build_default_transport(5.0)( + "POST", f"{FAKE_ENDPOINT}/chat/completions", {}, b"{}" + ) == (429, b"slow down") + assert response.closed is True + + +def test_default_transport_rejects_oversized_response_and_closes_resources( + monkeypatch, +) -> None: + """A provider cannot exhaust memory with an unbounded response body.""" + response = _FakeDirectResponse(200, b"x" * (nb.MAX_PROVIDER_RESPONSE_BYTES + 1)) + _install_direct_transport_fakes(monkeypatch, [response]) + + with pytest.raises(nb.BenchmarkContractError, match="response exceeds"): + nb.build_default_transport(5.0)( + "GET", f"{FAKE_ENDPOINT}/models", {}, None + ) + + assert response.closed is True + assert _FakeDirectConnection.instances[0].closed is True + + +def test_default_transport_rejects_redirect_without_following(monkeypatch) -> None: + response = _FakeDirectResponse(302, b"redirect") + _install_direct_transport_fakes(monkeypatch, [response]) + with pytest.raises(nb.BenchmarkContractError, match="redirects are not permitted"): + nb.build_default_transport(5.0)( + "POST", f"{FAKE_ENDPOINT}/chat/completions", {}, b"{}" + ) + assert response.closed is True + + +def test_default_transport_falls_back_only_to_another_validated_address(monkeypatch) -> None: + response = _FakeDirectResponse(200, b"catalog") + _install_direct_transport_fakes( + monkeypatch, + [OSError("first pin failed"), response], + addresses=("93.184.216.34", "93.184.216.35"), + ) + result = nb.build_default_transport(5.0)( + "GET", f"{FAKE_ENDPOINT}/models", {}, None + ) + assert result == (200, b"catalog") + assert [item.pinned_ip for item in _FakeDirectConnection.instances] == [ + "93.184.216.34", + "93.184.216.35", + ] + + +def test_default_transport_reports_failure_after_every_pin_fails(monkeypatch) -> None: + _install_direct_transport_fakes( + monkeypatch, + [OSError("first pin failed"), OSError("second pin failed")], + addresses=("93.184.216.34", "93.184.216.35"), + ) + with pytest.raises(urllib.error.URLError, match="second pin failed"): + nb.build_default_transport(5.0)( + "GET", f"{FAKE_ENDPOINT}/models", {}, None + ) + + +# -------------------------------------------------------------------------- +# Request budget +# -------------------------------------------------------------------------- + + +def test_request_budget_rejects_non_positive_cap() -> None: + with pytest.raises(nb.BenchmarkContractError): + nb.RequestBudget(0) + + +def test_request_budget_spends_then_exhausts() -> None: + budget = nb.RequestBudget(2) + assert budget.try_spend() and budget.try_spend() + assert not budget.try_spend() + assert budget.requests_spent == 2 + with pytest.raises(nb.BenchmarkBudgetError): + budget.spend_or_fail() + + +def test_budgeted_client_charges_each_chat_call() -> None: + budget = nb.RequestBudget(1) + client = nb._BudgetedModelClient(budget) + agent = _mock_agents("dryrun/chat-basic")[0] + assert client.chat(agent, [{"role": "user", "content": "hello there"}]) + with pytest.raises(nb.BenchmarkBudgetError): + client.chat(agent, [{"role": "user", "content": "over budget"}]) + + +# -------------------------------------------------------------------------- +# Catalog parsing (adversarial) +# -------------------------------------------------------------------------- + + +def test_catalog_parse_rejects_invalid_utf8() -> None: + with pytest.raises(nb.CatalogDiscoveryError): + nb.parse_model_catalog_body(b"\xff\xfe\xfa") + + +def test_catalog_parse_rejects_invalid_json() -> None: + with pytest.raises(nb.CatalogDiscoveryError): + nb.parse_model_catalog_body(b"{not json") + + +def test_catalog_parse_rejects_non_object_and_missing_data() -> None: + with pytest.raises(nb.CatalogDiscoveryError): + nb.parse_model_catalog_body(b"[1, 2, 3]") + with pytest.raises(nb.CatalogDiscoveryError): + nb.parse_model_catalog_body(json.dumps({"data": "nope"}).encode("utf-8")) + + +def test_catalog_parse_records_invalid_and_duplicate_entries() -> None: + body = json.dumps( + { + "data": [ + {"id": "vendor/model-b", "owned_by": "vendor"}, + "not-an-object", + {"owned_by": "vendor"}, + {"id": " ", "owned_by": "vendor"}, + {"id": 42}, + {"id": "vendor/model-a", "owned_by": 99}, + {"id": "vendor/model-b", "owned_by": "vendor"}, + ] + } + ).encode("utf-8") + catalog = nb.parse_model_catalog_body(body) + # Sorted output guards against provider response-order drift. + assert [row["model_id"] for row in catalog["models"]] == ["vendor/model-a", "vendor/model-b"] + assert catalog["models"][0]["owned_by"] == "" # non-string owner coerced + assert catalog["duplicate_model_ids"] == ["vendor/model-b"] + reasons = {entry["invalid_reason"] for entry in catalog["invalid_entries"]} + assert reasons == {"entry_not_an_object", "missing_model_id"} + assert len(catalog["invalid_entries"]) == 4 + + +def test_catalog_order_drift_never_reorders_models() -> None: + forward = json.dumps({"data": [{"id": "a/model-one"}, {"id": "b/model-two"}]}).encode("utf-8") + reverse = json.dumps({"data": [{"id": "b/model-two"}, {"id": "a/model-one"}]}).encode("utf-8") + assert nb.parse_model_catalog_body(forward)["models"] == nb.parse_model_catalog_body(reverse)["models"] + + +def test_discover_catalog_success_and_budget_charge() -> None: + budget = nb.RequestBudget(3) + catalog = nb.discover_model_catalog( + _fixed_transport(*_ok_json({"data": [{"id": "vendor/model-a"}]})), FAKE_ENDPOINT, "key", budget + ) + assert catalog["models"][0]["model_id"] == "vendor/model-a" + assert budget.requests_spent == 1 + + +def test_discover_catalog_fails_closed_on_auth_rejection() -> None: + for status in (401, 403): + with pytest.raises(nb.BenchmarkAuthError): + nb.discover_model_catalog(_fixed_transport(status, b"{}"), FAKE_ENDPOINT, "key", nb.RequestBudget(3)) + + +def test_discover_catalog_fails_closed_on_http_error() -> None: + with pytest.raises(nb.CatalogDiscoveryError): + nb.discover_model_catalog(_fixed_transport(500, b"{}"), FAKE_ENDPOINT, "key", nb.RequestBudget(3)) + + +def test_discover_catalog_fails_closed_on_network_error() -> None: + def transport(method, url, headers, body): + raise urllib.error.URLError("dns failure") + + with pytest.raises(nb.CatalogDiscoveryError): + nb.discover_model_catalog(transport, FAKE_ENDPOINT, "key", nb.RequestBudget(3)) + + +def test_discover_catalog_fails_closed_on_empty_inventory() -> None: + with pytest.raises(nb.CatalogDiscoveryError): + nb.discover_model_catalog(_fixed_transport(*_ok_json({"data": []})), FAKE_ENDPOINT, "key", nb.RequestBudget(3)) + + +# -------------------------------------------------------------------------- +# Capability probes — every NIM contract +# -------------------------------------------------------------------------- + + +def test_probe_registry_covers_every_nim_contract() -> None: + assert set(nb.CAPABILITY_PROBE_ORDER) == { + "chat_completion", + "text_completion", + "response_generation", + "text_embedding", + "image_understanding", + "video_understanding", + "audio_understanding", + "audio_transcription", + "audio_speech", + } + + +def test_probe_assets_are_deterministic_and_well_formed() -> None: + assert nb._tiny_wav_bytes() == nb._tiny_wav_bytes() + assert nb._tiny_wav_bytes().startswith(b"RIFF") + assert nb._image_data_uri().startswith("data:image/png;base64,") + assert nb._video_data_uri().startswith("data:video/mp4;base64,") + multipart = nb._multipart_transcription_body("vendor/asr-model") + assert b'name="model"' in multipart and b"vendor/asr-model" in multipart + assert b'filename="probe.wav"' in multipart and b"RIFF" in multipart + + +def test_response_validators_accept_and_reject_shapes() -> None: + assert nb._has_choice({"choices": [{"message": {"content": "x"}}]}) + assert not nb._has_choice({"choices": []}) and not nb._has_choice({}) + assert nb._has_embedding({"data": [{"embedding": [0.1]}]}) + assert not nb._has_embedding({"data": []}) + assert not nb._has_embedding({"data": ["oops"]}) + assert not nb._has_embedding({"data": [{"embedding": "oops"}]}) + assert not nb._has_embedding({}) + assert nb._has_response_output({"output_text": "x"}) + assert not nb._has_response_output({"unrelated": 1}) + assert nb._has_transcription_text({"text": "ok"}) + assert not nb._has_transcription_text({"text": 5}) + + +def test_probe_status_classification_table() -> None: + assert nb.classify_probe_status(200) == "supported" + for status in (400, 404, 405, 415, 422, 501): + assert nb.classify_probe_status(status) == "unsupported" + assert nb.classify_probe_status(401) == "auth_rejected" + assert nb.classify_probe_status(403) == "unavailable" + assert nb.classify_probe_status(408) == "timeout" + assert nb.classify_probe_status(429) == "rate_limited" + assert nb.classify_probe_status(500) == "unavailable" + assert nb.classify_probe_status(302) == "failed" + + +def _probe(transport, capability_name="chat_completion"): + return nb.execute_capability_probe(transport, FAKE_ENDPOINT, "key", "vendor/model-a", capability_name) + + +def test_probe_supported_chat() -> None: + row = _probe(_fixed_transport(*_ok_json({"choices": [{"message": {"content": "OK"}}]}))) + assert row["probe_outcome"] == "supported" + assert row["http_status"] == 200 + + +def test_probe_timeout_and_network_failures() -> None: + def timeout_transport(method, url, headers, body): + raise socket.timeout("slow") + + def broken_transport(method, url, headers, body): + raise ConnectionResetError("reset") + + assert _probe(timeout_transport)["probe_outcome"] == "timeout" + row = _probe(broken_transport) + assert row["probe_outcome"] == "failed" + assert row["outcome_reason"].startswith("network_error:") + + +def test_probe_auth_rejection_fails_closed() -> None: + with pytest.raises(nb.BenchmarkAuthError): + _probe(_fixed_transport(401, b"{}")) + + +def test_probe_unsupported_and_rate_limited() -> None: + assert _probe(_fixed_transport(404, b"{}"))["probe_outcome"] == "unsupported" + assert _probe(_fixed_transport(429, b"{}"))["probe_outcome"] == "rate_limited" + + +def test_probe_malformed_success_bodies() -> None: + assert _probe(_fixed_transport(200, b"not json"))["probe_outcome"] == "malformed_response" + assert _probe(_fixed_transport(200, b"[]"))["probe_outcome"] == "malformed_response" + assert _probe(_fixed_transport(*_ok_json({"choices": []})))["probe_outcome"] == "malformed_response" + + +def test_probe_binary_speech_contract() -> None: + supported = _probe(_fixed_transport(200, b"RIFFaudio"), "audio_speech") + assert supported["probe_outcome"] == "supported" + empty = _probe(_fixed_transport(200, b""), "audio_speech") + assert empty["probe_outcome"] == "malformed_response" + assert empty["outcome_reason"] == "http_200_with_empty_media_body" + + +def _rows(**outcome_by_capability: str) -> list[dict]: + return [ + {"capability_name": name, "probe_outcome": outcome} + for name, outcome in outcome_by_capability.items() + ] + + +def test_classification_covers_every_modality_class() -> None: + assert nb.classify_model_capabilities( + _rows(chat_completion="supported", image_understanding="supported", audio_understanding="supported") + )["model_classification"] == "omni_capable" + assert nb.classify_model_capabilities( + _rows(chat_completion="supported", image_understanding="supported") + )["model_classification"] == "vision_chat_capable" + assert nb.classify_model_capabilities( + _rows(chat_completion="supported", video_understanding="supported") + )["model_classification"] == "vision_chat_capable" + assert nb.classify_model_capabilities(_rows(chat_completion="supported"))["model_classification"] == "chat_capable" + assert nb.classify_model_capabilities(_rows(text_embedding="supported"))["model_classification"] == "embedding_only" + assert nb.classify_model_capabilities(_rows(text_completion="supported"))["model_classification"] == "completion_only" + assert nb.classify_model_capabilities( + _rows(response_generation="supported") + )["model_classification"] == "responses_only" + assert nb.classify_model_capabilities(_rows(audio_transcription="supported"))["model_classification"] == "audio_only" + assert nb.classify_model_capabilities(_rows(audio_speech="supported"))["model_classification"] == "audio_only" + assert nb.classify_model_capabilities(_rows(chat_completion="skipped"))["model_classification"] == "skipped" + assert nb.classify_model_capabilities( + _rows(chat_completion="rate_limited", text_embedding="unsupported") + )["model_classification"] == "rate_limited" + assert nb.classify_model_capabilities( + _rows(chat_completion="unavailable", text_embedding="unsupported") + )["model_classification"] == "unavailable" + assert nb.classify_model_capabilities( + _rows(chat_completion="timeout", text_embedding="unsupported") + )["model_classification"] == "failed" + assert nb.classify_model_capabilities( + _rows(chat_completion="unsupported", text_embedding="unsupported") + )["model_classification"] == "unsupported_for_contract" + + +def test_classification_reports_chat_eligibility() -> None: + assert nb.classify_model_capabilities(_rows(chat_completion="supported"))["chat_eligible"] + assert not nb.classify_model_capabilities(_rows(text_embedding="supported"))["chat_eligible"] + + +def test_probe_models_rejects_bad_concurrency() -> None: + with pytest.raises(nb.BenchmarkContractError): + nb.probe_discovered_models([], _fixed_transport(200, b"{}"), FAKE_ENDPOINT, "key", nb.RequestBudget(1), 0, lambda: 0.0) + + +def test_probe_models_sorted_despite_input_order_drift() -> None: + # Models arrive in reverse order; the snapshot must still come out sorted. + models = [{"model_id": "b/model-two", "owned_by": ""}, {"model_id": "a/model-one", "owned_by": ""}] + results = nb.probe_discovered_models( + models, + _fixed_transport(*_ok_json({"choices": [{"message": {"content": "OK"}}]})), + FAKE_ENDPOINT, + "key", + nb.RequestBudget(100), + 2, + lambda: 1234.0, + lambda: 0.0, + ) + assert [row["model_id"] for row in results] == ["a/model-one", "b/model-two"] + # The fixed transport answers every probe, so the model reads as omni. + assert results[0]["model_classification"] == "omni_capable" + assert results[0]["discovered_at_unix"] == 1234.0 + assert results[0]["endpoint"] == FAKE_ENDPOINT + + +def test_probe_models_rejects_incomplete_probe_budget_before_egress() -> None: + """A capability phase never emits biased partial-inventory evidence.""" + models = [ + {"model_id": "a/model-one", "owned_by": ""}, + {"model_id": "b/model-two", "owned_by": ""}, + ] + budget = nb.RequestBudget(5) + calls: list[str] = [] + + def transport( + _method: str, + _url: str, + _headers: dict[str, str], + _body: bytes | None, + ) -> tuple[int, bytes]: + calls.append("called") + return _ok_json({"choices": [{"message": {"content": "OK"}}]}) + + with pytest.raises(nb.BenchmarkBudgetError, match="capability probe plan needs 18"): + nb.probe_discovered_models( + models, + transport, + FAKE_ENDPOINT, + "key", + budget, + 1, + lambda: 1234.0, + lambda: 0.0, + ) + + assert calls == [] + assert budget.requests_spent == 0 + + +# -------------------------------------------------------------------------- +# Scorers, manifest, pricing +# -------------------------------------------------------------------------- + + +def test_scorers_match_and_miss() -> None: + assert nb.score_exact_number_match({"number": "21"}, "the answer is 21.") == 1.0 + assert nb.score_exact_number_match({"number": "21"}, "the answer is 210") == 0.0 + assert nb.score_exact_number_match({"number": "21"}, "the answer is 21.5") == 0.0 + assert nb.score_exact_number_match({"number": "21"}, "the answer is 121") == 0.0 + assert nb.score_exact_number_match({"number": "0.05"}, "It costs $0.05 total") == 1.0 + assert nb.score_substring_match({"substring": "Paris"}, "It is PARIS indeed") == 1.0 + assert nb.score_substring_match({"substring": "Paris"}, "It is Lyon") == 0.0 + + +def _write_json(tmp_path: str, name: str, payload: object) -> str: + path = os.path.join(tmp_path, name) + with open(path, "w", encoding="utf-8") as handle: + json.dump(payload, handle) + return path + + +def test_example_task_manifest_is_valid_and_split() -> None: + manifest = nb.load_task_manifest(TASK_MANIFEST_PATH) + locked = nb.locked_evaluation_tasks(manifest) + assert len(locked) == 30 + assert len(manifest["tasks"]) - len(locked) == 2 # exploratory tuning split stays out + + +def test_manifest_rejects_each_contract_violation() -> None: + with tempfile.TemporaryDirectory() as tmp: + bad_json = os.path.join(tmp, "bad.json") + with open(bad_json, "w", encoding="utf-8") as handle: + handle.write("{broken") + cases = [ + (bad_json, None), + (_write_json(tmp, "list.json", [1]), None), + (_write_json(tmp, "nover.json", {"tasks": []}), None), + (_write_json(tmp, "notasks.json", {"manifest_version": "1", "tasks": []}), None), + (_write_json(tmp, "taskstr.json", {"manifest_version": "1", "tasks": ["x"]}), None), + ] + for path, _ in cases: + with pytest.raises(nb.BenchmarkContractError): + nb.load_task_manifest(path) + + def manifest_with(**overrides: object) -> dict: + task = { + "task_id": "valid_task_one", + "split": "locked", + "prompt": "What color is the clear daytime sky?", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "blue"}, + } + task.update(overrides) + return {"manifest_version": "1", "tasks": [task]} + + violations = [ + manifest_with(task_id="Bad-Id"), + manifest_with(task_id="single"), + manifest_with(split="training"), + manifest_with(prompt=" "), + manifest_with(prompt=42), + manifest_with(scorer="substring_match"), + manifest_with(scorer={"name": "unknown_scorer", "version": "9"}), + manifest_with(expected={}), + manifest_with(expected="blue"), + # Leakage: the scorer would award the prompt itself a point. + manifest_with(prompt="Answer blue if the sky is blue."), + ] + for index, payload in enumerate(violations): + path = _write_json(tmp, f"violation_{index}.json", payload) + with pytest.raises(nb.BenchmarkContractError): + nb.load_task_manifest(path) + + duplicate = manifest_with() + duplicate["tasks"] = [duplicate["tasks"][0], dict(duplicate["tasks"][0])] + path = _write_json(tmp, "duplicate.json", duplicate) + with pytest.raises(nb.BenchmarkContractError): + nb.load_task_manifest(path) + + +def test_example_pricing_scenario_is_valid_and_none_passthrough() -> None: + scenario = nb.load_pricing_scenario(PRICING_SCENARIO_PATH) + assert scenario["scenario_status"] == "example_unreviewed" + assert nb.load_pricing_scenario(None) is None + + +def test_pricing_scenario_rejects_each_contract_violation() -> None: + with tempfile.TemporaryDirectory() as tmp: + bad_json = os.path.join(tmp, "bad.json") + with open(bad_json, "w", encoding="utf-8") as handle: + handle.write("{broken") + base = { + "scenario_version": "1", + "scenario_status": "reviewed", + "usd_per_million_tokens": {"vendor/model-a": {"input": 1.0, "output": 2.0}}, + } + violations = [ + dict(base, scenario_version=3), + dict(base, scenario_status="draft"), + dict(base, usd_per_million_tokens=[1]), + dict(base, usd_per_million_tokens={"vendor/model-a": "cheap"}), + dict(base, usd_per_million_tokens={"vendor/model-a": {"input": True, "output": 2.0}}), + dict(base, usd_per_million_tokens={"vendor/model-a": {"input": 1.0, "output": "two"}}), + dict(base, usd_per_million_tokens={"vendor/model-a": {"input": float("nan"), "output": 2.0}}), + dict(base, usd_per_million_tokens={"vendor/model-a": {"input": float("inf"), "output": 2.0}}), + dict(base, usd_per_million_tokens={"vendor/model-a": {"input": -0.1, "output": 2.0}}), + dict(base, usd_per_million_tokens={"vendor/model-a": {"output": 2.0}}), + ] + with pytest.raises(nb.BenchmarkContractError): + nb.load_pricing_scenario(bad_json) + for index, payload in enumerate(violations): + path = _write_json(tmp, f"pricing_{index}.json", payload) + with pytest.raises(nb.BenchmarkContractError): + nb.load_pricing_scenario(path) + + +def test_hypothetical_cost_paths() -> None: + scenario = { + "scenario_version": "1", + "scenario_status": "reviewed", + "usd_per_million_tokens": {"vendor/model-a": {"input": 1.0, "output": 2.0}}, + } + usage = {"vendor/model-a": {"prompt_tokens": 1_000_000, "completion_tokens": 500_000}} + assert nb.hypothetical_cost_usd(scenario, usage) == 2.0 + assert nb.hypothetical_cost_usd(None, usage) == "unknown" + unpriced = {"vendor/other-model": {"prompt_tokens": 10, "completion_tokens": 10}} + assert nb.hypothetical_cost_usd(scenario, unpriced) == "unknown" + + +# -------------------------------------------------------------------------- +# Worker pool + usage accounting +# -------------------------------------------------------------------------- + + +def test_sanitize_worker_agent_id_paths() -> None: + taken: set[str] = set() + assert nb.sanitize_worker_agent_id("meta/llama-3.1-8b", taken) == "meta_llama_3_1_8b" + assert nb.sanitize_worker_agent_id("meta/llama-3.1-8b", taken) == "meta_llama_3_1_8b_2" + assert nb.sanitize_worker_agent_id("meta/llama-3.1-8b", taken) == "meta_llama_3_1_8b_3" + assert nb.sanitize_worker_agent_id("gpt", taken) == "nim_gpt" + assert nb.sanitize_worker_agent_id("///", taken) == "unnamed_model" + + +def _probed(model_id: str, chat_eligible: bool = True) -> dict: + return { + "model_id": model_id, + "owned_by": "vendor", + "chat_eligible": chat_eligible, + "model_classification": "chat_capable" if chat_eligible else "embedding_only", + } + + +def test_build_worker_agents_filters_caps_and_validates() -> None: + with pytest.raises(nb.BenchmarkContractError): + nb.build_worker_agents([], "mock://x", 0) + probed = [_probed("a/chat-one"), _probed("b/embed-only", chat_eligible=False), _probed("c/chat-two"), _probed("d/chat-three")] + agents = nb.build_worker_agents(probed, "mock://x", 2) + assert [agent.model for agent in agents] == ["a/chat-one", "c/chat-two"] + assert all(agent.credential_key == nb.NIM_CREDENTIAL_NAME for agent in agents) + + +def test_token_count_coercion_guards_non_finite_values() -> None: + assert nb._coerce_token_count(7) == 7 + assert nb._coerce_token_count(7.9) == 7 + assert nb._coerce_token_count(True) is None + assert nb._coerce_token_count("7") is None + assert nb._coerce_token_count(float("nan")) is None + assert nb._coerce_token_count(float("inf")) is None + assert nb._coerce_token_count(-1) is None + assert nb._coerce_token_count(None) is None + + +def test_cell_usage_reported_vs_estimated_and_failover() -> None: + agents_by_id = {"worker_one": "vendor/model-a", "worker_two": "vendor/model-b"} + reported_trace = [ + {"id": 0, "role": "worker", "agent_id": "worker_one", "output": "x", + "usage": {"prompt_tokens": 10, "completion_tokens": 5}}, + {"id": 1, "role": "verifier", "agent_id": "worker_one", "served_agent_id": "worker_two", "output": "y", + "usage": {"prompt_tokens": 3, "completion_tokens": 2}}, + ] + usage_by_model, summary = nb._cell_usage(reported_trace, agents_by_id, "prompt text") + assert summary["token_usage_source"] == "reported" + assert usage_by_model["vendor/model-a"] == {"prompt_tokens": 10, "completion_tokens": 5} + assert usage_by_model["vendor/model-b"] == {"prompt_tokens": 3, "completion_tokens": 2} + assert summary["total_tokens"] == 20 + assert summary["models_used"][1]["agent_id"] == "worker_two" + + adversarial_trace = [ + {"id": 0, "role": "worker", "agent_id": "worker_one", "output": "answer text", + "usage": {"prompt_tokens": float("nan"), "completion_tokens": float("inf")}}, + {"id": 1, "role": "worker", "agent_id": "worker_one", "output": None, "usage": "corrupted"}, + ] + _usage, summary = nb._cell_usage(adversarial_trace, agents_by_id, "prompt text") + assert summary["token_usage_source"] == "estimated" + assert summary["total_tokens"] > 0 + + +def test_run_error_classification() -> None: + assert nb._classify_run_error(TimeoutError("slow")) == "timeout" + wrapped = RuntimeError("provider failed") + wrapped.__cause__ = socket.timeout("slow") + assert nb._classify_run_error(wrapped) == "timeout" + assert nb._classify_run_error(ValueError("bad")) == "failure" + + +def _task(task_id: str = "sample_task", expected: str = "zebra") -> dict: + return { + "task_id": task_id, + "split": "locked", + "prompt": "Where do stripes live?", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": expected}, + } + + +def test_run_policy_cell_success_failure_timeout_and_fail_closed() -> None: + agents_by_id = {"worker_one": "vendor/model-a"} + ok = nb.run_policy_cell( + "route_once", + _task(), + lambda: {"answer": "a zebra appears", "trace": [{"id": 0, "role": "worker", "agent_id": "worker_one", "output": "a zebra appears"}]}, + agents_by_id, + None, + nb._deterministic_timer(), + ) + assert ok["run_outcome"] == "success" and ok["task_score"] == 1.0 + assert ok["hypothetical_cost_usd"] == "unknown" and ok["actual_cost_usd"] == 0.0 + assert ok["response_sha256"] and ok["call_count"] == 1 + + def fail() -> dict: + raise RuntimeError("boom") + + failed = nb.run_policy_cell("route_once", _task(), fail, agents_by_id, None, nb._deterministic_timer()) + assert failed["run_outcome"] == "failure" and failed["task_score"] is None + + def slow() -> dict: + raise TimeoutError("deadline") + + timed_out = nb.run_policy_cell("route_once", _task(), slow, agents_by_id, None, nb._deterministic_timer()) + assert timed_out["run_outcome"] == "timeout" + + def out_of_budget() -> dict: + raise nb.BenchmarkBudgetError("exhausted") + + with pytest.raises(nb.BenchmarkBudgetError): + nb.run_policy_cell("route_once", _task(), out_of_budget, agents_by_id, None, nb._deterministic_timer()) + + +def test_cheapest_priced_agent_selection() -> None: + agents = _mock_agents("vendor/model-a", "vendor/model-b", "vendor/model-c") + scenario = { + "scenario_version": "1", + "scenario_status": "reviewed", + "usd_per_million_tokens": { + "vendor/model-b": {"input": 0.1, "output": 0.2}, + "vendor/model-c": {"input": 0.1, "output": 0.2}, + }, + } + assert nb.cheapest_priced_agent(agents, None) is None + assert nb.cheapest_priced_agent(agents, {"scenario_version": "1", "scenario_status": "reviewed", "usd_per_million_tokens": {}}) is None + # Deterministic tiebreak: equal combined rate resolves by model id. + assert nb.cheapest_priced_agent(agents, scenario).model == "vendor/model-b" + + +def test_planned_evaluation_requests_formula() -> None: + assert nb.planned_evaluation_requests(3, 10) == 10 * (3 + 1 + nb.MAX_WORKFLOW_DEPTH + 1) + + +def test_evaluate_policies_contract_failures() -> None: + client = ModelClient() + with pytest.raises(nb.BenchmarkContractError): + nb.evaluate_policies([], _mini_manifest(), None, client, nb.RequestBudget(100)) + agents = _mock_agents("vendor/model-a") + exploratory_only = {"manifest_version": "1", "tasks": [dict(_task(), split="exploratory")]} + with pytest.raises(nb.BenchmarkContractError): + nb.evaluate_policies(agents, exploratory_only, None, client, nb.RequestBudget(100)) + with pytest.raises(nb.BenchmarkBudgetError): + nb.evaluate_policies(agents, _mini_manifest(), None, client, nb.RequestBudget(2)) + + +def test_evaluate_policies_all_arms_with_pricing() -> None: + agents = _mock_agents("dryrun/chat-basic", "dryrun/chat-vision") + scenario = nb.load_pricing_scenario(PRICING_SCENARIO_PATH) + budget = nb.RequestBudget(200) + evaluation = nb.evaluate_policies( + agents, _mini_manifest(3), scenario, nb._BudgetedModelClient(budget), budget, nb._deterministic_timer() + ) + cells = evaluation["evaluation_cells"] + policies = {cell["policy_name"] for cell in cells} + assert policies == { + "direct_single_worker:dryrun/chat-basic", + "direct_single_worker:dryrun/chat-vision", + "route_once", + "conduct_bounded", + "cheapest_eligible_worker", + } + assert evaluation["cheapest_worker_skip_reason"] is None + conduct_cells = [cell for cell in cells if cell["policy_name"] == "conduct_bounded"] + assert all(cell["workflow_depth"] <= nb.MAX_WORKFLOW_DEPTH for cell in conduct_cells) + assert all(cell["configured_total_token_budget"] == 256 for cell in conduct_cells) + assert all(cell["configured_maximum_calls"] == nb.MAX_WORKFLOW_DEPTH for cell in conduct_cells) + assert all(cell["observed_budget_calls"] <= nb.MAX_WORKFLOW_DEPTH for cell in conduct_cells) + assert cells == sorted(cells, key=lambda cell: (cell["policy_name"], cell["task_id"])) + assert budget.requests_spent > 0 + + +def test_evaluate_policies_skip_reasons_without_pricing() -> None: + agents = _mock_agents("vendor/model-a") + budget = nb.RequestBudget(200) + evaluation = nb.evaluate_policies(agents, _mini_manifest(), None, ModelClient(), budget) + assert evaluation["cheapest_worker_skip_reason"] == "no_pricing_scenario_supplied" + unpriced_scenario = { + "scenario_version": "1", + "scenario_status": "reviewed", + "usd_per_million_tokens": {"vendor/other": {"input": 1.0, "output": 1.0}}, + } + evaluation = nb.evaluate_policies(agents, _mini_manifest(), unpriced_scenario, ModelClient(), nb.RequestBudget(200)) + assert evaluation["cheapest_worker_skip_reason"] == "no_worker_priced_by_scenario" + + +# -------------------------------------------------------------------------- +# Statistics +# -------------------------------------------------------------------------- + + +def test_paired_bootstrap_requires_pairs_and_is_deterministic() -> None: + with pytest.raises(nb.BenchmarkContractError): + nb.paired_bootstrap_mean_difference([]) + first = nb.paired_bootstrap_mean_difference([(1.0, 0.0), (0.5, 0.5), (1.0, 0.5)], seed=11) + second = nb.paired_bootstrap_mean_difference([(1.0, 0.0), (0.5, 0.5), (1.0, 0.5)], seed=11) + assert first == second + assert first["ci_low"] <= first["mean_difference"] <= first["ci_high"] + assert first["pair_count"] == 3 + + +def test_pareto_frontier_excludes_dominated_rows() -> None: + rows = [ + {"name": "good_cheap", "quality": 0.9, "cost": 1.0}, + {"name": "good_pricey", "quality": 0.9, "cost": 2.0}, + {"name": "bad_cheap", "quality": 0.1, "cost": 0.5}, + {"name": "bad_pricey", "quality": 0.1, "cost": 5.0}, + ] + frontier = nb.pareto_frontier(rows, "quality", "cost") + assert [row["name"] for row in frontier] == ["good_cheap", "bad_cheap"] + + +def _synthetic_cell(policy: str, task_id: str, score, outcome: str = "success", cost=0.5) -> dict: + return { + "policy_name": policy, + "task_id": task_id, + "task_split": "locked", + "scorer_name": "substring_match", + "scorer_version": "1", + "task_score": score, + "run_outcome": outcome, + "outcome_reason": "completed", + "end_to_end_latency_ms": 10.0, + "provider_latency_ms": None, + "call_count": 1, + "workflow_depth": 1, + "prompt_tokens": 4, + "completion_tokens": 4, + "total_tokens": 8, + "token_usage_source": "estimated", + "actual_cost_usd": 0.0, + "hypothetical_cost_usd": cost, + "models_used": [], + "response_sha256": "hash", + } + + +def test_summaries_label_unknown_costs_and_all_failure_policies() -> None: + cells = [ + _synthetic_cell("route_once", "task_one", 1.0, cost=0.5), + _synthetic_cell("route_once", "task_two", 0.0, cost="unknown"), + _synthetic_cell("broken_policy", "task_one", None, outcome="failure", cost="unknown"), + ] + summaries = {row["policy_name"]: row for row in nb.summarize_policies(cells)} + assert summaries["route_once"]["mean_task_score"] == 0.5 + assert summaries["route_once"]["mean_hypothetical_cost_usd"] == 0.5 + assert summaries["route_once"]["unknown_hypothetical_cost_cells"] == 1 + assert summaries["broken_policy"]["mean_task_score"] == 0.0 + assert summaries["broken_policy"]["mean_hypothetical_cost_usd"] == "unknown" + assert summaries["broken_policy"]["success_count"] == 0 + + +def test_best_single_worker_hindsight_selection() -> None: + assert nb.best_single_worker_hindsight([{"policy_name": "route_once", "mean_task_score": 1.0}]) is None + summaries = nb.summarize_policies( + [ + _synthetic_cell("direct_single_worker:vendor/model-a", "task_one", 0.0), + _synthetic_cell("direct_single_worker:vendor/model-b", "task_one", 1.0), + ] + ) + best = nb.best_single_worker_hindsight(summaries) + assert best["model_id"] == "vendor/model-b" + assert best["selection_basis"] == "hindsight_argmax_mean_locked_score" + + +def test_paired_policy_comparisons_skip_missing_and_disjoint() -> None: + disjoint = [ + _synthetic_cell("conduct_bounded", "task_one", 1.0), + _synthetic_cell("route_once", "task_two", 0.0), + ] + assert nb.paired_policy_comparisons(disjoint, seed=3) == [] + cells = [ + _synthetic_cell("conduct_bounded", "task_one", 1.0), + _synthetic_cell("route_once", "task_one", 0.0), + _synthetic_cell("direct_single_worker:vendor/model-a", "task_one", 1.0), + # Failed cells carry no score and must stay out of the pairing. + _synthetic_cell("route_once", "task_three", None, outcome="failure"), + ] + comparisons = nb.paired_policy_comparisons(cells, seed=3) + pairs = {(row["policy_a"], row["policy_b"]) for row in comparisons} + assert ("conduct_bounded", "route_once") in pairs + assert ("route_once", "direct_single_worker:vendor/model-a") in pairs + + +def test_pareto_frontiers_exclude_unknown_cost_policies() -> None: + summaries = nb.summarize_policies( + [ + _synthetic_cell("route_once", "task_one", 1.0, cost=0.5), + _synthetic_cell("conduct_bounded", "task_one", 1.0, cost="unknown"), + ] + ) + frontiers = nb.build_pareto_frontiers(summaries) + assert [row["policy_name"] for row in frontiers["quality_vs_hypothetical_cost"]] == ["route_once"] + assert frontiers["excluded_unknown_cost_policies"] == ["conduct_bounded"] + assert len(frontiers["quality_vs_latency"]) >= 1 + + +# -------------------------------------------------------------------------- +# Provenance, schema, artifacts, secrets +# -------------------------------------------------------------------------- + + +def test_hash_helpers_are_stable() -> None: + assert nb.sha256_of_json({"b": 1, "a": 2}) == nb.sha256_of_json({"a": 2, "b": 1}) + assert len(nb.sha256_of_file(TASK_MANIFEST_PATH)) == 64 + + +def test_provenance_fails_closed_for_live_without_identity() -> None: + with pytest.raises(nb.BenchmarkContractError): + nb.build_provenance("live", "", "", {}, TASK_MANIFEST_PATH, None, {}) + live = nb.build_provenance("live", "abc123", "run-9", {}, TASK_MANIFEST_PATH, PRICING_SCENARIO_PATH, {"seed": 7}) + assert live["pricing_scenario_sha256"] is not None + dry = nb.build_provenance("dry_run", "", "", {}, TASK_MANIFEST_PATH, None, {}) + assert dry["git_sha"] == nb.DRY_RUN_PROVENANCE_PLACEHOLDER + assert dry["pricing_scenario_sha256"] is None + + +def test_report_schema_validation_reports_missing_paths() -> None: + with pytest.raises(nb.BenchmarkContractError) as excinfo: + nb.validate_report_schema({"provenance": "not-a-dict"}) + assert "provenance.run_mode" in str(excinfo.value) + + +def _dry_report(output_dir: str) -> dict: + return nb.run_benchmark( + "dry_run", + TASK_MANIFEST_PATH, + PRICING_SCENARIO_PATH, + output_dir, + max_total_requests=600, + ) + + +def test_artifact_writer_refuses_secret_leak() -> None: + register_credential(nb.NIM_CREDENTIAL_NAME, "nvapi-super-secret-value") + with tempfile.TemporaryDirectory() as tmp: + report = _dry_report(os.path.join(tmp, "clean")) + # The honest artifacts never contain the credential... + serialized = json.dumps(report) + assert "nvapi-super-secret-value" not in serialized + # ...and a poisoned report is refused outright. + report["catalog_snapshot"]["probed_models"][0]["owned_by"] = "nvapi-super-secret-value" + with pytest.raises(nb.SecretLeakError): + nb.write_benchmark_artifacts(report, os.path.join(tmp, "leaky")) + + +def test_secret_guard_passes_when_no_secret_registered() -> None: + nb._ensure_secret_absent("no secret registered anywhere") + + +# -------------------------------------------------------------------------- +# Dry-run provider + full pipeline +# -------------------------------------------------------------------------- + + +def test_dry_run_transport_serves_all_paths() -> None: + transport = nb.build_dry_run_transport() + status, body = transport("GET", f"{FAKE_ENDPOINT}/models", {}, None) + assert status == 200 and b"dryrun/chat-omni" in body + status, _ = transport("POST", f"{FAKE_ENDPOINT}/chat/completions", {}, b'{"model": "dryrun/unknown-model"}') + assert status == 404 + status, _ = transport("POST", f"{FAKE_ENDPOINT}/chat/completions", {}, b"no model marker at all") + assert status == 404 + status, _ = transport("POST", f"{FAKE_ENDPOINT}/chat/completions", {}, b'{"model": "dryrun/throttled-model"}') + assert status == 429 + status, _ = transport("POST", f"{FAKE_ENDPOINT}/chat/completions", {}, b'{"model": "dryrun/outage-model"}') + assert status == 503 + status, _ = transport("POST", f"{FAKE_ENDPOINT}/chat/completions", {}, b'{"model": "dryrun/legacy-unsupported"}') + assert status == 404 + status, _ = transport("POST", f"{FAKE_ENDPOINT}/embeddings", {}, b'{"model": "dryrun/chat-basic"}') + assert status == 400 + status, body = transport("POST", f"{FAKE_ENDPOINT}/embeddings", {}, b'{"model": "dryrun/embed-basic"}') + assert status == 200 and b"embedding" in body + status, body = transport("POST", f"{FAKE_ENDPOINT}/responses", {}, b'{"model": "dryrun/responses-native"}') + assert status == 200 and b"output_text" in body + multipart = nb._multipart_transcription_body("dryrun/audio-transcribe") + status, body = transport("POST", f"{FAKE_ENDPOINT}/audio/transcriptions", {}, multipart) + assert status == 200 and b"text" in body + status, body = transport("POST", f"{FAKE_ENDPOINT}/audio/speech", {}, b'{"model": "dryrun/audio-speech"}') + assert status == 200 and body.startswith(b"RIFF") + with pytest.raises(nb.CatalogDiscoveryError): + transport("POST", f"{FAKE_ENDPOINT}/never/heard-of-it", {}, b'{"model": "dryrun/chat-basic"}') + + +def test_dry_run_success_bodies_per_endpoint() -> None: + assert b"embedding" in nb._dry_run_success_body("/v1/embeddings") + assert b"output_text" in nb._dry_run_success_body("/v1/responses") + assert b"text" in nb._dry_run_success_body("/v1/audio/transcriptions") + assert nb._dry_run_success_body("/v1/audio/speech").startswith(b"RIFF") + assert b"choices" in nb._dry_run_success_body("/v1/chat/completions") + + +def test_deterministic_timer_advances_monotonically() -> None: + timer = nb._deterministic_timer() + assert timer() < timer() < timer() + + +def test_run_benchmark_rejects_unknown_mode() -> None: + with pytest.raises(nb.BenchmarkContractError): + nb.run_benchmark("test", TASK_MANIFEST_PATH, None, "unused") + + +def test_dry_run_pipeline_covers_every_modality_and_is_deterministic() -> None: + with tempfile.TemporaryDirectory() as tmp: + first = _dry_report(os.path.join(tmp, "one")) + second = _dry_report(os.path.join(tmp, "two")) + assert first["capability_summary"] == { + "audio_only": 2, + "chat_capable": 1, + "completion_only": 1, + "embedding_only": 1, + "omni_capable": 1, + "rate_limited": 1, + "responses_only": 1, + "unavailable": 1, + "unsupported_for_contract": 1, + "vision_chat_capable": 2, + } + by_model = {row["model_id"]: row for row in first["catalog_snapshot"]["probed_models"]} + assert by_model["dryrun/chat-omni"]["model_classification"] == "omni_capable" + assert set(by_model["dryrun/chat-omni"]["supported_capabilities"]) >= { + "chat_completion", + "image_understanding", + "video_understanding", + "audio_understanding", + } + assert by_model["dryrun/audio-transcribe"]["supported_capabilities"] == ["audio_transcription"] + assert by_model["dryrun/audio-speech"]["supported_capabilities"] == ["audio_speech"] + assert by_model["dryrun/embed-basic"]["model_classification"] == "embedding_only" + assert by_model["dryrun/chat-video"]["model_classification"] == "vision_chat_capable" + # Catalog hygiene lists survive into the snapshot. + assert first["catalog_snapshot"]["duplicate_model_ids"] == ["dryrun/chat-basic"] + assert first["catalog_snapshot"]["invalid_entries"][0]["invalid_reason"] == "missing_model_id" + # The evaluation compares every required system. + assert first["evaluation"]["best_single_worker_hindsight"] is not None + assert first["evaluation"]["pareto_frontiers"]["quality_vs_latency"] + assert first["evaluation"]["paired_comparisons"] + # Deterministic artifacts: identical reports across runs. + with open(os.path.join(tmp, "one", "benchmark_report.json"), "rb") as handle: + first_bytes = handle.read() + with open(os.path.join(tmp, "two", "benchmark_report.json"), "rb") as handle: + second_bytes = handle.read() + assert first_bytes == second_bytes + assert first["provenance"]["catalog_snapshot_sha256"] == second["provenance"]["catalog_snapshot_sha256"] + for artifact in ("benchmark_report.json", "benchmark_cells.csv", "benchmark_summary.md"): + assert os.path.exists(os.path.join(tmp, "one", artifact)) + + +def test_dry_run_accepts_explicit_transport() -> None: + with tempfile.TemporaryDirectory() as tmp: + report = nb.run_benchmark( + "dry_run", + TASK_MANIFEST_PATH, + None, + tmp, + max_total_requests=600, + transport=nb.build_dry_run_transport(), + ) + assert report["provenance"]["pricing_scenario_sha256"] is None + assert report["evaluation"]["cheapest_worker_skip_reason"] == "no_pricing_scenario_supplied" + + +def test_live_run_fails_closed_without_credential() -> None: + with tempfile.TemporaryDirectory() as tmp: + with pytest.raises(NotConfigured): + nb.run_benchmark("live", TASK_MANIFEST_PATH, None, tmp, git_sha="abc", workflow_run_id="run-1") + + +def test_live_run_end_to_end_offline() -> None: + register_credential(nb.NIM_CREDENTIAL_NAME, "nvapi-test-credential") + original_validate = ModelClient._validate_provider + original_send = ModelClient._send + ModelClient._validate_provider = lambda self, agent: None + ModelClient._send = lambda self, agent, payload: "stub live answer" + try: + with tempfile.TemporaryDirectory() as tmp: + report = nb.run_benchmark( + "live", + TASK_MANIFEST_PATH, + None, + tmp, + max_total_requests=600, + git_sha="abc123", + workflow_run_id="run-42", + transport=nb.build_dry_run_transport(), + ) + finally: + ModelClient._validate_provider = original_validate + ModelClient._send = original_send + assert report["provenance"]["run_mode"] == "live" + assert report["provenance"]["git_sha"] == "abc123" + assert report["honesty_labels"]["actual_cost_basis"] == ( + "reviewed_nvidia_developer_program_hosted_endpoint_access" + ) + assert report["request_budget"]["requests_spent"] <= 400 + assert "nvapi-test-credential" not in json.dumps(report) + + +def test_live_run_uses_default_transport_builder_when_none_given() -> None: + register_credential(nb.NIM_CREDENTIAL_NAME, "nvapi-test-credential") + original_builder = nb.build_default_transport + nb.build_default_transport = lambda timeout_seconds: nb.build_dry_run_transport() + original_validate = ModelClient._validate_provider + original_send = ModelClient._send + ModelClient._validate_provider = lambda self, agent: None + ModelClient._send = lambda self, agent, payload: "stub live answer" + try: + with tempfile.TemporaryDirectory() as tmp: + report = nb.run_benchmark( + "live", TASK_MANIFEST_PATH, None, tmp, + max_total_requests=600, git_sha="abc123", workflow_run_id="run-43", + ) + finally: + nb.build_default_transport = original_builder + ModelClient._validate_provider = original_validate + ModelClient._send = original_send + assert report["provenance"]["workflow_run_id"] == "run-43" + + +# -------------------------------------------------------------------------- +# CLI + bootstrap +# -------------------------------------------------------------------------- + + +def test_bootstrap_live_credential_paths() -> None: + from contextual_orchestrator.credentials import get_credential + + nb._bootstrap_live_credential() # neither KV nor env: stays unset + assert get_credential(nb.NIM_CREDENTIAL_NAME) is None + os.environ[nb.NIM_CREDENTIAL_NAME] = "nvapi-from-env" + try: + nb._bootstrap_live_credential() # env seeds the KV (bootstrap transport) + assert get_credential(nb.NIM_CREDENTIAL_NAME) == "nvapi-from-env" + os.environ[nb.NIM_CREDENTIAL_NAME] = "nvapi-different" + nb._bootstrap_live_credential() # existing KV value wins; no re-seed + assert get_credential(nb.NIM_CREDENTIAL_NAME) == "nvapi-from-env" + finally: + os.environ.pop(nb.NIM_CREDENTIAL_NAME, None) + + +def test_cli_dry_run_succeeds() -> None: + with tempfile.TemporaryDirectory() as tmp: + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + exit_code = nb.run_benchmark_cli( + [ + "--dry-run", + "--task-manifest", TASK_MANIFEST_PATH, + "--pricing-scenario", PRICING_SCENARIO_PATH, + "--output-dir", tmp, + "--max-total-requests", "600", + ] + ) + assert exit_code == 0 + printed = json.loads(stdout.getvalue()) + assert printed["run_mode"] == "dry_run" + assert printed["capability_summary"]["omni_capable"] == 1 + + +def test_cli_fails_closed_on_missing_manifest() -> None: + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + exit_code = nb.run_benchmark_cli(["--dry-run", "--task-manifest", "does/not/exist.json"]) + assert exit_code == 1 + assert json.loads(stdout.getvalue())["benchmark_failed_closed"] is True + + +def test_cli_live_fails_closed_without_secret() -> None: + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + exit_code = nb.run_benchmark_cli( + ["--task-manifest", TASK_MANIFEST_PATH, "--git-sha", "abc", "--workflow-run-id", "run-1"] + ) + assert exit_code == 1 + assert json.loads(stdout.getvalue())["error_class"] == "NotConfigured" + + +if __name__ == "__main__": + for name, fn in sorted(globals().items()): + if name.startswith("test_") and callable(fn): + if inspect.signature(fn).parameters: + print(f"skip {name} (requires pytest fixtures)") + continue + set_backend(InMemoryCredentialBackend()) + saved = os.environ.pop(nb.NIM_CREDENTIAL_NAME, None) + try: + fn() + finally: + set_backend(None) + if saved is not None: + os.environ[nb.NIM_CREDENTIAL_NAME] = saved + print(f"ok {name}") + print("ok") diff --git a/tests/test_nim_benchmark_budget_view.py b/tests/test_nim_benchmark_budget_view.py new file mode 100644 index 00000000..ba939a85 --- /dev/null +++ b/tests/test_nim_benchmark_budget_view.py @@ -0,0 +1,19 @@ +"""Buyer-facing request-plan regressions for the NVIDIA NIM benchmark.""" + +from contextual_orchestrator import nim_benchmark as nb + + +def test_planned_complete_run_requests_translates_the_internal_plan() -> None: + """Expose complete catalog, probe, evaluation, and total request counts.""" + assert nb.planned_complete_run_requests( + model_count=127, + locked_task_count=10, + max_eval_models=7, + ) == { + "catalog_discovery_requests": 1, + "capability_probe_requests": 127 * len(nb.CAPABILITY_PROBE_ORDER), + "evaluation_worker_ceiling": 7, + "evaluation_requests": 140, + "requests_after_catalog": 1283, + "total_requests": 1284, + } diff --git a/tests/test_nim_benchmark_release_acceptance.py b/tests/test_nim_benchmark_release_acceptance.py new file mode 100644 index 00000000..65d0e23c --- /dev/null +++ b/tests/test_nim_benchmark_release_acceptance.py @@ -0,0 +1,620 @@ +"""Release-level security, fairness, and evidence contracts for the NIM benchmark.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import threading +import urllib.parse + +import pytest + +from contextual_orchestrator import nim_benchmark as nb +from contextual_orchestrator.credentials import ( + InMemoryCredentialBackend, + register_credential, + set_backend, +) + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +TASK_MANIFEST_PATH = str(REPOSITORY_ROOT / "examples" / "nim_task_manifest.json") +EXAMPLE_PRICING_PATH = REPOSITORY_ROOT / "examples" / "nim_pricing_scenario.json" +FAKE_ENDPOINT = "https://nim.example.test/v1" + + +@pytest.fixture(autouse=True) +def _isolated_credentials() -> None: + """Give every test a fresh KV backend and remove it after the assertion.""" + set_backend(InMemoryCredentialBackend()) + try: + yield + finally: + set_backend(None) + + +def _write_json(path: Path, payload: object) -> str: + """Write one deterministic JSON fixture and return its string path.""" + path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8") + return str(path) + + +def _reviewed_pricing_scenario(**overrides: object) -> dict[str, object]: + """Return a complete reviewed hypothetical-price evidence fixture.""" + scenario: dict[str, object] = { + "scenario_version": "test-reviewed.1", + "scenario_status": "reviewed", + "source_url": "https://pricing.example.test/reviewed-rate-card", + "reviewed_by": "independent_pricing_reviewer", + "reviewed_at_date": "2026-08-05", + "valid_until_date": "2026-09-04", + "rate_basis": "hypothetical_usd_per_million_prompt_and_completion_tokens", + "uncertainty": "Scenario rates are explicit assumptions, not NVIDIA model prices.", + "usd_per_million_tokens": { + "vendor/model-one": {"input": 1.0, "output": 2.0} + }, + } + scenario.update(overrides) + return scenario + + +def _unexpected_transport(*_args: object, **_kwargs: object) -> tuple[int, bytes]: + """Fail a test when validation did not stop before provider egress.""" + raise AssertionError("provider transport must not run before evidence validation") + + +def test_package_import_does_not_eagerly_load_optional_benchmark() -> None: + """Normal gateway imports must not load or mutate the optional evaluator.""" + command = [ + sys.executable, + "-c", + ( + "import sys; import contextual_orchestrator; " + "assert 'contextual_orchestrator.nim_benchmark' not in sys.modules; " + "assert 'contextual_orchestrator.nim_benchmark_hardening' not in sys.modules" + ), + ] + completed = subprocess.run(command, check=False, capture_output=True, text=True) + assert completed.returncode == 0, completed.stderr + + +def test_live_run_rejects_unreviewed_pricing_before_egress(tmp_path: Path) -> None: + """Schema-demo prices can support dry runs but can never drive a live policy.""" + register_credential(nb.NIM_CREDENTIAL_NAME, "secret-test-key") + scenario = json.loads(EXAMPLE_PRICING_PATH.read_text(encoding="utf-8")) + scenario_path = _write_json(tmp_path / "unreviewed_pricing.json", scenario) + + with pytest.raises(nb.BenchmarkContractError, match="reviewed"): + nb.run_benchmark( + "live", + TASK_MANIFEST_PATH, + scenario_path, + str(tmp_path / "artifacts"), + endpoint=FAKE_ENDPOINT, + git_sha="a" * 40, + workflow_run_id="123", + transport=_unexpected_transport, + ) + + +def test_live_run_rejects_incomplete_or_expired_pricing_before_egress( + tmp_path: Path, +) -> None: + """Live hypothetical prices need complete, current, independently reviewed evidence.""" + register_credential(nb.NIM_CREDENTIAL_NAME, "secret-test-key") + incomplete = _reviewed_pricing_scenario() + del incomplete["reviewed_by"] + incomplete_path = _write_json(tmp_path / "incomplete_pricing.json", incomplete) + with pytest.raises(nb.BenchmarkContractError, match="reviewed_by"): + nb.run_benchmark( + "live", + TASK_MANIFEST_PATH, + incomplete_path, + str(tmp_path / "incomplete_artifacts"), + endpoint=FAKE_ENDPOINT, + git_sha="b" * 40, + workflow_run_id="124", + transport=_unexpected_transport, + ) + + expired_path = _write_json( + tmp_path / "expired_pricing.json", + _reviewed_pricing_scenario( + reviewed_at_date="1999-01-01", valid_until_date="2000-01-01" + ), + ) + with pytest.raises(nb.BenchmarkContractError, match="expired"): + nb.run_benchmark( + "live", + TASK_MANIFEST_PATH, + expired_path, + str(tmp_path / "expired_artifacts"), + endpoint=FAKE_ENDPOINT, + git_sha="c" * 40, + workflow_run_id="125", + transport=_unexpected_transport, + ) + + +def test_probe_concurrency_executes_the_complete_cartesian_plan() -> None: + """Thread scheduling cannot turn a complete probe plan into a biased prefix.""" + models = [ + {"model_id": "a/model-one", "owned_by": "vendor"}, + {"model_id": "b/model-two", "owned_by": "vendor"}, + ] + first_model_started = threading.Event() + second_model_four_calls = threading.Event() + second_model_call_count = 0 + count_lock = threading.Lock() + + def transport( + _method: str, + _url: str, + _headers: dict[str, str], + body: bytes | None, + ) -> tuple[int, bytes]: + """Force completion-order drift while every preflighted cell still runs.""" + nonlocal second_model_call_count + payload = (body or b"").decode("utf-8", errors="ignore") + if "a/model-one" in payload: + first_model_started.set() + second_model_four_calls.wait(timeout=0.25) + elif "b/model-two" in payload: + first_model_started.wait(timeout=0.25) + with count_lock: + second_model_call_count += 1 + if second_model_call_count == 4: + second_model_four_calls.set() + return 400, b"{}" + + budget = nb.RequestBudget( + len(models) * len(nb.CAPABILITY_PROBE_ORDER) + ) + rows = nb.probe_discovered_models( + models, + transport, + FAKE_ENDPOINT, + "credential-redacted", + budget, + probe_concurrency=2, + clock=lambda: 1.0, + timer=lambda: 0.0, + ) + + assert [row["model_id"] for row in rows] == ["a/model-one", "b/model-two"] + for model_row in rows: + assert [ + probe_row["capability_name"] + for probe_row in model_row["capability_probe_rows"] + ] == list(nb.CAPABILITY_PROBE_ORDER) + assert all( + probe_row["probe_outcome"] != "skipped" + for probe_row in model_row["capability_probe_rows"] + ) + assert budget.requests_spent == 18 + + +def test_complete_request_plan_rejects_invalid_counts() -> None: + """Planning inputs are positive integers, never booleans or empty counts.""" + invalid_cases = [ + {"discovered_model_count": 0, "max_eval_models": 7, "locked_task_count": 10}, + {"discovered_model_count": True, "max_eval_models": 7, "locked_task_count": 10}, + {"discovered_model_count": 1, "max_eval_models": 0, "locked_task_count": 10}, + {"discovered_model_count": 1, "max_eval_models": 7, "locked_task_count": 0}, + ] + + for case in invalid_cases: + with pytest.raises(nb.BenchmarkContractError, match="positive integer"): + nb.plan_complete_request_budget(**case) + + +def test_complete_request_plan_covers_a_127_model_catalog() -> None: + """The reviewed current-catalog scale fits only when probes and eval are reserved.""" + plan = nb.plan_complete_request_budget( + discovered_model_count=127, + max_eval_models=7, + locked_task_count=10, + ) + + assert plan == { + "catalog_request_count": 1, + "capability_probe_request_count": 127 * 9, + "evaluation_reserve_request_count": 140, + "planned_worker_count": 7, + "total_required_request_count": 1284, + } + + +def test_one_request_short_fails_after_catalog_before_any_probe(tmp_path: Path) -> None: + """An undersized live-style plan spends discovery only, then fails closed.""" + model_rows = [ + {"id": f"vendor/model-{index:03d}", "owned_by": "vendor"} + for index in range(127) + ] + calls: list[tuple[str, str]] = [] + + def transport( + method: str, + url: str, + _headers: dict[str, str], + _body: bytes | None, + ) -> tuple[int, bytes]: + calls.append((method, urllib.parse.urlparse(url).path)) + if method == "GET": + return 200, json.dumps({"data": model_rows}).encode("utf-8") + raise AssertionError("capability egress must not begin after failed preflight") + + with pytest.raises( + nb.BenchmarkBudgetError, + match="complete benchmark needs 1564 requests but configured cap is 1563", + ): + nb.run_benchmark( + "dry_run", + TASK_MANIFEST_PATH, + None, + str(tmp_path / "insufficient"), + endpoint=FAKE_ENDPOINT, + max_total_requests=1563, + max_eval_models=7, + transport=transport, + ) + + assert calls == [("GET", "/v1/models")] + + +def test_exact_complete_request_boundary_runs_and_records_plan(tmp_path: Path) -> None: + """The exact conservative boundary succeeds and records configured reserves.""" + manifest_path = _write_json( + tmp_path / "boundary_manifest.json", + { + "manifest_version": "boundary.1", + "tasks": [ + { + "task_id": "locked_boundary_task", + "split": "locked", + "prompt": "Name a striped animal.", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": "zebra"}, + } + ], + }, + ) + + def transport( + method: str, + url: str, + _headers: dict[str, str], + _body: bytes | None, + ) -> tuple[int, bytes]: + path = urllib.parse.urlparse(url).path + if method == "GET": + return 200, json.dumps( + {"data": [{"id": "vendor/model-one", "owned_by": "vendor"}]} + ).encode("utf-8") + return 200, nb._dry_run_success_body(path) + + report = nb.run_benchmark( + "dry_run", + manifest_path, + None, + str(tmp_path / "exact_boundary"), + endpoint=FAKE_ENDPOINT, + max_total_requests=18, + max_eval_models=1, + transport=transport, + ) + + assert report["request_budget"]["max_total_requests"] == 18 + assert report["request_budget"]["planned_total_requests"] == 18 + assert report["request_budget"]["catalog_requests"] == 1 + assert report["request_budget"]["capability_probe_requests"] == 9 + assert report["request_budget"]["evaluation_reserve_requests"] == 8 + assert report["request_budget"]["requests_spent"] <= 18 + + +def test_video_probe_fixture_is_one_decodable_frame_with_stable_hash() -> None: + """A video-capable model receives a real one-frame MP4, not an ftyp-only stub.""" + fixture = nb._tiny_mp4_bytes() + metadata = nb.validate_video_probe_fixture(fixture) + + assert metadata == { + "codec_name": "h264", + "width": 16, + "height": 16, + "frame_count": 1, + } + assert hashlib.sha256(fixture).hexdigest() == nb.VIDEO_PROBE_FIXTURE_SHA256 + assert len(fixture) > 1000 + + +def test_smoke_manifest_cannot_authorize_production_routing(tmp_path: Path) -> None: + """Ten smoke tasks produce diagnostics, not a buyer-facing routing decision.""" + report = nb.run_benchmark( + "dry_run", + TASK_MANIFEST_PATH, + None, + str(tmp_path), + max_total_requests=500, + max_eval_models=2, + ) + evaluation = report["evaluation"] + + assert evaluation["evidence_status"] == "insufficient_evidence" + assert evaluation["decision_use"] == "benchmark_smoke_only" + assert evaluation["minimum_paired_task_count"] == 30 + assert evaluation["required_completion_fraction"] == 0.9 + assert evaluation["routing_recommendation"] is None + assert report["honesty_labels"]["actual_cost_basis"] == ( + "deterministic_dry_run_no_provider_egress" + ) + + +class _BudgetDelegate: + """Minimal provider client used to exercise direct equal-budget behavior.""" + + def __init__(self, answer: str = "ok", usage: object = None) -> None: + """Configure one answer and optional provider usage payload.""" + self.max_output_tokens = 256 + self.answer = answer + self.usage = usage + self.observed_caps: list[int] = [] + + def chat(self, _agent, _messages, _temperature=0.2) -> str: + """Record the temporary output cap and return the configured answer.""" + self.observed_caps.append(self.max_output_tokens) + return self.answer + + def take_usage(self): + """Return the configured provider usage payload.""" + return self.usage + + +def _budget_agent(): + """Return one valid mock worker for cell-budget tests.""" + from contextual_orchestrator.orchestrator import ModelAgent + + return ModelAgent( + id="nim_budget_worker", + model="dryrun/chat-basic", + base_url="mock://nim-budget-test", + credential_key=nb.NIM_CREDENTIAL_NAME, + tags=("reasoning", "writing"), + ) + + +def _mp4_box(box_type: bytes, payload: bytes = b"") -> bytes: + """Build one small ISO-BMFF box for malformed-fixture regression tests.""" + import struct + + return struct.pack(">I4s", len(payload) + 8, box_type) + payload + + +def test_default_transport_rejects_invalid_timeout_values() -> None: + """Only finite positive real timeout values can reach socket setup.""" + for value in (False, 0, -1, "5", float("nan"), float("inf")): + with pytest.raises(nb.BenchmarkContractError, match="timeout_seconds"): + nb.build_default_transport(value) + + +def test_equal_budget_client_validates_and_exposes_delegate_cap() -> None: + """Equal budgets are positive integers and preserve the client cap interface.""" + for token_budget in (False, 0, 1.5): + with pytest.raises(ValueError, match="total_token_budget"): + nb.EqualBudgetModelClient(_BudgetDelegate(), token_budget, 5) + for maximum_calls in (False, 0, 1.5): + with pytest.raises(ValueError, match="maximum_calls"): + nb.EqualBudgetModelClient(_BudgetDelegate(), 20, maximum_calls) + + delegate = _BudgetDelegate() + client = nb.EqualBudgetModelClient(delegate, 20, 5) + assert client.max_output_tokens == 256 + client.max_output_tokens = 128 + assert delegate.max_output_tokens == 128 + + +@pytest.mark.parametrize("value", [True, "3", float("nan"), float("inf"), -1]) +def test_equal_budget_usage_count_rejects_invalid_values(value: object) -> None: + """Provider token counts must be finite non-negative real numbers, never booleans.""" + assert nb.EqualBudgetModelClient._coerce_usage_count(value) is None + assert nb.EqualBudgetModelClient._coerce_usage_count(3.9) == 3 + + +def test_equal_budget_usage_reconciliation_covers_all_sources() -> None: + """Reported usage replaces estimates only when both counts are usable.""" + no_usage = nb.EqualBudgetModelClient(_BudgetDelegate(usage=None), 100, 5) + assert no_usage.take_usage() is None + + non_mapping = nb.EqualBudgetModelClient(_BudgetDelegate(usage="unknown"), 100, 5) + non_mapping.chat(_budget_agent(), [{"role": "user", "content": "hi"}], 0.0) + estimated_non_mapping = non_mapping.observed_tokens + assert non_mapping.take_usage() == "unknown" + assert non_mapping.observed_tokens == estimated_non_mapping + + invalid_counts = nb.EqualBudgetModelClient( + _BudgetDelegate(usage={"prompt_tokens": True, "completion_tokens": -1}), + 100, + 5, + ) + invalid_counts.chat(_budget_agent(), [{"role": "user", "content": "hi"}], 0.0) + estimated_invalid = invalid_counts.observed_tokens + assert invalid_counts.take_usage() == { + "prompt_tokens": True, + "completion_tokens": -1, + } + assert invalid_counts.observed_tokens == estimated_invalid + + reported = nb.EqualBudgetModelClient( + _BudgetDelegate(usage={"prompt_tokens": 2, "completion_tokens": 3}), + 100, + 5, + ) + reported.chat(_budget_agent(), [{"role": "user", "content": "hi"}], 0.0) + assert reported.take_usage() == {"prompt_tokens": 2, "completion_tokens": 3} + assert reported.observed_tokens == 5 + + +def test_mp4_parser_rejects_every_malformed_box_class() -> None: + """Fixture validation fails closed on truncation, bad bounds, and missing evidence.""" + import struct + + with pytest.raises(nb.BenchmarkContractError, match="truncated box header"): + list(nb._iter_mp4_boxes(b"x")) + with pytest.raises(nb.BenchmarkContractError, match="truncated extended box"): + list(nb._iter_mp4_boxes(struct.pack(">I4s", 1, b"free"))) + + extended = struct.pack(">I4sQ", 1, b"free", 16) + assert list(nb._iter_mp4_boxes(extended)) == [(b"free", 16, 16)] + zero_sized = struct.pack(">I4s", 0, b"free") + b"payload" + assert list(nb._iter_mp4_boxes(zero_sized)) == [ + (b"free", 8, len(zero_sized)) + ] + with pytest.raises(nb.BenchmarkContractError, match="parent bounds"): + list(nb._iter_mp4_boxes(struct.pack(">I4s", 20, b"free"))) + + with pytest.raises(nb.BenchmarkContractError, match="meta box"): + list(nb._walk_mp4_boxes(_mp4_box(b"meta"))) + with pytest.raises(nb.BenchmarkContractError, match="lacks ftyp"): + nb.validate_video_probe_fixture(_mp4_box(b"ftyp")) + + required_top_level = _mp4_box(b"ftyp") + _mp4_box(b"moov") + _mp4_box(b"mdat") + with pytest.raises(nb.BenchmarkContractError, match="one 16x16 one-frame"): + nb.validate_video_probe_fixture(required_top_level) + + truncated_tkhd = ( + _mp4_box(b"ftyp") + + _mp4_box(b"moov", _mp4_box(b"tkhd", b"x")) + + _mp4_box(b"mdat") + ) + with pytest.raises(nb.BenchmarkContractError, match="tkhd box"): + nb.validate_video_probe_fixture(truncated_tkhd) + + truncated_stsz = ( + _mp4_box(b"ftyp") + + _mp4_box(b"moov", _mp4_box(b"stsz", b"short")) + + _mp4_box(b"mdat") + ) + with pytest.raises(nb.BenchmarkContractError, match="stsz box"): + nb.validate_video_probe_fixture(truncated_stsz) + + +def test_video_fixture_checksum_mismatch_fails_closed(monkeypatch: pytest.MonkeyPatch) -> None: + """A changed embedded media payload cannot silently enter provider probes.""" + monkeypatch.setattr(nb, "VIDEO_PROBE_FIXTURE_SHA256", "0" * 64) + with pytest.raises(nb.BenchmarkContractError, match="checksum"): + nb._tiny_mp4_bytes() + + +def test_reviewed_pricing_metadata_rejects_invalid_provenance(tmp_path: Path) -> None: + """Reviewed scenarios require valid dates, HTTPS source, and non-empty review fields.""" + invalid_scenarios = [ + _reviewed_pricing_scenario(source_url="http://pricing.example.test/rates"), + _reviewed_pricing_scenario(reviewed_by=""), + _reviewed_pricing_scenario(rate_basis=""), + _reviewed_pricing_scenario(uncertainty=""), + _reviewed_pricing_scenario(reviewed_at_date=3), + _reviewed_pricing_scenario(reviewed_at_date="not-a-date"), + _reviewed_pricing_scenario( + reviewed_at_date="2026-08-05", valid_until_date="2026-08-04" + ), + _reviewed_pricing_scenario( + usd_per_million_tokens={"": {"input": 1.0, "output": 2.0}} + ), + ] + for index, scenario in enumerate(invalid_scenarios): + path = _write_json(tmp_path / f"invalid_reviewed_{index}.json", scenario) + with pytest.raises(nb.BenchmarkContractError): + nb.load_pricing_scenario(path) + + +def test_live_pricing_rejects_future_review_and_accepts_current_evidence() -> None: + """A live run date must fall within the reviewed pricing validity interval.""" + future = _reviewed_pricing_scenario( + reviewed_at_date="2026-08-06", valid_until_date="2026-09-04" + ) + with pytest.raises(nb.BenchmarkContractError, match="future"): + nb.validate_live_pricing_scenario( + future, + today=__import__("datetime").date(2026, 8, 5), + ) + nb.validate_live_pricing_scenario( + _reviewed_pricing_scenario(), + today=__import__("datetime").date(2026, 8, 5), + ) + + +def test_actual_cost_evidence_validation_and_expiry_paths( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The zero access-cost claim remains complete, official, and time bounded.""" + with pytest.raises(nb.BenchmarkContractError, match="missing actual_cost_evidence"): + nb._validate_actual_cost_evidence({}) + + missing = {"actual_cost_evidence": dict(nb.ACTUAL_COST_EVIDENCE)} + del missing["actual_cost_evidence"]["source_title"] + with pytest.raises(nb.BenchmarkContractError, match="missing fields"): + nb._validate_actual_cost_evidence(missing) + + wrong_cost = {"actual_cost_evidence": dict(nb.ACTUAL_COST_EVIDENCE)} + wrong_cost["actual_cost_evidence"]["actual_cost_usd"] = 1.0 + with pytest.raises(nb.BenchmarkContractError, match="zero-cost"): + nb._validate_actual_cost_evidence(wrong_cost) + + wrong_source = {"actual_cost_evidence": dict(nb.ACTUAL_COST_EVIDENCE)} + wrong_source["actual_cost_evidence"]["source_url"] = "https://example.test" + with pytest.raises(nb.BenchmarkContractError, match="General FAQ"): + nb._validate_actual_cost_evidence(wrong_source) + + invalid_dates = {"actual_cost_evidence": dict(nb.ACTUAL_COST_EVIDENCE)} + invalid_dates["actual_cost_evidence"]["reviewed_at_date"] = "2026-09-05" + with pytest.raises(nb.BenchmarkContractError, match="validity precedes"): + nb._validate_actual_cost_evidence(invalid_dates) + + monkeypatch.setitem(nb.ACTUAL_COST_EVIDENCE, "reviewed_at_date", "2026-08-06") + with pytest.raises(nb.BenchmarkContractError, match="future"): + nb._require_current_actual_cost_evidence( + __import__("datetime").date(2026, 8, 5) + ) + monkeypatch.setitem(nb.ACTUAL_COST_EVIDENCE, "reviewed_at_date", "2026-08-05") + monkeypatch.setitem(nb.ACTUAL_COST_EVIDENCE, "valid_until_date", "2026-08-05") + nb._require_current_actual_cost_evidence(__import__("datetime").date(2026, 8, 5)) + with pytest.raises(nb.BenchmarkContractError, match="expired"): + nb._require_current_actual_cost_evidence( + __import__("datetime").date(2026, 8, 6) + ) + + +def test_sufficient_evidence_is_still_human_review_gated() -> None: + """Meeting sample thresholds changes status but never auto-selects a route.""" + cells = [] + for task_index in range(nb.MINIMUM_PAIRED_TASK_COUNT): + task_id = f"paired_task_{task_index}" + for policy_name in ("route_once", "conduct_bounded"): + cells.append( + { + "policy_name": policy_name, + "task_id": task_id, + "run_outcome": "success", + } + ) + summary = nb._evaluation_evidence_summary( + cells, + nb.MINIMUM_PAIRED_TASK_COUNT, + ) + assert summary["evidence_status"] == "evidence_review_required" + assert summary["decision_use"] == "production_candidate_review" + assert summary["routing_recommendation"] is None + + +def test_live_run_requires_provenance_before_transport(tmp_path: Path) -> None: + """Missing live revision identity fails before credentials or transport are used.""" + with pytest.raises(nb.BenchmarkContractError, match="git-sha"): + nb.run_benchmark( + "live", + TASK_MANIFEST_PATH, + None, + str(tmp_path), + transport=_unexpected_transport, + ) diff --git a/tests/test_nim_benchmark_review_regressions.py b/tests/test_nim_benchmark_review_regressions.py new file mode 100644 index 00000000..ce0b3c45 --- /dev/null +++ b/tests/test_nim_benchmark_review_regressions.py @@ -0,0 +1,169 @@ +"""Regressions for exact-head NIM benchmark review findings. + +These tests intentionally exercise the buyer-visible evidence and failure +boundaries called out by the current review. They remain fully offline and do +not read provider credentials. +""" + +from __future__ import annotations + +import re +import subprocess +import sys +from pathlib import Path +from typing import Any + +import pytest + +from contextual_orchestrator import nim_benchmark as nb + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +TASK_MANIFEST_PATH = REPOSITORY_ROOT / "examples" / "nim_task_manifest.json" +_MARKDOWN_REQUIRED_PATHS = ( + "evaluation.observed_paired_task_count", + "evaluation.observed_completion_fraction", + "honesty_labels.hypothetical_cost_source", +) + + +def _set_report_path(report: dict[str, Any], path: str, value: Any) -> None: + """Populate one dotted report path for focused schema tests.""" + node = report + keys = path.split(".") + for key in keys[:-1]: + child = node.setdefault(key, {}) + assert isinstance(child, dict) + node = child + node[keys[-1]] = value + + +def _delete_report_path(report: dict[str, Any], path: str) -> None: + """Delete one dotted report path from a complete synthetic report.""" + node = report + keys = path.split(".") + for key in keys[:-1]: + child = node[key] + assert isinstance(child, dict) + node = child + del node[keys[-1]] + + +def _complete_schema_report() -> dict[str, Any]: + """Return a synthetic report containing every declared and rendered path.""" + report: dict[str, Any] = {} + for path in (*nb._REPORT_REQUIRED_PATHS, *_MARKDOWN_REQUIRED_PATHS): + _set_report_path(report, path, None) + return report + + +@pytest.mark.parametrize("missing_path", _MARKDOWN_REQUIRED_PATHS) +def test_report_schema_rejects_missing_markdown_input(missing_path: str) -> None: + """Schema acceptance must guarantee that Markdown rendering cannot KeyError.""" + report = _complete_schema_report() + _delete_report_path(report, missing_path) + + with pytest.raises(nb.BenchmarkContractError, match=re.escape(missing_path)): + nb.validate_report_schema(report) + + +def test_pareto_frontiers_exclude_policies_without_successful_cells() -> None: + """A failed policy must not appear efficient merely because its metrics are zero.""" + summaries = [ + { + "policy_name": "failed_policy", + "success_count": 0, + "mean_task_score": 1.0, + "mean_latency_ms": 0.0, + "mean_hypothetical_cost_usd": 0.0, + }, + { + "policy_name": "valid_policy", + "success_count": 3, + "mean_task_score": 0.8, + "mean_latency_ms": 10.0, + "mean_hypothetical_cost_usd": 0.1, + }, + ] + + frontiers = nb.build_pareto_frontiers(summaries) + + assert [row["policy_name"] for row in frontiers["quality_vs_latency"]] == [ + "valid_policy" + ] + assert [ + row["policy_name"] + for row in frontiers["quality_vs_hypothetical_cost"] + ] == ["valid_policy"] + assert frontiers["excluded_zero_success_policies"] == ["failed_policy"] + + +def test_catalog_json_recursion_is_normalized_to_domain_error(monkeypatch) -> None: + """Attacker-controlled JSON depth must not escape as a raw RecursionError.""" + + def raise_recursion(_text: str) -> Any: + raise RecursionError("catalog nesting exceeded interpreter depth") + + monkeypatch.setattr(nb.json, "loads", raise_recursion) + + with pytest.raises(nb.CatalogDiscoveryError, match="not valid JSON"): + nb.parse_model_catalog_body(b'[{"nested": true}]') + + +def test_fuzz_target_does_not_ignore_raw_catalog_recursion() -> None: + """The fuzz target must trust only the parser's normalized domain failure.""" + target_text = (REPOSITORY_ROOT / "fuzz" / "targets.py").read_text( + encoding="utf-8" + ) + catalog_target = target_text.split("def exercise_nim_catalog", 1)[1] + assert "except RecursionError" not in catalog_target + assert "plain json RecursionError" not in catalog_target + + +class _CapturedModelTimeout(RuntimeError): + """Stop a live setup immediately after recording the model-client timeout.""" + + +def test_live_benchmark_preserves_positive_subsecond_model_timeout( + monkeypatch, + tmp_path: Path, +) -> None: + """A valid 250ms operator timeout must not be truncated to zero seconds.""" + observed: dict[str, Any] = {} + + def capture_client(_request_budget: nb.RequestBudget, **kwargs: Any) -> Any: + observed.update(kwargs) + raise _CapturedModelTimeout + + monkeypatch.setattr(nb, "get_credential", lambda _name: "test-secret") + monkeypatch.setattr(nb, "_BudgetedModelClient", capture_client) + + with pytest.raises(_CapturedModelTimeout): + nb.run_benchmark( + "live", + str(TASK_MANIFEST_PATH), + None, + str(tmp_path), + endpoint="https://nim.example.test/v1", + timeout_seconds=0.25, + git_sha="a" * 40, + workflow_run_id="12345", + transport=lambda *_args: (500, b"{}"), + ) + + assert observed["timeout"] == 0.25 + + +def test_standalone_nim_test_runner_skips_fixture_callables() -> None: + """The optional direct runner must execute only zero-argument test functions.""" + completed = subprocess.run( + [sys.executable, "tests/test_nim_benchmark.py"], + cwd=REPOSITORY_ROOT, + capture_output=True, + text=True, + timeout=180, + check=False, + ) + + assert completed.returncode == 0, completed.stderr + assert completed.stdout.rstrip().endswith("ok") diff --git a/tests/test_nim_benchmark_workflow_contract.py b/tests/test_nim_benchmark_workflow_contract.py new file mode 100644 index 00000000..ebd1c45b --- /dev/null +++ b/tests/test_nim_benchmark_workflow_contract.py @@ -0,0 +1,119 @@ +"""Static least-privilege contracts for scheduled NIM benchmark automation.""" + +from __future__ import annotations + +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] + + +def _ordered_step_blocks( + workflow: str, + first_step_name: str, + second_step_name: str, +) -> tuple[str, str]: + """Return non-empty ordered workflow slices for two named steps. + + Raises: + AssertionError: If either step is absent, reversed, or has an empty + structural slice. + """ + first_marker = f"- name: {first_step_name}" + second_marker = f"- name: {second_step_name}" + first_start = workflow.index(first_marker) + second_start = workflow.index(second_marker) + assert first_start < second_start, ( + f"{first_step_name!r} must precede {second_step_name!r}" + ) + first_block = workflow[first_start:second_start] + second_block = workflow[second_start:] + assert first_block.strip(), f"{first_step_name!r} block must not be empty" + assert second_block.strip(), f"{second_step_name!r} block must not be empty" + return first_block, second_block + + +def test_dry_run_workflow_never_receives_live_nvidia_secret() -> None: + """The zero-egress dry path must have no NVIDIA credential in its environment.""" + workflow = (REPOSITORY_ROOT / ".github/workflows/nim-benchmark.yml").read_text( + encoding="utf-8" + ) + dry_block, live_block = _ordered_step_blocks( + workflow, + "Run dry benchmark", + "Run live benchmark", + ) + + assert "NVIDIA_NIM_API_KEY" not in dry_block + assert live_block.count("NVIDIA_NIM_API_KEY:") == 1 + assert "secrets.NVIDIA_NIM_API_KEY" in live_block + + +def test_dry_run_workflow_honors_optional_pricing_scenario_without_secret() -> None: + """Manual dry runs may validate an explicit scenario without live credentials.""" + workflow = (REPOSITORY_ROOT / ".github/workflows/nim-benchmark.yml").read_text( + encoding="utf-8" + ) + dry_block, _ = _ordered_step_blocks( + workflow, + "Run dry benchmark", + "Run live benchmark", + ) + + assert "PRICING_SCENARIO: ${{ inputs.pricing_scenario }}" in dry_block + assert 'extra_args+=(--pricing-scenario "$PRICING_SCENARIO")' in dry_block + assert '"${extra_args[@]}"' in dry_block + + +def test_temporary_review_export_job_is_absent_from_mergeable_tests_workflow() -> None: + """Mergeable CI must not retain the one-use exact-head export mechanism.""" + workflow = (REPOSITORY_ROOT / ".github/workflows/tests.yml").read_text( + encoding="utf-8" + ) + assert "export_review_workspace" not in workflow + assert "Recover reviewed transformation source as inert evidence" not in workflow + + +def test_temporary_review_evidence_source_is_absent() -> None: + """Mergeable source must not retain any one-use transformation payload.""" + assert not (REPOSITORY_ROOT / ".review-evidence/nim-source-repair.yml").exists() + assert not ( + REPOSITORY_ROOT / ".github/workflows/export-pr90-workspace.yml" + ).exists() + + +def test_compatibility_monkeypatch_module_is_absent() -> None: + """Security and budget behavior must live directly in the optional benchmark.""" + assert not ( + REPOSITORY_ROOT / "contextual_orchestrator/nim_benchmark_hardening.py" + ).exists() + assert not (REPOSITORY_ROOT / "tests/test_nim_benchmark_hardening.py").exists() + + +def test_tests_workflow_enforces_nim_coverage_docstrings_and_package_smoke() -> None: + """The exact PR head must prove 100% branches, docstrings, and installability.""" + workflow = (REPOSITORY_ROOT / ".github/workflows/tests.yml").read_text( + encoding="utf-8" + ) + assert "nim_benchmark_quality:" in workflow + assert "coverage run --branch" in workflow + assert "--source=contextual_orchestrator.nim_benchmark" in workflow + assert "tests/test_nim_benchmark_review_regressions.py" in workflow + assert "coverage report" in workflow and "--fail-under=100" in workflow + assert "interrogate -f 100 contextual_orchestrator/nim_benchmark.py" in workflow + assert "pip wheel --no-deps . --wheel-dir dist" in workflow + assert "--no-build-isolation" not in workflow + assert '--target "$RUNNER_TEMP/nim-wheel-site"' in workflow + assert 'cd "$RUNNER_TEMP"' in workflow + assert 'PYTHONPATH="$RUNNER_TEMP/nim-wheel-site"' in workflow + assert "import contextual_orchestrator.nim_benchmark" in workflow + + +def test_scheduled_live_budget_covers_the_reviewed_current_catalog_scale() -> None: + """Monthly live runs reserve enough calls for full probes plus evaluation.""" + workflow = (REPOSITORY_ROOT / ".github/workflows/nim-benchmark.yml").read_text( + encoding="utf-8" + ) + + assert 'echo "max_requests=2000" >> "$GITHUB_OUTPUT"' in workflow + assert 'echo "max_requests=300" >> "$GITHUB_OUTPUT"' not in workflow diff --git a/tests/test_nim_csv_evidence.py b/tests/test_nim_csv_evidence.py new file mode 100644 index 00000000..6a4d9b3c --- /dev/null +++ b/tests/test_nim_csv_evidence.py @@ -0,0 +1,326 @@ +"""Regression tests for complete, fail-closed NIM CSV assignment evidence.""" + +from __future__ import annotations + +import csv +import io +import json +from pathlib import Path + +import pytest + +from contextual_orchestrator import nim_csv_evidence as csv_evidence + + +def _write_report(path: Path, cells: list[dict[str, object]]) -> None: + path.write_text( + json.dumps({"evaluation": {"evaluation_cells": cells}}), + encoding="utf-8", + ) + + +def _write_csv(path: Path, rows: list[dict[str, str]], *, include_assignment: bool = False) -> None: + fieldnames = ["policy_name", "task_id", "task_score"] + if include_assignment: + fieldnames.append("models_used_json") + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + +def _model_use(step_id: str, model_id: str) -> dict[str, str]: + return { + "step_id": step_id, + "role": "worker", + "agent_id": f"agent_{step_id}", + "model_id": model_id, + } + + +def test_enrich_csv_adds_deterministic_role_and_model_assignment_evidence( + tmp_path: Path, +) -> None: + report_path = tmp_path / "benchmark_report.json" + csv_path = tmp_path / "benchmark_cells.csv" + cells = [ + { + "policy_name": "conduct_bounded", + "task_id": "locked_task_two", + "models_used": [ + _model_use("step_two", "vendor/model-b"), + _model_use("step_one", "vendor/model-a"), + ], + }, + { + "policy_name": "route_once", + "task_id": "locked_task_one", + "models_used": [], + }, + ] + _write_report(report_path, cells) + _write_csv( + csv_path, + [ + { + "policy_name": "route_once", + "task_id": "locked_task_one", + "task_score": "1.0", + }, + { + "policy_name": "conduct_bounded", + "task_id": "locked_task_two", + "task_score": "0.5", + }, + ], + ) + + csv_evidence.enrich_benchmark_cell_csv(report_path, csv_path) + + with csv_path.open(encoding="utf-8", newline="") as handle: + rows = list(csv.DictReader(handle)) + assert list(rows[0]) == [ + "policy_name", + "task_id", + "task_score", + "models_used_json", + ] + assert json.loads(rows[0]["models_used_json"]) == [] + assert json.loads(rows[1]["models_used_json"]) == cells[0]["models_used"] + assert rows[1]["models_used_json"] == json.dumps( + cells[0]["models_used"], + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + + # Re-enrichment is idempotent and replaces rather than duplicates the column. + first_bytes = csv_path.read_bytes() + csv_evidence.enrich_benchmark_cell_csv(report_path, csv_path) + assert csv_path.read_bytes() == first_bytes + + +@pytest.mark.parametrize( + ("report", "error_match"), + [ + ({}, "evaluation.evaluation_cells"), + ({"evaluation": {"evaluation_cells": "not-a-list"}}, "evaluation.evaluation_cells"), + ( + { + "evaluation": { + "evaluation_cells": [ + {"policy_name": "route_once", "task_id": "task_one"} + ] + } + }, + "models_used", + ), + ( + { + "evaluation": { + "evaluation_cells": [ + { + "policy_name": "route_once", + "task_id": "task_one", + "models_used": ["not-an-object"], + } + ] + } + }, + "model assignment", + ), + ( + { + "evaluation": { + "evaluation_cells": [ + { + "policy_name": "route_once", + "task_id": "task_one", + "models_used": [ + { + "step_id": "step_one", + "role": "worker", + "agent_id": "agent_one", + "model_id": "", + } + ], + } + ] + } + }, + "model_id", + ), + ], +) +def test_enrich_csv_rejects_malformed_report_without_replacing_existing_csv( + tmp_path: Path, + report: dict[str, object], + error_match: str, +) -> None: + report_path = tmp_path / "benchmark_report.json" + csv_path = tmp_path / "benchmark_cells.csv" + report_path.write_text(json.dumps(report), encoding="utf-8") + _write_csv( + csv_path, + [{"policy_name": "route_once", "task_id": "task_one", "task_score": "1"}], + ) + original = csv_path.read_bytes() + + with pytest.raises(csv_evidence.CsvEvidenceError, match=error_match): + csv_evidence.enrich_benchmark_cell_csv(report_path, csv_path) + + assert csv_path.read_bytes() == original + + +def test_enrich_csv_rejects_duplicate_or_mismatched_cell_keys(tmp_path: Path) -> None: + report_path = tmp_path / "benchmark_report.json" + csv_path = tmp_path / "benchmark_cells.csv" + duplicated_cell = { + "policy_name": "route_once", + "task_id": "task_one", + "models_used": [], + } + _write_report(report_path, [duplicated_cell, duplicated_cell]) + _write_csv( + csv_path, + [{"policy_name": "route_once", "task_id": "task_one", "task_score": "1"}], + ) + with pytest.raises(csv_evidence.CsvEvidenceError, match="duplicate report cell"): + csv_evidence.enrich_benchmark_cell_csv(report_path, csv_path) + + _write_report(report_path, [duplicated_cell]) + _write_csv( + csv_path, + [ + {"policy_name": "route_once", "task_id": "task_one", "task_score": "1"}, + {"policy_name": "route_once", "task_id": "task_one", "task_score": "1"}, + ], + ) + with pytest.raises(csv_evidence.CsvEvidenceError, match="duplicate CSV cell"): + csv_evidence.enrich_benchmark_cell_csv(report_path, csv_path) + + _write_csv( + csv_path, + [{"policy_name": "route_once", "task_id": "task_two", "task_score": "1"}], + ) + with pytest.raises(csv_evidence.CsvEvidenceError, match="cell identity mismatch"): + csv_evidence.enrich_benchmark_cell_csv(report_path, csv_path) + + +def test_enrich_csv_rejects_missing_identity_columns_and_invalid_json(tmp_path: Path) -> None: + report_path = tmp_path / "benchmark_report.json" + csv_path = tmp_path / "benchmark_cells.csv" + report_path.write_text("{not json", encoding="utf-8") + csv_path.write_text("policy_name\nroute_once\n", encoding="utf-8") + with pytest.raises(csv_evidence.CsvEvidenceError, match="valid JSON"): + csv_evidence.enrich_benchmark_cell_csv(report_path, csv_path) + + _write_report( + report_path, + [{"policy_name": "route_once", "task_id": "task_one", "models_used": []}], + ) + with pytest.raises(csv_evidence.CsvEvidenceError, match="task_id"): + csv_evidence.enrich_benchmark_cell_csv(report_path, csv_path) + + +def test_output_directory_parser_supports_default_split_and_equals_forms() -> None: + assert csv_evidence.output_directory_from_argv(["--dry-run"]) == Path( + "benchmark_artifacts" + ) + assert csv_evidence.output_directory_from_argv( + ["--dry-run", "--output-dir", "custom evidence"] + ) == Path("custom evidence") + assert csv_evidence.output_directory_from_argv( + ["--output-dir=equals-evidence"] + ) == Path("equals-evidence") + with pytest.raises(csv_evidence.CsvEvidenceError, match="non-empty"): + csv_evidence.output_directory_from_argv(["--output-dir="]) + with pytest.raises(csv_evidence.CsvEvidenceError, match="requires a value"): + csv_evidence.output_directory_from_argv(["--output-dir"]) + + +def test_cli_wrapper_publishes_success_only_after_csv_enrichment(tmp_path: Path) -> None: + output_dir = tmp_path / "evidence" + + def benchmark_cli(argv: list[str]) -> int: + staged_output_dir = csv_evidence.output_directory_from_argv(argv) + staged_output_dir.mkdir(exist_ok=True) + _write_report( + staged_output_dir / "benchmark_report.json", + [{"policy_name": "route_once", "task_id": "task_one", "models_used": []}], + ) + _write_csv( + staged_output_dir / "benchmark_cells.csv", + [{"policy_name": "route_once", "task_id": "task_one", "task_score": "1"}], + ) + (staged_output_dir / "benchmark_summary.md").write_text( + "# summary\n", + encoding="utf-8", + ) + print( + json.dumps( + { + "run_mode": "dry_run", + "artifact_paths": { + "json_path": str(staged_output_dir / "benchmark_report.json"), + "csv_path": str(staged_output_dir / "benchmark_cells.csv"), + "markdown_path": str(staged_output_dir / "benchmark_summary.md"), + }, + } + ) + ) + return 0 + + stdout = io.StringIO() + result = csv_evidence.run_benchmark_cli_with_complete_csv( + ["--output-dir", str(output_dir)], + benchmark_cli=benchmark_cli, + stdout=stdout, + ) + + assert result == 0 + payload = json.loads(stdout.getvalue()) + assert payload["run_mode"] == "dry_run" + assert payload["artifact_paths"] == { + "json_path": str(output_dir / "benchmark_report.json"), + "csv_path": str(output_dir / "benchmark_cells.csv"), + "markdown_path": str(output_dir / "benchmark_summary.md"), + } + assert "models_used_json" in (output_dir / "benchmark_cells.csv").read_text( + encoding="utf-8" + ) + + +def test_cli_wrapper_preserves_benchmark_failure_and_fails_closed_on_enrichment( + tmp_path: Path, +) -> None: + stdout = io.StringIO() + + def failed_benchmark_cli(argv: list[str]) -> int: + print(json.dumps({"benchmark_failed_closed": True, "error_class": "TestError"})) + return 1 + + assert ( + csv_evidence.run_benchmark_cli_with_complete_csv( + [], benchmark_cli=failed_benchmark_cli, stdout=stdout + ) + == 1 + ) + assert json.loads(stdout.getvalue())["error_class"] == "TestError" + + stdout = io.StringIO() + + def incomplete_success(argv: list[str]) -> int: + print(json.dumps({"run_mode": "dry_run"})) + return 0 + + result = csv_evidence.run_benchmark_cli_with_complete_csv( + ["--output-dir", str(tmp_path / "missing")], + benchmark_cli=incomplete_success, + stdout=stdout, + ) + failure = json.loads(stdout.getvalue()) + assert result == 1 + assert failure["benchmark_failed_closed"] is True + assert failure["error_class"] in {"CsvEvidenceError", "FileNotFoundError"} + assert "run_mode" not in failure diff --git a/tests/test_nim_csv_evidence_edges.py b/tests/test_nim_csv_evidence_edges.py new file mode 100644 index 00000000..78c15737 --- /dev/null +++ b/tests/test_nim_csv_evidence_edges.py @@ -0,0 +1,95 @@ +"""Edge coverage for the NIM CSV evidence adapter's fail-closed branches.""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path + +import pytest + +from contextual_orchestrator import nim_csv_evidence as csv_evidence + + +def _write_valid_csv(path: Path, rows: list[dict[str, str]]) -> None: + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=["policy_name", "task_id"]) + writer.writeheader() + writer.writerows(rows) + + +def _write_valid_report(path: Path, cells: list[object]) -> None: + path.write_text( + json.dumps({"evaluation": {"evaluation_cells": cells}}), + encoding="utf-8", + ) + + +def test_report_rejects_non_object_root_and_non_object_cells(tmp_path: Path) -> None: + report_path = tmp_path / "benchmark_report.json" + csv_path = tmp_path / "benchmark_cells.csv" + _write_valid_csv(csv_path, []) + + report_path.write_text("[]", encoding="utf-8") + with pytest.raises(csv_evidence.CsvEvidenceError, match="evaluation.evaluation_cells"): + csv_evidence.enrich_benchmark_cell_csv(report_path, csv_path) + + _write_valid_report(report_path, ["not-an-object"]) + with pytest.raises(csv_evidence.CsvEvidenceError, match="report cell must be an object"): + csv_evidence.enrich_benchmark_cell_csv(report_path, csv_path) + + +@pytest.mark.parametrize( + "cell", + [ + {"policy_name": "", "task_id": "task_one", "models_used": []}, + {"policy_name": "route_once", "task_id": "", "models_used": []}, + {"policy_name": 7, "task_id": "task_one", "models_used": []}, + ], +) +def test_report_rejects_empty_or_non_string_cell_identity( + tmp_path: Path, + cell: dict[str, object], +) -> None: + report_path = tmp_path / "benchmark_report.json" + csv_path = tmp_path / "benchmark_cells.csv" + _write_valid_report(report_path, [cell]) + _write_valid_csv(csv_path, []) + with pytest.raises(csv_evidence.CsvEvidenceError, match="requires non-empty"): + csv_evidence.enrich_benchmark_cell_csv(report_path, csv_path) + + +def test_csv_rejects_invalid_utf8_empty_header_and_empty_identity(tmp_path: Path) -> None: + report_path = tmp_path / "benchmark_report.json" + csv_path = tmp_path / "benchmark_cells.csv" + _write_valid_report(report_path, []) + + csv_path.write_bytes(b"\xff\xfe") + with pytest.raises(csv_evidence.CsvEvidenceError, match="not readable"): + csv_evidence.enrich_benchmark_cell_csv(report_path, csv_path) + + csv_path.write_text("", encoding="utf-8") + with pytest.raises(csv_evidence.CsvEvidenceError, match="identity columns"): + csv_evidence.enrich_benchmark_cell_csv(report_path, csv_path) + + _write_valid_report( + report_path, + [{"policy_name": "route_once", "task_id": "task_one", "models_used": []}], + ) + _write_valid_csv(csv_path, [{"policy_name": "", "task_id": "task_one"}]) + with pytest.raises(csv_evidence.CsvEvidenceError, match="CSV cell requires non-empty"): + csv_evidence.enrich_benchmark_cell_csv(report_path, csv_path) + + +def test_empty_report_and_csv_are_enriched_deterministically(tmp_path: Path) -> None: + report_path = tmp_path / "benchmark_report.json" + csv_path = tmp_path / "benchmark_cells.csv" + _write_valid_report(report_path, []) + _write_valid_csv(csv_path, []) + + csv_evidence.enrich_benchmark_cell_csv(report_path, csv_path) + + with csv_path.open(encoding="utf-8", newline="") as handle: + reader = csv.DictReader(handle) + assert reader.fieldnames == ["policy_name", "task_id", "models_used_json"] + assert list(reader) == [] diff --git a/tests/test_nim_fuzz_instrumentation_contract.py b/tests/test_nim_fuzz_instrumentation_contract.py new file mode 100644 index 00000000..a6452a96 --- /dev/null +++ b/tests/test_nim_fuzz_instrumentation_contract.py @@ -0,0 +1,42 @@ +"""Contracts for coverage-guided instrumentation of the NIM catalog parser.""" + +from __future__ import annotations + +import ast +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +HARNESS_PATH = REPOSITORY_ROOT / "fuzz" / "fuzz_nim_catalog.py" + + +def _instrumented_imports() -> set[str]: + """Return module names imported inside ``atheris.instrument_imports``.""" + module = ast.parse(HARNESS_PATH.read_text(encoding="utf-8")) + imported: set[str] = set() + for node in ast.walk(module): + if not isinstance(node, ast.With): + continue + if not any( + isinstance(item.context_expr, ast.Call) + and isinstance(item.context_expr.func, ast.Attribute) + and isinstance(item.context_expr.func.value, ast.Name) + and item.context_expr.func.value.id == "atheris" + and item.context_expr.func.attr == "instrument_imports" + for item in node.items + ): + continue + for statement in node.body: + if isinstance(statement, ast.Import): + imported.update(alias.name for alias in statement.names) + elif isinstance(statement, ast.ImportFrom) and statement.module: + imported.add(statement.module) + return imported + + +def test_nim_parser_module_is_imported_inside_atheris_instrumentation() -> None: + """Parser branches must be loaded while Atheris import hooks are active.""" + imported = _instrumented_imports() + + assert "contextual_orchestrator.nim_benchmark" in imported + assert "fuzz.targets" in imported diff --git a/tests/test_nim_readme_evidence_status.py b/tests/test_nim_readme_evidence_status.py new file mode 100644 index 00000000..e1c82797 --- /dev/null +++ b/tests/test_nim_readme_evidence_status.py @@ -0,0 +1,26 @@ +"""Contract tests for buyer-facing NIM benchmark evidence-status wording.""" + +from pathlib import Path + + +README_PATH = Path(__file__).resolve().parents[1] / "README.md" + + +def test_nim_readme_describes_completion_dependent_evidence_status() -> None: + """Explain both evidence outcomes without claiming automatic routing changes.""" + readme = README_PATH.read_text(encoding="utf-8") + normalized = " ".join(readme.split()) + assert "The bundled manifest contains thirty locked tasks." in normalized + assert ( + "It may reach `evidence_review_required` when at least 90% of policy-task " + "cells complete and the paired-task floor is met; otherwise it reports " + "`insufficient_evidence`." in normalized + ) + assert ( + "no benchmark artifact automatically changes production routing" + in normalized.casefold() + ) + assert ( + "manifest is smoke-sized and reports `insufficient_evidence`" + not in normalized + ) diff --git a/tests/test_nim_strict_scorer_validity.py b/tests/test_nim_strict_scorer_validity.py new file mode 100644 index 00000000..9aa2caf7 --- /dev/null +++ b/tests/test_nim_strict_scorer_validity.py @@ -0,0 +1,379 @@ +"""Validity contracts for strict locked-answer benchmark scoring.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from contextual_orchestrator import nim_benchmark as nb +from contextual_orchestrator import nim_strict_scoring as strict + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +TASK_MANIFEST_PATH = REPOSITORY_ROOT / "examples" / "nim_task_manifest.json" + + +def _source_manifest() -> dict[str, Any]: + """Load the reviewed authoring manifest without changing its source file.""" + return json.loads(TASK_MANIFEST_PATH.read_text(encoding="utf-8")) + + +def test_strict_numeric_scorer_requires_the_entire_answer() -> None: + """Contradictory prose must not earn credit merely by containing the answer.""" + strict.enable_strict_evidence_scoring() + scorer = nb.SCORER_REGISTRY[("exact_number_match", "2")] + + assert scorer({"number": "21"}, "21") == 1.0 + assert scorer({"number": "21"}, " 21.0 ") == 1.0 + assert scorer({"number": "0.05"}, ".05") == 1.0 + assert scorer({"number": "21"}, "The answer is 21.") == 0.0 + assert scorer({"number": "21"}, "not 21") == 0.0 + assert scorer({"number": "21"}, "21 22") == 0.0 + assert scorer({"number": "21"}, "NaN") == 0.0 + + +@pytest.mark.parametrize("expected", [{}, {"number": 21}, {"number": "NaN"}]) +def test_strict_numeric_scorer_rejects_invalid_expected_literals( + expected: dict[str, object], +) -> None: + """Invalid numeric answer keys must fail manifest validation, not score zero.""" + with pytest.raises(nb.BenchmarkContractError, match="finite numeric literal"): + strict.score_exact_number_match_v2(expected, "21") + + +def test_exact_text_scorer_rejects_substrings_and_honors_case_policy() -> None: + """Complete text matching must preserve each task's declared case semantics.""" + strict.enable_strict_evidence_scoring() + scorer = nb.SCORER_REGISTRY[("exact_text_match", "1")] + + assert scorer({"texts": ["Au"], "case_sensitive": False}, " au ") == 1.0 + assert scorer({"texts": ["Au"], "case_sensitive": True}, "Au") == 1.0 + assert scorer({"texts": ["Au"], "case_sensitive": True}, "au") == 0.0 + assert ( + scorer( + {"texts": ["Pacific", "Pacific Ocean"], "case_sensitive": False}, + "PACIFIC OCEAN", + ) + == 1.0 + ) + assert scorer({"texts": ["caf\u00e9"]}, "cafe\u0301") == 1.0 + assert scorer({"texts": ["Au"]}, "Australia") == 0.0 + assert scorer({"texts": ["Au"]}, "not Au") == 0.0 + + +@pytest.mark.parametrize( + ("expected", "message"), + [ + ({}, "non-empty texts list"), + ({"texts": "Au"}, "non-empty texts list"), + ({"texts": [7]}, "list of strings"), + ({"texts": [" "]}, "list of strings"), + ({"texts": ["Au", " au "]}, "duplicate normalized answer"), + ({"texts": ["Au"], "case_sensitive": "yes"}, "case_sensitive must be boolean"), + ], +) +def test_exact_text_scorer_rejects_invalid_answer_keys( + expected: dict[str, object], + message: str, +) -> None: + """Malformed text alternatives must never silently create scoring evidence.""" + with pytest.raises(nb.BenchmarkContractError, match=message): + strict.score_exact_text_match(expected, "Au") + + +def test_activation_is_idempotent_and_fails_closed_on_identity_collision( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Versioned scorer ownership may be repeated but never silently replaced.""" + strict.enable_strict_evidence_scoring() + strict.enable_strict_evidence_scoring() + assert nb.SCORER_REGISTRY[("exact_number_match", "2")] is strict.score_exact_number_match_v2 + + monkeypatch.setitem( + nb.SCORER_REGISTRY, + ("exact_number_match", "2"), + lambda _expected, _answer: 0.0, + ) + with pytest.raises(nb.BenchmarkContractError, match="identity collision"): + strict.enable_strict_evidence_scoring() + + +def test_authoring_manifest_declares_objective_text_aliases_and_context() -> None: + """Locked text tasks must declare material aliases and disambiguating context.""" + tasks = {task["task_id"]: task for task in _source_manifest()["tasks"]} + + assert tasks["chemical_symbol_gold"]["expected"]["strict_case_sensitive"] is True + assert tasks["largest_ocean_name"]["expected"]["strict_texts"] == [ + "Pacific", + "Pacific Ocean", + ] + assert "fruit" in tasks["korean_word_translation"]["prompt"].casefold() + + +def test_strict_manifest_derivation_upgrades_only_locked_tasks( + tmp_path: Path, +) -> None: + """Headline evidence uses strict versions while exploratory tuning stays legacy.""" + strict.enable_strict_evidence_scoring() + source = _source_manifest() + derived = strict.strict_task_manifest_payload(source) + + assert derived is not source + assert derived["scoring_policy_version"] == strict.STRICT_SCORING_POLICY_VERSION + assert derived["manifest_version"].endswith( + "+strict." + strict.STRICT_SCORING_POLICY_VERSION + ) + assert source["tasks"][0]["scorer"]["version"] == "1" + + locked = nb.locked_evaluation_tasks(derived) + locked_scorers = { + (task["scorer"]["name"], task["scorer"]["version"]) + for task in locked + } + assert locked_scorers == { + ("exact_number_match", "2"), + ("exact_text_match", "1"), + } + exploratory = [task for task in derived["tasks"] if task["split"] == "exploratory"] + assert { + (task["scorer"]["name"], task["scorer"]["version"]) + for task in exploratory + } == {("substring_match", "1")} + + derived_tasks = {task["task_id"]: task for task in derived["tasks"]} + assert derived_tasks["capital_recall_france"]["expected"] == { + "texts": ["Paris"], + "case_sensitive": False, + } + assert derived_tasks["chemical_symbol_gold"]["expected"] == { + "texts": ["Au"], + "case_sensitive": True, + } + assert derived_tasks["largest_ocean_name"]["expected"] == { + "texts": ["Pacific", "Pacific Ocean"], + "case_sensitive": False, + } + + derived_path = tmp_path / "derived.json" + derived_path.write_text(json.dumps(derived), encoding="utf-8") + validated = nb.load_task_manifest(str(derived_path)) + assert len(nb.locked_evaluation_tasks(validated)) == 30 + + +def test_strict_manifest_preserves_already_strict_locked_tasks() -> None: + """A caller-supplied strict manifest must remain semantically unchanged.""" + source = { + "manifest_version": "strict-source.1", + "tasks": [ + { + "task_id": "strict_number_task", + "split": "locked", + "prompt": "Return one number only.", + "scorer": {"name": "exact_number_match", "version": "2"}, + "expected": {"number": "7"}, + }, + { + "task_id": "strict_text_task", + "split": "locked", + "prompt": "Return one word only.", + "scorer": {"name": "exact_text_match", "version": "1"}, + "expected": {"texts": ["answer"], "case_sensitive": True}, + }, + ], + } + + derived = strict.strict_task_manifest_payload(source) + assert derived["tasks"] == source["tasks"] + + +@pytest.mark.parametrize( + ("manifest", "message"), + [ + ([], "manifest object"), + ({"manifest_version": "1", "tasks": "bad"}, "tasks list"), + ({"manifest_version": "1", "tasks": ["bad"]}, "every task"), + ( + { + "manifest_version": "1", + "tasks": [ + { + "task_id": "bad_locked_task", + "split": "locked", + "scorer": "bad", + "expected": {}, + } + ], + }, + "scorer and expected objects", + ), + ( + { + "manifest_version": "1", + "tasks": [ + { + "task_id": "bad_text_task", + "split": "locked", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": {"substring": ""}, + } + ], + }, + "non-empty string", + ), + ( + { + "manifest_version": "1", + "tasks": [ + { + "task_id": "bad_alias_task", + "split": "locked", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": { + "substring": "Pacific", + "strict_texts": "Pacific Ocean", + }, + } + ], + }, + "non-empty texts list", + ), + ( + { + "manifest_version": "1", + "tasks": [ + { + "task_id": "bad_case_task", + "split": "locked", + "scorer": {"name": "substring_match", "version": "1"}, + "expected": { + "substring": "Au", + "strict_case_sensitive": "yes", + }, + } + ], + }, + "case_sensitive must be boolean", + ), + ( + { + "manifest_version": "1", + "tasks": [ + { + "task_id": "unknown_scorer_task", + "split": "locked", + "scorer": {"name": "rubric_match", "version": "9"}, + "expected": {"rubric": "x"}, + } + ], + }, + "unsupported strict scorer", + ), + ({"manifest_version": "", "tasks": []}, "manifest_version"), + ], +) +def test_strict_manifest_rejects_ambiguous_authoring_contracts( + manifest: object, + message: str, +) -> None: + """Unconvertible authoring contracts must fail before benchmark egress.""" + with pytest.raises(nb.BenchmarkContractError, match=message): + strict.strict_task_manifest_payload(manifest) + + +def test_task_manifest_argument_supports_default_split_and_equals_forms() -> None: + """CLI rewriting preserves unrelated arguments and accepts both path forms.""" + source_path, remaining = strict._task_manifest_argument(["--dry-run"]) + assert source_path == Path("examples/nim_task_manifest.json") + assert remaining == ["--dry-run"] + + source_path, remaining = strict._task_manifest_argument( + ["--dry-run", "--task-manifest", "custom.json", "--seed", "9"] + ) + assert source_path == Path("custom.json") + assert remaining == ["--dry-run", "--seed", "9"] + + source_path, remaining = strict._task_manifest_argument( + ["--task-manifest=equals.json"] + ) + assert source_path == Path("equals.json") + assert remaining == [] + + +@pytest.mark.parametrize( + "argv", + [ + ["--task-manifest"], + ["--task-manifest", "one.json", "--task-manifest", "two.json"], + ["--task-manifest=one.json", "--task-manifest=two.json"], + ["--task-manifest="], + ], +) +def test_task_manifest_argument_rejects_missing_or_duplicate_values( + argv: list[str], +) -> None: + """Ambiguous task-manifest selectors must fail before reading any provider key.""" + with pytest.raises(nb.BenchmarkContractError, match="task-manifest"): + strict._task_manifest_argument(argv) + + +def test_strict_manifest_writer_rejects_invalid_source_and_uses_private_mode( + tmp_path: Path, +) -> None: + """The derived evidence manifest is deterministic, valid, and owner-readable.""" + invalid_source = tmp_path / "invalid.json" + invalid_source.write_text("{broken", encoding="utf-8") + with pytest.raises(nb.BenchmarkContractError, match="valid task manifest"): + strict._write_strict_manifest(invalid_source, tmp_path / "unused.json") + + source = tmp_path / "source.json" + source.write_text(json.dumps(_source_manifest()), encoding="utf-8") + destination = tmp_path / "strict.json" + strict._write_strict_manifest(source, destination) + assert destination.stat().st_mode & 0o777 == 0o600 + derived = json.loads(destination.read_text(encoding="utf-8")) + assert derived["scoring_policy_version"] == strict.STRICT_SCORING_POLICY_VERSION + + with pytest.raises(FileExistsError): + strict._write_strict_manifest(source, destination) + + +def test_strict_cli_wrapper_passes_only_the_derived_manifest( + tmp_path: Path, +) -> None: + """The supported CLI runs with strict provenance and removes the private file.""" + source = tmp_path / "source.json" + source.write_text(json.dumps(_source_manifest()), encoding="utf-8") + observed: dict[str, object] = {} + + def benchmark_cli(argv: list[str]) -> int: + observed["argv"] = list(argv) + manifest_index = argv.index("--task-manifest") + 1 + manifest_path = Path(argv[manifest_index]) + observed["manifest_path"] = manifest_path + payload = json.loads(manifest_path.read_text(encoding="utf-8")) + observed["policy_version"] = payload["scoring_policy_version"] + observed["locked_scorers"] = { + (task["scorer"]["name"], task["scorer"]["version"]) + for task in payload["tasks"] + if task["split"] == "locked" + } + return 17 + + result = strict.run_strict_benchmark_cli( + ["--dry-run", "--task-manifest", str(source), "--seed", "3"], + benchmark_cli=benchmark_cli, + ) + + assert result == 17 + assert observed["policy_version"] == strict.STRICT_SCORING_POLICY_VERSION + assert observed["locked_scorers"] == { + ("exact_number_match", "2"), + ("exact_text_match", "1"), + } + assert observed["argv"][0:3] == ["--dry-run", "--seed", "3"] + manifest_path = observed["manifest_path"] + assert isinstance(manifest_path, Path) + assert not manifest_path.exists() diff --git a/tests/test_nim_strict_scoring_bounds.py b/tests/test_nim_strict_scoring_bounds.py new file mode 100644 index 00000000..664955e8 --- /dev/null +++ b/tests/test_nim_strict_scoring_bounds.py @@ -0,0 +1,66 @@ +"""Resource-bound contracts for strict NIM benchmark answer scoring.""" + +from __future__ import annotations + +import pytest + +from contextual_orchestrator import nim_benchmark as nb +from contextual_orchestrator import nim_strict_scoring as strict + + +def test_strict_answer_character_budget_is_explicit_and_bounded() -> None: + """Keep answer-key and model-output normalization inside one reviewable cap.""" + assert strict.MAX_STRICT_ANSWER_CHARACTERS == 4096 + + +def test_numeric_scorer_rejects_oversized_or_unrepresentable_model_answers() -> None: + """Hostile numeric output must score zero rather than exhaust or abort the run.""" + oversized = "9" * (strict.MAX_STRICT_ANSWER_CHARACTERS + 1) + exponent_overflow = "1e" + "9" * 80 + + assert strict.score_exact_number_match_v2({"number": "9"}, oversized) == 0.0 + assert ( + strict.score_exact_number_match_v2( + {"number": "9"}, + exponent_overflow, + ) + == 0.0 + ) + + +def test_unrepresentable_numeric_answer_key_fails_before_provider_egress() -> None: + """A grammar-valid but unrepresentable exponent is an invalid answer key.""" + exponent_overflow = "1e" + "9" * 80 + + with pytest.raises(nb.BenchmarkContractError, match="finite numeric literal"): + strict.score_exact_number_match_v2({"number": exponent_overflow}, "1") + + +def test_numeric_answer_key_over_budget_fails_before_provider_egress() -> None: + """An oversized expected literal is an invalid manifest, not a failed model cell.""" + oversized = "9" * (strict.MAX_STRICT_ANSWER_CHARACTERS + 1) + + with pytest.raises(nb.BenchmarkContractError, match="character budget"): + strict.score_exact_number_match_v2({"number": oversized}, "9") + + +def test_text_scorer_rejects_oversized_model_answers_without_normalizing_them() -> None: + """Oversized free text must score zero under both case policies.""" + oversized = "A" * (strict.MAX_STRICT_ANSWER_CHARACTERS + 1) + + assert strict.score_exact_text_match({"texts": ["A"]}, oversized) == 0.0 + assert ( + strict.score_exact_text_match( + {"texts": ["A"], "case_sensitive": True}, + oversized, + ) + == 0.0 + ) + + +def test_text_answer_key_over_budget_fails_before_provider_egress() -> None: + """Every declared alias must fit the same strict-scoring character budget.""" + oversized = "A" * (strict.MAX_STRICT_ANSWER_CHARACTERS + 1) + + with pytest.raises(nb.BenchmarkContractError, match="character budget"): + strict.score_exact_text_match({"texts": [oversized]}, "A") diff --git a/tests/test_nim_strict_scoring_integration.py b/tests/test_nim_strict_scoring_integration.py new file mode 100644 index 00000000..6ca10a70 --- /dev/null +++ b/tests/test_nim_strict_scoring_integration.py @@ -0,0 +1,112 @@ +"""End-to-end contracts for the supported strict NIM benchmark command.""" + +from __future__ import annotations + +import io +import json +from pathlib import Path +import subprocess +import sys + +from contextual_orchestrator.nim_csv_evidence import ( + run_benchmark_cli_with_complete_csv, +) +from contextual_orchestrator.nim_strict_scoring import ( + STRICT_SCORING_POLICY_VERSION, + run_strict_benchmark_cli, +) + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] + + +def test_ordinary_package_import_does_not_activate_optional_nim_modules() -> None: + """Gateway consumers must not import or activate benchmark-only adapters.""" + script = """ +import json +import sys + +import contextual_orchestrator + +print(json.dumps({ + "benchmark": "contextual_orchestrator.nim_benchmark" in sys.modules, + "csv_evidence": "contextual_orchestrator.nim_csv_evidence" in sys.modules, + "strict_scoring": "contextual_orchestrator.nim_strict_scoring" in sys.modules, +})) +""" + completed = subprocess.run( + [sys.executable, "-c", script], + cwd=REPOSITORY_ROOT, + check=True, + capture_output=True, + text=True, + ) + + assert json.loads(completed.stdout) == { + "benchmark": False, + "csv_evidence": False, + "strict_scoring": False, + } + + +def test_supported_dry_run_publishes_only_strict_locked_scores( + tmp_path: Path, +) -> None: + """The real supported CLI must bind artifacts to strict scorer versions.""" + output_directory = tmp_path / "strict-artifacts" + stdout = io.StringIO() + + result = run_benchmark_cli_with_complete_csv( + [ + "--dry-run", + "--output-dir", + str(output_directory), + "--max-total-requests", + "2000", + ], + benchmark_cli=run_strict_benchmark_cli, + stdout=stdout, + ) + + assert result == 0, stdout.getvalue() + success = json.loads(stdout.getvalue()) + report_path = Path(success["artifact_paths"]["json_path"]) + csv_path = Path(success["artifact_paths"]["csv_path"]) + markdown_path = Path(success["artifact_paths"]["markdown_path"]) + assert report_path.parent == output_directory + assert csv_path.parent == output_directory + assert markdown_path.parent == output_directory + + report = json.loads(report_path.read_text(encoding="utf-8")) + parameters = report["provenance"]["benchmark_parameters"] + assert parameters["task_manifest_version"].endswith( + "+strict." + STRICT_SCORING_POLICY_VERSION + ) + assert report["provenance"]["task_manifest_sha256"] + assert { + (cell["scorer_name"], cell["scorer_version"]) + for cell in report["evaluation"]["evaluation_cells"] + } == { + ("exact_number_match", "2"), + ("exact_text_match", "1"), + } + assert report["evaluation"]["routing_recommendation"] is None + + +def test_entrypoint_and_permanent_quality_gate_bind_strict_scoring() -> None: + """Static contracts prevent the supported path from bypassing strict scoring.""" + entrypoint = ( + REPOSITORY_ROOT / "contextual_orchestrator" / "__main__.py" + ).read_text(encoding="utf-8") + workflow = ( + REPOSITORY_ROOT / ".github" / "workflows" / "tests.yml" + ).read_text(encoding="utf-8") + + assert "from .nim_strict_scoring import run_strict_benchmark_cli" in entrypoint + assert "benchmark_cli=run_strict_benchmark_cli" in entrypoint + assert "contextual_orchestrator.nim_strict_scoring" in workflow + assert "tests/test_nim_strict_scorer_validity.py" in workflow + assert "tests/test_nim_strict_scoring_bounds.py" in workflow + assert "tests/test_nim_strict_scoring_integration.py" in workflow + assert "tests/test_nim_strict_scoring_leakage.py" in workflow + assert "interrogate -f 100 contextual_orchestrator/nim_strict_scoring.py" in workflow diff --git a/tests/test_nim_strict_scoring_leakage.py b/tests/test_nim_strict_scoring_leakage.py new file mode 100644 index 00000000..528d6c2e --- /dev/null +++ b/tests/test_nim_strict_scoring_leakage.py @@ -0,0 +1,174 @@ +"""No-leakage contracts for derived strict NIM benchmark manifests.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from contextual_orchestrator import nim_benchmark as nb +from contextual_orchestrator import nim_strict_scoring as strict + + +def _locked_manifest( + *, + prompt: object, + scorer_name: str, + expected: dict[str, object], +) -> dict[str, Any]: + """Build one minimal locked authoring manifest for leakage validation.""" + return { + "manifest_version": "strict-leakage-test.1", + "tasks": [ + { + "task_id": "strict_leakage_task", + "split": "locked", + "prompt": prompt, + "scorer": {"name": scorer_name, "version": "1"}, + "expected": expected, + } + ], + } + + +@pytest.mark.parametrize( + ("prompt", "expected"), + [ + ("Hint: the result is 21.0. Return a number only.", {"number": "21"}), + ("Reject 0.050 and solve independently.", {"number": "0.05"}), + ], +) +def test_strict_numeric_manifest_rejects_equivalent_answer_leakage( + prompt: str, + expected: dict[str, object], +) -> None: + """Equivalent decimal literals embedded in prompts must fail before egress.""" + manifest = _locked_manifest( + prompt=prompt, + scorer_name="exact_number_match", + expected=expected, + ) + + with pytest.raises(nb.BenchmarkContractError, match="leaks its expected answer"): + strict.strict_task_manifest_payload(manifest) + + +def test_strict_numeric_leakage_uses_complete_numeric_tokens() -> None: + """The answer 21 must not be inferred from the unrelated larger number 121.""" + manifest = _locked_manifest( + prompt="Return the requested value, not 121.", + scorer_name="exact_number_match", + expected={"number": "21"}, + ) + + derived = strict.strict_task_manifest_payload(manifest) + assert derived["tasks"][0]["scorer"] == { + "name": "exact_number_match", + "version": "2", + } + + +def test_invalid_numeric_prompt_token_does_not_abort_leakage_review() -> None: + """An unrepresentable unrelated exponent must not escape the manifest guard.""" + manifest = _locked_manifest( + prompt="Ignore the malformed value 1e" + "9" * 80 + ".", + scorer_name="exact_number_match", + expected={"number": "7"}, + ) + + derived = strict.strict_task_manifest_payload(manifest) + assert derived["tasks"][0]["expected"] == {"number": "7"} + + +def test_strict_text_manifest_rejects_declared_alias_leakage() -> None: + """Any declared complete-answer alias embedded in a prompt must be rejected.""" + manifest = _locked_manifest( + prompt="Choose between the Atlantic and PACIFIC OCEAN, then answer only.", + scorer_name="substring_match", + expected={ + "substring": "Pacific", + "strict_texts": ["Pacific", "Pacific Ocean"], + }, + ) + + with pytest.raises(nb.BenchmarkContractError, match="leaks its expected answer"): + strict.strict_task_manifest_payload(manifest) + + +def test_case_sensitive_text_leakage_preserves_declared_case() -> None: + """A lower-case token is not the case-sensitive chemical-symbol answer key.""" + safe_manifest = _locked_manifest( + prompt="The letters au appear in an unrelated lower-case label.", + scorer_name="substring_match", + expected={"substring": "Au", "strict_case_sensitive": True}, + ) + strict.strict_task_manifest_payload(safe_manifest) + + leaking_manifest = _locked_manifest( + prompt="Do not simply copy Au from this instruction.", + scorer_name="substring_match", + expected={"substring": "Au", "strict_case_sensitive": True}, + ) + with pytest.raises(nb.BenchmarkContractError, match="leaks its expected answer"): + strict.strict_task_manifest_payload(leaking_manifest) + + +def test_text_leakage_does_not_match_inside_a_larger_word() -> None: + """The symbol Au must not be treated as leaked by the word Australia.""" + manifest = _locked_manifest( + prompt="Australia is unrelated to the requested symbol.", + scorer_name="substring_match", + expected={"substring": "Au", "strict_case_sensitive": True}, + ) + + derived = strict.strict_task_manifest_payload(manifest) + assert derived["tasks"][0]["expected"] == { + "texts": ["Au"], + "case_sensitive": True, + } + + +def test_nonword_alias_boundaries_are_checked_without_word_lookarounds() -> None: + """Punctuation-delimited aliases must still be detected as prompt leakage.""" + manifest = _locked_manifest( + prompt="Do not copy (ok) from this instruction.", + scorer_name="substring_match", + expected={"substring": "(ok)"}, + ) + + with pytest.raises(nb.BenchmarkContractError, match="leaks its expected answer"): + strict.strict_task_manifest_payload(manifest) + + +@pytest.mark.parametrize( + ("prompt", "message"), + [ + (None, "non-empty locked task prompt"), + ("x" * (strict.MAX_STRICT_ANSWER_CHARACTERS + 1), "character budget"), + ], +) +def test_invalid_or_oversized_locked_prompts_fail_before_egress( + prompt: object, + message: str, +) -> None: + """Leakage review must not accept absent or unbounded prompt inputs.""" + manifest = _locked_manifest( + prompt=prompt, + scorer_name="exact_number_match", + expected={"number": "7"}, + ) + + with pytest.raises(nb.BenchmarkContractError, match=message): + strict.strict_task_manifest_payload(manifest) + + +def test_strict_task_leakage_rejects_unknown_direct_scorer_identity() -> None: + """The private leakage dispatcher must fail closed on unowned identities.""" + task = { + "prompt": "No answer is present.", + "scorer": {"name": "unknown_match", "version": "9"}, + "expected": {}, + } + + with pytest.raises(nb.BenchmarkContractError, match="unsupported strict scorer"): + strict._strict_task_leaks_expected(task) diff --git a/tests/test_nim_task_manifest_evidence_floor.py b/tests/test_nim_task_manifest_evidence_floor.py new file mode 100644 index 00000000..c57ee638 --- /dev/null +++ b/tests/test_nim_task_manifest_evidence_floor.py @@ -0,0 +1,66 @@ +"""Release acceptance for the locked NIM evaluation manifest and request ceiling.""" + +from __future__ import annotations + +import inspect +from pathlib import Path + +from contextual_orchestrator import nim_benchmark as nb + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +TASK_MANIFEST_PATH = REPOSITORY_ROOT / "examples" / "nim_task_manifest.json" +BENCHMARK_WORKFLOW_PATH = REPOSITORY_ROOT / ".github" / "workflows" / "nim-benchmark.yml" +BENCHMARK_GUIDE_PATH = REPOSITORY_ROOT / "docs" / "nim_benchmark.md" +BENCHMARK_SOURCE_PATH = REPOSITORY_ROOT / "contextual_orchestrator" / "nim_benchmark.py" + + +def test_locked_manifest_reaches_the_declared_paired_evidence_floor() -> None: + """Keep the scheduled benchmark capable of producing non-smoke paired evidence.""" + manifest = nb.load_task_manifest(str(TASK_MANIFEST_PATH)) + locked_tasks = nb.locked_evaluation_tasks(manifest) + + assert len(locked_tasks) == nb.MINIMUM_PAIRED_TASK_COUNT == 30 + assert len({task["task_id"] for task in locked_tasks}) == 30 + + +def test_scheduled_ceiling_reserves_a_complete_127_model_run() -> None: + """Cover every probe and thirty-task policy cell within one bounded live run.""" + manifest = nb.load_task_manifest(str(TASK_MANIFEST_PATH)) + locked_task_count = len(nb.locked_evaluation_tasks(manifest)) + plan = nb.plan_complete_request_budget( + discovered_model_count=127, + locked_task_count=locked_task_count, + max_eval_models=7, + ) + workflow = BENCHMARK_WORKFLOW_PATH.read_text(encoding="utf-8") + guide = BENCHMARK_GUIDE_PATH.read_text(encoding="utf-8") + + assert plan == { + "catalog_request_count": 1, + "capability_probe_request_count": 1143, + "evaluation_reserve_request_count": 420, + "planned_worker_count": 7, + "total_required_request_count": 1564, + } + assert 'echo "max_requests=2000"' in workflow + assert "--max-total-requests 2000" in guide + + +def test_default_request_caps_can_run_the_bundled_thirty_task_manifest() -> None: + """Keep API, CLI, and manual workflow defaults above the dry-run plan.""" + workflow = BENCHMARK_WORKFLOW_PATH.read_text(encoding="utf-8") + source = BENCHMARK_SOURCE_PATH.read_text(encoding="utf-8") + default_request_cap = inspect.signature(nb.run_benchmark).parameters[ + "max_total_requests" + ].default + dry_plan = nb.plan_complete_request_budget( + discovered_model_count=len(nb._DRY_RUN_MODEL_BEHAVIOR), + locked_task_count=nb.MINIMUM_PAIRED_TASK_COUNT, + max_eval_models=7, + ) + + assert dry_plan["total_required_request_count"] == 529 + assert default_request_cap == 2000 + assert 'parser.add_argument("--max-total-requests", type=int, default=2000)' in source + assert 'default: 2000' in workflow diff --git a/tests/test_pr_exact_head_workflows.py b/tests/test_pr_exact_head_workflows.py new file mode 100644 index 00000000..7b7fbd2a --- /dev/null +++ b/tests/test_pr_exact_head_workflows.py @@ -0,0 +1,30 @@ +"""Contracts that prevent local pull-request workflows from testing stale or synthetic heads.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +EXACT_HEAD_REF = ( + "ref: ${{ github.event_name == 'pull_request' " + "&& github.event.pull_request.head.sha || github.sha }}" +) +WORKFLOW_PATHS = ( + Path(".github/workflows/tests.yml"), + Path(".github/workflows/fuzz.yml"), + Path(".github/workflows/security.yml"), +) + + +@pytest.mark.parametrize("relative_path", WORKFLOW_PATHS) +def test_pull_request_workflows_cover_stacked_exact_heads(relative_path: Path) -> None: + """Require all PR bases to run while every checkout selects the contributor head.""" + workflow = (REPOSITORY_ROOT / relative_path).read_text(encoding="utf-8") + checkout_count = workflow.count("uses: actions/checkout@") + assert checkout_count > 0 + assert "pull_request:\n branches: [main]" not in workflow + assert workflow.count(EXACT_HEAD_REF) == checkout_count + assert workflow.count("persist-credentials: false") >= checkout_count diff --git a/tests/test_pr_workflow_oidc_boundary.py b/tests/test_pr_workflow_oidc_boundary.py new file mode 100644 index 00000000..47ac8d6a --- /dev/null +++ b/tests/test_pr_workflow_oidc_boundary.py @@ -0,0 +1,107 @@ +"""Regression tests for the pull-request workflow credential boundary. + +Pull-request heads are untrusted input. A workflow may inspect or test that code, +but it must not give the same job an OpenID Connect token that can be exchanged +for repository-writing credentials. Publication belongs to a separately trusted +workflow whose executable source is not selected by the pull-request branch. +""" + +from pathlib import Path +import re + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +WORKFLOW_DIRECTORY = REPOSITORY_ROOT / ".github" / "workflows" +TEMPORARY_REPAIR_WORKFLOW_PATHS = ( + WORKFLOW_DIRECTORY / "nim-source-repair.yml", + WORKFLOW_DIRECTORY / "nim-source-repair-trigger.yml", +) +_YAML_KEY_TEMPLATE = r"(?:{plain}|'(?:{plain})'|\"(?:{plain})\")" +_ID_TOKEN_WRITE_RE = re.compile( + r"^\s*id-token\s*:\s*['\"]?write['\"]?\s*(?:#.*)?$", + re.MULTILINE, +) + + +def _mapping_entry( + lines: list[str], + key: str, + indent: int, +) -> tuple[str, list[str]] | None: + """Return one indentation-bounded YAML mapping entry. + + This focused scanner intentionally recognizes only mapping structure needed + by the workflow contracts. It avoids dependency on a permissive YAML loader + while still making assertions independent of exact whitespace and sibling + ordering. + """ + key_pattern = _YAML_KEY_TEMPLATE.format(plain=re.escape(key)) + pattern = re.compile( + rf"^ {{{indent}}}{key_pattern}\s*:\s*(?P[^#]*?)(?:\s+#.*)?$" + ) + matches = [ + (index, match) + for index, line in enumerate(lines) + if (match := pattern.match(line)) is not None + ] + if not matches: + return None + assert len(matches) == 1, f"duplicate YAML mapping key {key!r}" + start_index, match = matches[0] + body: list[str] = [] + for line in lines[start_index + 1 :]: + stripped = line.strip() + if stripped and not stripped.startswith("#"): + current_indent = len(line) - len(line.lstrip(" ")) + if current_indent <= indent: + break + body.append(line) + return match.group("inline").strip(), body + + +def _direct_child_indent(lines: list[str]) -> int | None: + """Return the indentation of direct child entries in one YAML mapping body.""" + indents = [ + len(line) - len(line.lstrip(" ")) + for line in lines + if line.strip() and not line.lstrip().startswith("#") + ] + return min(indents) if indents else None + + +def _workflow_triggers_pull_request(workflow_text: str) -> bool: + """Return whether one workflow structurally declares a pull-request trigger.""" + on_entry = _mapping_entry(workflow_text.splitlines(), "on", 0) + assert on_entry is not None, "workflow must declare a top-level 'on' key" + inline_value, body = on_entry + if inline_value: + return re.search( + r"(? None: + """Keep every pull-request workflow read-only and unable to mint write tokens.""" + triggered_workflows: list[tuple[Path, str]] = [] + for workflow_path in sorted(WORKFLOW_DIRECTORY.glob("*.y*ml")): + workflow_text = workflow_path.read_text(encoding="utf-8") + if _workflow_triggers_pull_request(workflow_text): + triggered_workflows.append((workflow_path, workflow_text)) + + assert triggered_workflows, "at least one pull-request workflow must be inspected" + for workflow_path, workflow_text in triggered_workflows: + assert not _ID_TOKEN_WRITE_RE.search(workflow_text), workflow_path + assert "ACTIONS_ID_TOKEN_REQUEST_TOKEN" not in workflow_text, workflow_path + assert "exchange_github_app_token" not in workflow_text, workflow_path + assert "git push origin" not in workflow_text, workflow_path + + assert all( + not workflow_path.exists() + for workflow_path in TEMPORARY_REPAIR_WORKFLOW_PATHS + ) diff --git a/tests/test_repository_security_metadata.py b/tests/test_repository_security_metadata.py index b1c9ee78..9d4983ff 100644 --- a/tests/test_repository_security_metadata.py +++ b/tests/test_repository_security_metadata.py @@ -3,12 +3,71 @@ ROOT_DIR = Path(__file__).resolve().parents[1] +_YAML_KEY_TEMPLATE = r"(?:{plain}|'(?:{plain})'|\"(?:{plain})\")" def read_text(relative_path: str) -> str: return (ROOT_DIR / relative_path).read_text(encoding="utf-8") +def _mapping_entry( + lines: list[str], + key: str, + indent: int, +) -> tuple[str, list[str]] | None: + """Return one indentation-bounded YAML mapping entry.""" + key_pattern = _YAML_KEY_TEMPLATE.format(plain=re.escape(key)) + pattern = re.compile( + rf"^ {{{indent}}}{key_pattern}\s*:\s*(?P[^#]*?)(?:\s+#.*)?$" + ) + matches = [ + (index, match) + for index, line in enumerate(lines) + if (match := pattern.match(line)) is not None + ] + if not matches: + return None + assert len(matches) == 1, f"duplicate YAML mapping key {key!r}" + start_index, match = matches[0] + body: list[str] = [] + for line in lines[start_index + 1 :]: + stripped = line.strip() + if stripped and not stripped.startswith("#"): + current_indent = len(line) - len(line.lstrip(" ")) + if current_indent <= indent: + break + body.append(line) + return match.group("inline").strip(), body + + +def _direct_child_indent(lines: list[str]) -> int | None: + """Return the indentation of direct child entries in one YAML mapping body.""" + indents = [ + len(line) - len(line.lstrip(" ")) + for line in lines + if line.strip() and not line.lstrip().startswith("#") + ] + return min(indents) if indents else None + + +def _pull_request_trigger_entry(workflow_text: str) -> tuple[str, list[str]]: + """Return the structurally parsed pull-request trigger entry.""" + workflow_lines = workflow_text.splitlines() + on_entry = _mapping_entry(workflow_lines, "on", 0) + assert on_entry is not None, "workflow must declare a top-level 'on' mapping" + inline_on, on_body = on_entry + assert not inline_on, "required workflows must use a mapping-style 'on' block" + trigger_indent = _direct_child_indent(on_body) + assert trigger_indent is not None, "workflow 'on' mapping must not be empty" + pull_request_entry = _mapping_entry( + on_body, + "pull_request", + trigger_indent, + ) + assert pull_request_entry is not None, "workflow must trigger on pull_request" + return pull_request_entry + + def test_readme_links_deepwiki_and_security_workflow_badges(): readme_text = read_text("README.md") @@ -29,7 +88,7 @@ def test_security_workflow_covers_core_repository_security_process(): expected_tokens = [ "name: Security", - "branches: [main]", + "push:\n branches: [main]", "cron:", "workflow_dispatch:", "contents: read", @@ -68,6 +127,28 @@ def test_security_workflow_covers_core_repository_security_process(): assert all(re.search(r"@[0-9a-f]{40}(?:\s+#|$)", line) for line in uses_lines) +def test_required_pull_request_workflows_cover_stacked_bases(): + """Run required local gates for stacked PRs, not only PRs targeting main.""" + for workflow_path in ( + ".github/workflows/tests.yml", + ".github/workflows/fuzz.yml", + ".github/workflows/security.yml", + ): + workflow_text = read_text(workflow_path) + inline_value, pull_request_body = _pull_request_trigger_entry(workflow_text) + assert not re.search(r"(?