From e28ca1c785dee66b01b6f9b16bad33c7694f67d3 Mon Sep 17 00:00:00 2001
From: coreytshaffer <78175888+coreytshaffer@users.noreply.github.com>
Date: Tue, 16 Jun 2026 20:58:12 -0700
Subject: [PATCH] add model manifest runtime warning report

---
 docs/change/change_log.md                     |   1 +
 ...036-model-manifest-runtime-warning-gate.md |  50 ++++++++
 tests/test_model_manifest.py                  |  95 ++++++++++++++++
 triage_core/model_manifest.py                 | 107 ++++++++++++++++++
 4 files changed, 253 insertions(+)
 create mode 100644 docs/change/requests/CR-036-model-manifest-runtime-warning-gate.md

diff --git a/docs/change/change_log.md b/docs/change/change_log.md
index 8abdf76..a0f474e 100644
--- a/docs/change/change_log.md
+++ b/docs/change/change_log.md
@@ -5,6 +5,7 @@ This file provides a chronological, human-readable record of applied codebase an
 *Note: For operational task and run history, consult `.triagecore/ledger.jsonl`.*
 
 ## [Unreleased]
+- Implemented CR-036 (Model Manifest Runtime Warning Gate): Add a pure metadata-only route-to-manifest comparison report that warns on backend/model/route, alias-only identity, and incomplete integrity mismatches without blocking runtime execution.
 - Implemented CR-035 (Backlog and Status Alignment Pass): Update current backlog and README status markers after CR-034, keeping runtime integrity enforcement and identity signing expansion out of scope.
 - Implemented CR-034 (Repository Consistency and Secrets Hygiene): Align Python metadata with supported syntax, consolidate package metadata in `pyproject.toml`, expand CI's Python matrix, require environment-only Qwen API keys, reject secret-bearing persistent keys, sanitize backend HTTP errors, and add `SECURITY.md`.
 - Implemented CR-033 (Model Manifest Check CLI): Add `tc model check --manifest <path>` to validate documented route-manifest fields without probing live backends or enforcing routing yet.
diff --git a/docs/change/requests/CR-036-model-manifest-runtime-warning-gate.md b/docs/change/requests/CR-036-model-manifest-runtime-warning-gate.md
new file mode 100644
index 0000000..a34a9e8
--- /dev/null
+++ b/docs/change/requests/CR-036-model-manifest-runtime-warning-gate.md
@@ -0,0 +1,50 @@
+# CR-036: Model Manifest Runtime Warning Gate
+
+## Status
+
+Implemented
+
+## Scope
+
+Add a warning-only comparison layer between route metadata and a validated model
+route manifest.
+
+This change:
+
+- adds a pure `compare_route_to_manifest(route_payload, manifest)` function
+- returns a metadata-only `ManifestRouteWarningReport`
+- warns when selected backend, selected model, or selected route metadata does
+  not match the manifest
+- warns when the manifest uses alias-only model identity
+- warns when manifest integrity status is incomplete
+- adds focused unit tests
+
+## Non-Scope
+
+- Do not block routing.
+- Do not wire this into runtime execution yet.
+- Do not probe Ollama, LM Studio, Qwen Cloud, local files, or model artifacts.
+- Do not hash model files.
+- Do not expand signing.
+- Do not mutate the ledger schema.
+
+## Acceptance Criteria
+
+- [x] Add a pure function that compares route metadata to a validated manifest.
+- [x] Matching route/backend/model metadata produces no warnings.
+- [x] Backend mismatch produces a warning.
+- [x] Model mismatch or alias-only identity produces a warning.
+- [x] Incomplete manifest integrity status produces a warning.
+- [x] Warning report contains only metadata, no raw prompt/data.
+- [x] Add focused unit tests.
+- [x] No runtime blocking behavior.
+
+## Validation
+
+```powershell
+python -m py_compile triage_core\model_manifest.py
+python -m pytest tests\test_model_manifest.py -q
+python -m pytest -q
+git diff --check
+git status --short
+```
\ No newline at end of file
diff --git a/tests/test_model_manifest.py b/tests/test_model_manifest.py
index 40b5ae2..f53227e 100644
--- a/tests/test_model_manifest.py
+++ b/tests/test_model_manifest.py
@@ -1,6 +1,7 @@
 from pathlib import Path
 
 from triage_core.model_manifest import (
+    compare_route_to_manifest,
     load_model_manifest,
     validate_model_manifest,
 )
@@ -58,3 +59,97 @@ def test_manifest_missing_required_field_fails():
 
     assert result.is_valid is False
     assert any(issue.reason == "missing_required_field" for issue in result.issues)
+
+
+def test_matching_route_manifest_metadata_has_no_warnings():
+    manifest = load_model_manifest(_example_path("model_route_manifest_local_ollama.json"))
+    route_payload = {
+        "selected_backend": "ollama",
+        "selected_model": "qwen2.5:7b-instruct-q4_K_M",
+        "selected_route": "local-ollama-qwen2.5-7b-instruct-q4km",
+    }
+
+    report = compare_route_to_manifest(route_payload, manifest)
+
+    assert report.has_warnings is False
+    assert report.warnings == []
+
+
+def test_backend_mismatch_produces_warning():
+    manifest = load_model_manifest(_example_path("model_route_manifest_local_ollama.json"))
+    route_payload = {
+        "selected_backend": "qwen_cloud",
+        "selected_model": "qwen2.5:7b-instruct-q4_K_M",
+        "selected_route": "local-ollama-qwen2.5-7b-instruct-q4km",
+    }
+
+    report = compare_route_to_manifest(route_payload, manifest)
+
+    assert {warning.reason for warning in report.warnings} == {"backend_mismatch"}
+
+
+def test_model_mismatch_produces_warning():
+    manifest = load_model_manifest(_example_path("model_route_manifest_local_ollama.json"))
+    route_payload = {
+        "selected_backend": "ollama",
+        "selected_model": "different-model",
+        "selected_route": "local-ollama-qwen2.5-7b-instruct-q4km",
+    }
+
+    report = compare_route_to_manifest(route_payload, manifest)
+
+    assert {warning.reason for warning in report.warnings} == {"model_mismatch"}
+
+
+def test_alias_only_identity_produces_warning():
+    manifest = load_model_manifest(
+        _example_path("model_route_manifest_invalid_alias_only.json")
+    )
+    route_payload = {
+        "selected_backend": "ollama",
+        "selected_model": "latest",
+        "selected_route": "alias-only-fast-route",
+    }
+
+    report = compare_route_to_manifest(route_payload, manifest)
+
+    reasons = {warning.reason for warning in report.warnings}
+    assert "alias_only_model_identity" in reasons
+
+
+def test_incomplete_manifest_integrity_produces_warning():
+    manifest = load_model_manifest(
+        _example_path("model_route_manifest_invalid_alias_only.json")
+    )
+    route_payload = {
+        "selected_backend": "ollama",
+        "selected_model": "latest",
+        "selected_route": "alias-only-fast-route",
+    }
+
+    report = compare_route_to_manifest(route_payload, manifest)
+
+    reasons = {warning.reason for warning in report.warnings}
+    assert "incomplete_integrity_status" in reasons
+
+
+def test_warning_report_contains_only_metadata_without_raw_payload_echo():
+    manifest = load_model_manifest(_example_path("model_route_manifest_local_ollama.json"))
+    route_payload = {
+        "selected_backend": "qwen_cloud",
+        "selected_model": "private-model-alias",
+        "selected_route": "private-route-alias",
+        "prompt": "raw private prompt",
+        "data": "raw private data",
+    }
+
+    report = compare_route_to_manifest(route_payload, manifest)
+    rendered = "\n".join(
+        f"{warning.reason} {warning.path} {warning.message}"
+        for warning in report.warnings
+    )
+
+    assert "raw private prompt" not in rendered
+    assert "raw private data" not in rendered
+    assert "private-model-alias" not in rendered
+    assert "private-route-alias" not in rendered
diff --git a/triage_core/model_manifest.py b/triage_core/model_manifest.py
index fc9205c..0e9c8a4 100644
--- a/triage_core/model_manifest.py
+++ b/triage_core/model_manifest.py
@@ -49,6 +49,27 @@ def add_issue(self, reason: str, path: str, message: str) -> None:
             self.issues.append(issue)
 
 
+@dataclass(frozen=True)
+class ManifestRouteWarning:
+    reason: str
+    path: str
+    message: str
+
+
+@dataclass
+class ManifestRouteWarningReport:
+    warnings: list[ManifestRouteWarning] = field(default_factory=list)
+
+    @property
+    def has_warnings(self) -> bool:
+        return bool(self.warnings)
+
+    def add_warning(self, reason: str, path: str, message: str) -> None:
+        warning = ManifestRouteWarning(reason=reason, path=path, message=message)
+        if warning not in self.warnings:
+            self.warnings.append(warning)
+
+
 def load_model_manifest(manifest_path: str | Path) -> dict[str, Any]:
     path = Path(manifest_path)
     return json.loads(path.read_text(encoding="utf-8"))
@@ -126,6 +147,84 @@ def validate_model_manifest(manifest: dict[str, Any]) -> ManifestCheckResult:
     return result
 
 
+def compare_route_to_manifest(
+    route_payload: dict[str, Any],
+    manifest: dict[str, Any],
+) -> ManifestRouteWarningReport:
+    report = ManifestRouteWarningReport()
+
+    selected_backend = _route_text(
+        route_payload,
+        "selected_backend",
+        "backend_type",
+        "backend",
+    )
+    selected_model = _route_text(
+        route_payload,
+        "selected_model",
+        "model",
+        "model_id",
+        "exact_model_id",
+    )
+    selected_route = _route_text(
+        route_payload,
+        "selected_route",
+        "recommended_route",
+        "route_id",
+        "requested_route",
+    )
+
+    manifest_backend = _string_at_path(manifest, "$.backend.backend_type")
+    manifest_model = _string_at_path(manifest, "$.model.exact_model_id")
+    manifest_route = _string_at_path(manifest, "$.route_id")
+    manifest_mutable_reference = _bool_at_path(manifest, "$.model.mutable_reference")
+    manifest_integrity_status = _string_at_path(
+        manifest,
+        "$.integrity.integrity_status",
+    )
+    manifest_provenance_complete = _bool_at_path(
+        manifest,
+        "$.integrity.provenance_complete",
+    )
+
+    if selected_backend and manifest_backend and selected_backend != manifest_backend:
+        report.add_warning(
+            "backend_mismatch",
+            "$.backend.backend_type",
+            "Route selected backend does not match manifest backend_type.",
+        )
+
+    if selected_model and manifest_model and selected_model != manifest_model:
+        report.add_warning(
+            "model_mismatch",
+            "$.model.exact_model_id",
+            "Route selected model does not match manifest exact_model_id.",
+        )
+
+    if manifest_mutable_reference and manifest_model.lower() in ALIASED_MODEL_IDENTITIES:
+        report.add_warning(
+            "alias_only_model_identity",
+            "$.model.exact_model_id",
+            "Manifest model identity is an alias-only mutable reference.",
+        )
+
+    if manifest_integrity_status != "complete" or not manifest_provenance_complete:
+        report.add_warning(
+            "incomplete_integrity_status",
+            "$.integrity.integrity_status",
+            "Manifest integrity status is not complete.",
+        )
+
+    if selected_route and manifest_route and selected_route != manifest_route:
+        report.add_warning(
+            "route_mismatch",
+            "$.route_id",
+            "Route id does not match manifest route_id.",
+        )
+
+    return report
+
+
 def summarize_model_manifest_check(
     manifest_path: str | Path,
     manifest: dict[str, Any],
@@ -174,3 +273,11 @@ def _bool_at_path(payload: dict[str, Any], path: str) -> bool:
     if not found:
         return False
     return bool(value)
+
+
+def _route_text(payload: dict[str, Any], *keys: str) -> str:
+    for key in keys:
+        value = payload.get(key)
+        if value is not None:
+            return str(value)
+    return ""