diff --git a/scripts/tests/test_validate_index_coverage.py b/scripts/tests/test_validate_index_coverage.py new file mode 100644 index 00000000..91614f7a --- /dev/null +++ b/scripts/tests/test_validate_index_coverage.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +""" +Tests for check_skill_coverage in scripts/validate-index-integrity.py (Track M, M4). + +Contract under test: the disk -> index direction. Check 1 already walks +index -> disk and catches entries whose file vanished. Check 3 catches the +opposite and quieter failure: a SKILL.md that exists, parses, and is healthy +but was never registered, so the router cannot reach it. + +Skills carrying ``promoted_to`` are husks folded into an umbrella. The +generator skips them on purpose so a folded skill cannot shadow the umbrella +that replaced it, so the check must skip them too or it would demand entries +the generator will never emit. + +Run with: python3 -m pytest scripts/tests/test_validate_index_coverage.py -v +""" + +import importlib.util +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parent.parent / "validate-index-integrity.py" + +_spec = importlib.util.spec_from_file_location("validate_index_integrity", SCRIPT) +assert _spec is not None and _spec.loader is not None +_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_mod) + +check_skill_coverage = _mod.check_skill_coverage + +FRONTMATTER = '---\nname: {name}\ndescription: "fixture"\n---\nbody\n' +FRONTMATTER_PROMOTED = '---\nname: {name}\npromoted_to: {target}\ndescription: "fixture"\n---\nbody\n' + + +def _make_skill(repo_root: Path, category: str, name: str, content: str) -> None: + skill_dir = repo_root / "skills" / category / name + skill_dir.mkdir(parents=True, exist_ok=True) + (skill_dir / "SKILL.md").write_text(content, encoding="utf-8") + + +def test_indexed_skill_passes(tmp_path: Path) -> None: + """A skill present on disk and in the index produces no error.""" + _make_skill(tmp_path, "research", "mapped", FRONTMATTER.format(name="mapped")) + index = {"skills": {"mapped": {"file": "skills/research/mapped/SKILL.md"}}} + + errors, _ = check_skill_coverage(index, tmp_path) + + assert errors == [] + + +def test_unindexed_skill_is_an_error(tmp_path: Path) -> None: + """A healthy SKILL.md with no index entry is unroutable and must fail.""" + _make_skill(tmp_path, "research", "orphan", FRONTMATTER.format(name="orphan")) + index: dict = {"skills": {}} + + errors, _ = check_skill_coverage(index, tmp_path) + + assert len(errors) == 1 + assert "orphan" in errors[0] + assert "no INDEX entry" in errors[0] + # The message must name the fix, not just the fault. + assert "generate-skill-index.py" in errors[0] + + +def test_promoted_husk_is_exempt(tmp_path: Path) -> None: + """A husk folded into an umbrella is skipped, matching the generator.""" + _make_skill( + tmp_path, + "research", + "husk", + FRONTMATTER_PROMOTED.format(name="husk", target="codebase-overview"), + ) + index: dict = {"skills": {}} + + errors, _ = check_skill_coverage(index, tmp_path) + + assert errors == [] + + +def test_reports_every_unindexed_skill(tmp_path: Path) -> None: + """All gaps are reported at once, and promoted husks stay excluded.""" + _make_skill(tmp_path, "research", "gap-a", FRONTMATTER.format(name="gap-a")) + _make_skill(tmp_path, "meta", "gap-b", FRONTMATTER.format(name="gap-b")) + _make_skill( + tmp_path, + "meta", + "folded", + FRONTMATTER_PROMOTED.format(name="folded", target="workflow"), + ) + index: dict = {"skills": {}} + + errors, _ = check_skill_coverage(index, tmp_path) + + reported = {name for name in ("gap-a", "gap-b", "folded") if any(name in e for e in errors)} + assert reported == {"gap-a", "gap-b"} + + +def test_no_skills_dir_is_not_an_error(tmp_path: Path) -> None: + """An absent skills/ tree yields no findings rather than a crash.""" + errors, warnings = check_skill_coverage({"skills": {}}, tmp_path) + + assert errors == [] + assert warnings == [] + + +def test_real_repo_has_full_coverage() -> None: + """The live tree must stay fully indexed; this is the anti-rot assertion.""" + repo_root = SCRIPT.parent.parent + index_path = repo_root / "skills" / "INDEX.json" + if not index_path.is_file(): + # INDEX.json is generated and gitignored; skip when it has not been built. + return + index = _mod.merge_skill_indexes(index_path) + + errors, _ = check_skill_coverage(index, repo_root) + + assert errors == [], "on-disk skills missing from INDEX.json:\n" + "\n".join(errors) diff --git a/scripts/validate-index-integrity.py b/scripts/validate-index-integrity.py index 1b6b126a..45358efc 100755 --- a/scripts/validate-index-integrity.py +++ b/scripts/validate-index-integrity.py @@ -8,9 +8,12 @@ deployed symlink root (~/.claude/skills/) or the private-skills directory. 2. All agent ``file`` fields in agents/INDEX.json point to existing files (paths resolved relative to the repo root). - 3. No skill or agent has fewer than 5 triggers (warn) or 0 triggers (error). - 4. No triggers are duplicated within a single entry. - 5. No triggers are duplicated across entries (cross-entry overlap warning). + 3. Every routable SKILL.md on disk has an INDEX entry (the reverse of check + 1). Skills carrying ``promoted_to`` are excluded — those are husks the + generator skips on purpose so a folded skill cannot shadow its umbrella. + 4. No skill or agent has fewer than 5 triggers (warn) or 0 triggers (error). + 5. No triggers are duplicated within a single entry. + 6. No triggers are duplicated across entries (cross-entry overlap warning). Note: routing-tables.md coverage check was removed in PR #653 — routing-tables.md was absorbed into INDEX.json (PR #626) and check-routing-drift.py now covers this in CI. @@ -24,6 +27,7 @@ """ import json +import re import sys from pathlib import Path @@ -156,6 +160,46 @@ def check_skill_files( return errors, warnings +def check_skill_coverage(skills_index: dict, repo_root: Path) -> tuple[list[str], list[str]]: + """Check 3: every routable SKILL.md on disk has an INDEX entry. + + Check 1 walks index -> disk and catches entries pointing at files that no + longer exist. This walks disk -> index, the direction that catches silent + rot: a skill added to skills/ that no generator run ever registered is + invisible to the router even though its file is present and healthy. + + Skills carrying ``promoted_to`` are excluded. Those are husks whose content + was folded into an umbrella skill; the generator deliberately skips them so + a folded skill cannot re-enter routing and shadow the umbrella that + replaced it. Excluding them here keeps this check agreeing with the + generator instead of demanding entries the generator will never emit. + """ + errors: list[str] = [] + warnings: list[str] = [] + + indexed = set(skills_index.get("skills", {})) + skills_dir = repo_root / "skills" + + for skill_md in sorted(skills_dir.glob("*/*/SKILL.md")): + name = skill_md.parent.name + if name in indexed: + continue + try: + head = skill_md.read_text(encoding="utf-8")[:2000] + except (OSError, UnicodeDecodeError): + continue + # Husks folded into an umbrella are skipped by the generator by design. + if re.search(r"^promoted_to:", head, re.MULTILINE): + continue + rel = skill_md.relative_to(repo_root) + errors.append( + f" [skill not indexed] '{name}': {rel} exists but has no INDEX entry — " + f"it is unroutable. Run: python3 scripts/generate-skill-index.py" + ) + + return errors, warnings + + def check_agent_files(agents_index: dict, repo_root: Path) -> tuple[list[str], list[str]]: """Check 2: every agent file field points to an existing file.""" errors: list[str] = [] @@ -281,6 +325,7 @@ def main() -> int: checks = [ ("Check 1: skill files on disk", check_skill_files(skills_index, repo_root, overlay_roots)), ("Check 2: agent files on disk", check_agent_files(agents_index, repo_root)), + ("Check 3: skill index coverage", check_skill_coverage(skills_index, repo_root)), ("Check 4a: skill trigger counts", check_trigger_counts(skills_index, "skills")), ("Check 4b: agent trigger counts", check_trigger_counts(agents_index, "agents")), (