From 943eee9bbc364da5f99082aea027467866332be1 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Fri, 15 May 2026 11:54:44 -0400 Subject: [PATCH 1/5] feat(.claude): bug-repro plugin for OpenTDF (DSPX-3302) Adds five Claude Code skills under tests/.claude/skills/ that together turn a Jira bug ticket into a running reproduction, plus a downstream- installable plugin manifest under .claude/plugin/. Why --- The end-to-end goal of DSPX-3302 is to make bug reproduction approachable for QA, downstream-product engineers, and CI. PRs 1-4 build the plumbing (shared schema, platform installer, multi-instance otdf-local, xtest conftest hooks). This PR is the user-facing surface: a Claude can pull context from Jira, draft an xtest/scenarios/.yaml (and, when needed, an xtest/bug__test.py), bring the environment up at the right version pins, run the scenario's pytest selection, and tear down. Skills ------ scenario-from-bug-report Pulls the Jira issue and its comments via `acli jira workitem view --fields '*all' --json` and `acli jira workitem comment list`, extracts version pins / KAS topology / container type / feature flags, then writes xtest/scenarios/.yaml validated against otdf_sdk_mgr.schema.Scenario. Drafts a new xtest/bug__test.py only when no existing pytest covers the case; never silently lands assertions. scenario-up Runs `otdf-sdk-mgr install scenario`, then `otdf-local instance init --from-scenario`, then `otdf-local --instance up`, and polls status until healthy. Surfaces logs rather than retrying blindly when something stays unhealthy. scenario-run Invokes `otdf-local scenario run ` and classifies the result: "bug reproduced" / "not reproduced" / "unrelated failure". Cites the evidence line and points at per-service logs. scenario-tear-down Stops the instance and optionally removes the directory after explicit user confirmation. instance-status Lists known instances, their port bases, health, and flags port collisions. Jira-safety ----------- Permissions in both .claude/settings.json and the plugin manifest allow only read+comment via acli jira: workitem view, workitem search, workitem comment list, workitem comment create, plus a handful of read-only project/board/sprint queries. edit, delete, transition, assign, archive, link create, watcher add are all denied. The plugin.json carries a permission_notes block explaining the policy. Plugin manifest --------------- .claude/plugin/plugin.json declares the skill names, runtime requirements (uv, go, git, docker, acli), and the canonical permission allowlist, so downstream first/third-party integrators can install this plugin into their own Claude Code setups. Refs: https://virtru.atlassian.net/browse/DSPX-3302 Co-Authored-By: Claude Opus 4.7 (1M context) --- .claude/plugin/plugin.json | 38 ++++++ .claude/settings.json | 32 +++++ .claude/skills/instance-status/SKILL.md | 36 ++++++ .../skills/scenario-from-bug-report/SKILL.md | 111 ++++++++++++++++++ .claude/skills/scenario-run/SKILL.md | 43 +++++++ .claude/skills/scenario-tear-down/SKILL.md | 42 +++++++ .claude/skills/scenario-up/SKILL.md | 51 ++++++++ 7 files changed, 353 insertions(+) create mode 100644 .claude/plugin/plugin.json create mode 100644 .claude/settings.json create mode 100644 .claude/skills/instance-status/SKILL.md create mode 100644 .claude/skills/scenario-from-bug-report/SKILL.md create mode 100644 .claude/skills/scenario-run/SKILL.md create mode 100644 .claude/skills/scenario-tear-down/SKILL.md create mode 100644 .claude/skills/scenario-up/SKILL.md diff --git a/.claude/plugin/plugin.json b/.claude/plugin/plugin.json new file mode 100644 index 00000000..fce6cee4 --- /dev/null +++ b/.claude/plugin/plugin.json @@ -0,0 +1,38 @@ +{ + "name": "opentdf-test-harness", + "version": "0.1.0", + "description": "Skills for reproducing OpenTDF bugs locally via otdf-local and otdf-sdk-mgr. Pulls bug context from Jira (acli), provisions pinned platform/KAS/SDK versions, runs the xtest pytest suite, and tears down. Useful for QA, platform/SDK developers, and downstream first/third-party integrators.", + "skills_dir": "../skills", + "skills": [ + "scenario-from-bug-report", + "scenario-up", + "scenario-run", + "scenario-tear-down", + "instance-status" + ], + "requirements": [ + "uv (python package manager) on PATH", + "go toolchain (platform binaries are built from source)", + "git (for worktrees of opentdf/platform)", + "docker (for keycloak/postgres dependencies)", + "acli (Atlassian CLI; needed for the scenario-from-bug-report skill)" + ], + "permissions": { + "allow": [ + "Bash(uv run otdf-local *)", + "Bash(uv run otdf-sdk-mgr *)", + "Bash(uv run pytest *)", + "Bash(acli jira workitem view *)", + "Bash(acli jira workitem search *)", + "Bash(acli jira workitem comment list *)", + "Bash(acli jira workitem comment create *)", + "Bash(acli jira workitem attachment list *)", + "Bash(acli jira workitem link list *)", + "Bash(acli jira project view *)", + "Write(xtest/scenarios/**)", + "Write(xtest/bug_*_test.py)", + "Write(tests/instances/**)" + ] + }, + "permission_notes": "acli jira write-paths intentionally excluded: edit/delete/transition/assign/archive/clone/create/create-bulk/link create/watcher add/comment update/comment delete. Add them explicitly via .claude/settings.local.json if your team needs them; the default plugin is read+comment only." +} diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 00000000..a1dba2d8 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,32 @@ +{ + "permissions": { + "allow": [ + "Bash(uv run otdf-local *)", + "Bash(uv run otdf-sdk-mgr *)", + "Bash(uv run pytest *)", + "Bash(uv sync *)", + "Bash(git status *)", + "Bash(git diff *)", + "Bash(git log *)", + "Bash(git show *)", + "Bash(gh api *)", + "Bash(gh issue view *)", + "Bash(gh pr view *)", + "Bash(gh run *)", + "Bash(acli jira workitem view *)", + "Bash(acli jira workitem search *)", + "Bash(acli jira workitem comment list *)", + "Bash(acli jira workitem comment create *)", + "Bash(acli jira workitem attachment list *)", + "Bash(acli jira workitem link list *)", + "Bash(acli jira workitem watcher list *)", + "Bash(acli jira project view *)", + "Bash(acli jira board view *)", + "Bash(acli jira sprint view *)", + "Write(xtest/scenarios/**)", + "Write(xtest/bug_*_test.py)", + "Write(tests/instances/**)", + "Write(.claude/tmp/**)" + ] + } +} diff --git a/.claude/skills/instance-status/SKILL.md b/.claude/skills/instance-status/SKILL.md new file mode 100644 index 00000000..64bd545a --- /dev/null +++ b/.claude/skills/instance-status/SKILL.md @@ -0,0 +1,36 @@ +--- +name: instance-status +description: Report which test instances exist on disk, which are running, and the health of each service. Use when the user asks "what's running" or before bringing up another scenario to avoid port collisions. +allowed-tools: Bash, Read +--- + +# instance-status + +You give the user a snapshot of all test instances in this checkout: what's defined, what's running, and whether each service is healthy. + +## Process + +1. **List instances on disk**: + + ```bash + uv run otdf-local instance ls --json + ``` + + Each entry includes `name`, `platform` version, `ports_base`, and the `kas:` keys. Flag any two instances that share a `ports_base` — they cannot run concurrently. + +2. **For each instance**, check service status: + + ```bash + uv run otdf-local --instance status --json + ``` + + Each service reports `running`, `healthy`, and the bound port. Don't run all instances in parallel — iterate; a status query is cheap. + +3. **Summarize**: + - A short table per instance: service → port → state. + - Flag any unhealthy service with the path to its log (e.g. `tests/instances//logs/kas-alpha.log`). + - Mention port conflicts if two instances would collide on `ports.base`. + +## When ports collide + +`otdf-local instance init` warns about this at creation time but does not enforce it. If you see two instances with the same `ports_base`, recommend the user reassign one via `uv run otdf-local instance init --from-scenario --ports-base ` (or hand-edit the `instance.yaml`). diff --git a/.claude/skills/scenario-from-bug-report/SKILL.md b/.claude/skills/scenario-from-bug-report/SKILL.md new file mode 100644 index 00000000..3c28144f --- /dev/null +++ b/.claude/skills/scenario-from-bug-report/SKILL.md @@ -0,0 +1,111 @@ +--- +name: scenario-from-bug-report +description: Pull a Jira bug into context (via `acli jira workitem view`) and turn it into an xtest/scenarios/.yaml manifest, optionally drafting xtest/bug__test.py when no existing pytest covers it. Use when the user mentions a Jira issue key like DSPX-1234 (or another [PROJECT]-[NUMBER] format) and asks for a reproducer. +allowed-tools: Bash, Read, Write, Grep, Glob +--- + +# scenario-from-bug-report + +Bugs are tracked in Jira. The user will reference an issue by its key in the form `[PROJECT]-[NUMBER]` — examples: `DSPX-3302`, `DSPX-1234`. `DSPX` is the current project's prefix but the prefix can change (e.g. `OPS-`, `SDK-`); accept any short uppercase prefix. + +You produce two artifacts the rest of the toolchain consumes: + +1. `xtest/scenarios/.yaml` — validated against `otdf_sdk_mgr.schema.Scenario`. +2. (Optional) `xtest/bug__test.py` — only if no existing xtest pytest already exercises the bug. + +The Jira key also becomes the working **branch name** (`-repro` if a fresh branch is needed) and the scenario file's `metadata.id`. + +## Step 1 — Pull the Jira issue into context + +Always start by fetching the full issue content. Don't proceed on the user's free-text summary alone — the issue body has the version pins and reproduction details you need. + +```bash +acli jira workitem view --fields '*all' --json +acli jira workitem comment list +``` + +The first command's JSON output includes `summary`, `description`, `status`, and labels. The second lists comments. Extract: + +- The **summary** (becomes scenario `metadata.title`). +- The **description** (read carefully — version numbers, KAS topology, container types, and feature flags typically live here). +- Recent **comments** — reproductions and "what changed" notes often appear in comments rather than the original description. + +If the issue references attached logs, screenshots, or linked PRs, list them via `acli jira workitem attachment list ` and `acli jira workitem link list ` and mention them in your reply. + +**Permitted Jira writes**: only `acli jira workitem comment create ...` (to post a reproduction-status update if the user asks). Everything else — `edit`, `transition`, `assign`, `archive`, `delete`, `link create`, `watcher add` — is explicitly disallowed by the plugin's permissions; if the user wants those actions, instruct them to run the command themselves. + +## Step 2 — Identify the scenario inputs + +From the issue text, extract: + +- **Encrypt-side SDKs** — which SDKs *create* the TDF? (`go`, `java`, `js`). Pin versions. +- **Decrypt-side SDKs** — which SDKs *consume* the TDF? Pin versions. +- **Platform version** — git tag like `v0.9.0` (resolves to the `service/v0.9.0` tag in `opentdf/platform`). +- **KAS topology** — which KAS instances must be running (`alpha`, `beta`, `gamma`, `delta`, `km1`, `km2`) and whether any need a different pinned version than the platform. +- **Container type** — `ztdf`, `ztdf-ecwrap`, `nano`, or `nano-with-policy`. +- **Feature flags** — e.g. `ec_tdf_enabled`. +- **Expected vs actual behavior** — copy concise prose from the issue. + +If anything is ambiguous in the Jira issue, ask the user — don't guess at versions. + +## Step 3 — Pick the id and (optionally) the branch + +- `metadata.id = ` — e.g. `DSPX-3302` → `dspx-3302`. +- Scenario file path: `xtest/scenarios/.yaml`. +- If you need a new git branch, propose `-repro` (e.g. `DSPX-3302-repro`) and let the user confirm before switching. + +## Step 4 — Search for an existing pytest + +```bash +grep -rn "" xtest/test_*.py +``` + +Likely candidates: `test_tdfs.py` (roundtrip), `test_abac.py` (ABAC), `test_legacy.py` (golden), `test_pqc.py`. If a test already asserts the relevant behavior, reuse it — only the scenario changes, not the code. + +## Step 5 — Write `xtest/scenarios/.yaml` + +Exact field shape (the schema rejects unknown fields): + +```yaml +apiVersion: opentdf.io/v1alpha1 +kind: Scenario +metadata: + id: + title: "" + created: +instance: + metadata: { name: } + platform: { dist: } + ports: { base: } + kas: + : { dist: , mode: standard } # or mode: key_management +sdks: + encrypt: + : { version: } + decrypt: + : { version: } +suite: + select: "" + containers: + # markers: "not slow" + # extra_args: ["--no-audit-logs"] +expected: "" +actual: "" +``` + +Validate before reporting success: + +```bash +uv run python -m otdf_sdk_mgr.schema validate xtest/scenarios/.yaml +``` + +## Step 6 — If no existing test fits + +Draft `xtest/bug__test.py` using the `encrypt_sdk` / `decrypt_sdk` fixtures (pattern: `xtest/test_tdfs.py`). Surface the new file in your reply for the user to review — never silently land assertions. + +## Notes + +- `sdks.encrypt` and `sdks.decrypt` map to xtest's `--sdks-encrypt` / `--sdks-decrypt`. After PR #446 those pytest options take `sdk@version` specifiers like `go@v0.24.0`, `go@main`, or `go@*`. **Do NOT write those tokens in the YAML** — write a normal `{ version: lts }` (or any version string `otdf-sdk-mgr resolve` accepts: `v0.24.0`, `main`, an SDK-specific SHA, etc.). The `scenario-up` skill runs `otdf-sdk-mgr install scenario`, which records the resolved dist directory names in `xtest/scenarios/.installed.json`; the bridge layers (`otdf-local scenario run` and pytest's `--scenario` default in `xtest/conftest.py`) read that file to emit the right `sdk@` tokens. If you forget the install step, those commands fail with `.installed.json not found — run otdf-sdk-mgr install scenario first`. +- List the same SDK in both `encrypt` and `decrypt` maps to reproduce xtest's legacy "all pairs" mode. Listing it on only one side keeps the scenario focused (a→b without b→a). +- `instance.platform.dist` and each `kas..dist` need `otdf-sdk-mgr install scenario ` (or `install release platform:`) to have built the binary first. `scenario-up` handles that downstream. +- One-line summary when done: report the scenario path, the new test file (if any), and the Jira link `https://virtru.atlassian.net/browse/` so the user can cross-reference. diff --git a/.claude/skills/scenario-run/SKILL.md b/.claude/skills/scenario-run/SKILL.md new file mode 100644 index 00000000..633846cf --- /dev/null +++ b/.claude/skills/scenario-run/SKILL.md @@ -0,0 +1,43 @@ +--- +name: scenario-run +description: Execute the pytest suite declared by a scenarios.yaml against the running instance, then classify the result as "bug reproduced", "not reproduced", or "unrelated failure". Use after `scenario-up` has confirmed the instance is healthy. +allowed-tools: Bash, Read +--- + +# scenario-run + +You run the pytest selection declared by the scenario's `suite` block against the running instance and interpret the result in terms of the bug being investigated. + +## Inputs + +- Path to the scenario YAML (`xtest/scenarios/.yaml`). +- (Optional) the user's expected outcome, if the scenario's `expected:` field is sparse. + +## Process + +1. **Invoke the runner**: + + ```bash + uv run otdf-local scenario run xtest/scenarios/.yaml + ``` + + This translates the scenario's `suite.select`, `suite.containers`, `suite.markers`, and `sdks.{encrypt,decrypt}` into the equivalent `pytest --sdks-encrypt ... --sdks-decrypt ... --containers ...` invocation under `xtest/` with `OTDF_LOCAL_INSTANCE_NAME` set. SDK tokens are emitted in xtest's `sdk@version` form (see PR #446) — the resolved version names come from the sibling `.installed.json` that `otdf-sdk-mgr install scenario` writes. + + If `scenario run` exits with `Error: .installed.json not found`, the user skipped the install step. Tell them to run `uv run otdf-sdk-mgr install scenario ` (or re-run `scenario-up`) before retrying. + +2. **Capture exit code and tail of output**. The pytest output is the source of truth; don't re-interpret. + +3. **Classify**: + - **Bug reproduced** — the test failed with an assertion or stderr that matches the scenario's `actual:` field. Cite the matching line. + - **Bug NOT reproduced** — the test passed. This is meaningful: either the bug is fixed at this version combination, or the scenario doesn't capture it precisely yet. Suggest the user widen the assertion or pick a different version pin. + - **Unrelated failure** — pytest errored out (collection error, environment issue, import error, timeout). Don't claim repro success or failure; report the error and recommend a next diagnostic step. + +4. **Record artifacts**. The pytest run leaves logs under `tests/instances//logs/`. List the relevant log files in your reply so the user can attach them to the Jira ticket. + +## Output format + +One-line headline (`bug reproduced` / `not reproduced` / `unrelated failure`), then a short bulleted summary: +- `select:` the pytest selector +- `exit_code:` the return value +- `evidence:` 1-2 lines from the output that justify the classification +- `logs:` paths to the relevant per-service logs diff --git a/.claude/skills/scenario-tear-down/SKILL.md b/.claude/skills/scenario-tear-down/SKILL.md new file mode 100644 index 00000000..39398c76 --- /dev/null +++ b/.claude/skills/scenario-tear-down/SKILL.md @@ -0,0 +1,42 @@ +--- +name: scenario-tear-down +description: Stop the services for a scenario's instance and optionally delete the instance directory. Use when the user is done with a reproduction or wants to free ports/disk for a different scenario. +allowed-tools: Bash, Read +--- + +# scenario-tear-down + +You stop a running scenario cleanly and optionally remove its on-disk state. + +## Inputs + +- The instance name (typically the lowercased Jira key, e.g. `dspx-3302`). If the user passes the scenario YAML path instead, read its `instance.metadata.name`. +- Whether the user wants the instance directory preserved (default: yes — keep it for re-runs). + +## Process + +1. **Stop services**: + + ```bash + uv run otdf-local --instance down + ``` + + The `down` command halts the platform process, all KAS instances under management, and the docker dependencies (keycloak, postgres) — unless another instance is still using them, in which case docker is left running. + +2. **Optionally clean state**. Only if the user explicitly asked to remove: + + ```bash + uv run otdf-local instance rm -y + ``` + + This deletes `tests/instances//` including its `logs/`, `keys/`, and per-KAS configs. The platform binary at `xtest/platform/dist//service` is shared and is NOT removed (`otdf-sdk-mgr clean --dist-only` is the right command if the user wants to free that too). + +3. **Confirm port range is free** (useful if the user is about to bring up another scenario on the same base): + + ```bash + uv run otdf-local instance ls --json + ``` + +## Caution + +Never remove an instance without explicit user confirmation. The directory may contain golden keys or generated configs that took time to assemble. If unsure, leave it. diff --git a/.claude/skills/scenario-up/SKILL.md b/.claude/skills/scenario-up/SKILL.md new file mode 100644 index 00000000..955fce8b --- /dev/null +++ b/.claude/skills/scenario-up/SKILL.md @@ -0,0 +1,51 @@ +--- +name: scenario-up +description: Provision artifacts, scaffold the instance directory, and start the test environment for a given xtest/scenarios/.yaml. Use after `scenario-from-bug-report` (or when the user already has a scenario YAML) and wants the environment running. +allowed-tools: Bash, Read +--- + +# scenario-up + +You bring the environment described by a `scenarios.yaml` up and confirm it's healthy. The three steps are non-negotiable; do them in order. + +## Inputs + +- Path to a validated `xtest/scenarios/.yaml`. If the user doesn't provide one, ask. + +## Process + +1. **Install artifacts** — platform binary, per-KAS binaries, helper scripts, and the encrypt+decrypt SDKs declared in the scenario: + + ```bash + uv run otdf-sdk-mgr install scenario xtest/scenarios/.yaml + ``` + + This writes `xtest/scenarios/.installed.json` next to the scenario with the resolved dist paths. The first `go build` per platform version takes ~30-60s; subsequent runs reuse the cached binary. + +2. **Scaffold the instance directory** (creates `tests/instances//`): + + ```bash + uv run otdf-local instance init --from-scenario xtest/scenarios/.yaml + ``` + + If the instance already exists, this is a no-op for the existing files; double-check with `uv run otdf-local instance ls` first to avoid surprising the user. + +3. **Bring it up**: + + ```bash + uv run otdf-local --instance up + ``` + + Then poll status until everything is healthy (don't proceed before this succeeds): + + ```bash + uv run otdf-local --instance status --json + ``` + + If any service stays unhealthy after ~60 seconds, surface the relevant log via `uv run otdf-local --instance logs -n 50` and report the failure mode rather than retrying blindly. + +## Output + +Once healthy, report: +- The instance name and which ports it occupies (look at `instance.yaml`'s `ports.base`). +- The next command the user is likely to run (`scenario-run`). From b548228099aeadd98c93108b5fd917ffd491679d Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Fri, 15 May 2026 15:24:16 -0400 Subject: [PATCH 2/5] =?UTF-8?q?refactor(.claude):=20generalize=20scenario-?= =?UTF-8?q?from-bug-report=20=E2=86=92=20scenario-from-ticket=20(DSPX-3302?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Headless dogfooding (run-1 on DSPX-2719) showed the bug-only framing was too narrow — the common workflow is writing tests for new features first (TDD), not reproducing version-pinned bugs. - Rename and rewrite the skill to branch on Jira Issue Type. Bug follows the old expected/actual flow; Story/Task uses ref pins (`main`, feature branch, PR SHA via `gh pr view --json headRefOid`) for forward-looking regression gates; Spike bails out rather than fabricating. Mandates `acli workitem comment list` and steers away from cli.sh greps (both were run-1 gaps). - New `scenario-matrix` sibling skill: write N scenario files from a base × N refs (PRs/branches/releases). Schema/installer support was already there via `PlatformPin.source.ref` and `install_platform_source(ref)` — no other changes needed. - `scenario-run` output classification generalized from "bug reproduced / not reproduced" to "expected / unexpected outcome", with explicit branches for bug-repro vs TDD interpretations. - `scenario-up` description and `plugin.json` (description, skills array, requirements) updated to match. Co-Authored-By: Claude Opus 4.7 (1M context) --- .claude/plugin/plugin.json | 8 +- .../skills/scenario-from-bug-report/SKILL.md | 111 ------------ .claude/skills/scenario-from-ticket/SKILL.md | 159 ++++++++++++++++++ .claude/skills/scenario-matrix/SKILL.md | 91 ++++++++++ .claude/skills/scenario-run/SKILL.md | 19 ++- .claude/skills/scenario-up/SKILL.md | 2 +- 6 files changed, 268 insertions(+), 122 deletions(-) delete mode 100644 .claude/skills/scenario-from-bug-report/SKILL.md create mode 100644 .claude/skills/scenario-from-ticket/SKILL.md create mode 100644 .claude/skills/scenario-matrix/SKILL.md diff --git a/.claude/plugin/plugin.json b/.claude/plugin/plugin.json index fce6cee4..bb4c1311 100644 --- a/.claude/plugin/plugin.json +++ b/.claude/plugin/plugin.json @@ -1,10 +1,11 @@ { "name": "opentdf-test-harness", "version": "0.1.0", - "description": "Skills for reproducing OpenTDF bugs locally via otdf-local and otdf-sdk-mgr. Pulls bug context from Jira (acli), provisions pinned platform/KAS/SDK versions, runs the xtest pytest suite, and tears down. Useful for QA, platform/SDK developers, and downstream first/third-party integrators.", + "description": "Jira-ticket-driven scenarios for the OpenTDF test harness. Pulls ticket context from Jira (acli) — any ticket type, including bugs, feature stories, and PR-driven work — provisions pinned platform/KAS/SDK versions or refs (released versions, main, feature branches, PR SHAs), runs the xtest pytest suite, and tears down. Useful for QA, platform/SDK developers writing tests for new features first, and downstream first/third-party integrators.", "skills_dir": "../skills", "skills": [ - "scenario-from-bug-report", + "scenario-from-ticket", + "scenario-matrix", "scenario-up", "scenario-run", "scenario-tear-down", @@ -15,7 +16,8 @@ "go toolchain (platform binaries are built from source)", "git (for worktrees of opentdf/platform)", "docker (for keycloak/postgres dependencies)", - "acli (Atlassian CLI; needed for the scenario-from-bug-report skill)" + "acli (Atlassian CLI; needed for the scenario-from-ticket skill)", + "gh (GitHub CLI; needed for scenario-matrix to resolve PR refs)" ], "permissions": { "allow": [ diff --git a/.claude/skills/scenario-from-bug-report/SKILL.md b/.claude/skills/scenario-from-bug-report/SKILL.md deleted file mode 100644 index 3c28144f..00000000 --- a/.claude/skills/scenario-from-bug-report/SKILL.md +++ /dev/null @@ -1,111 +0,0 @@ ---- -name: scenario-from-bug-report -description: Pull a Jira bug into context (via `acli jira workitem view`) and turn it into an xtest/scenarios/.yaml manifest, optionally drafting xtest/bug__test.py when no existing pytest covers it. Use when the user mentions a Jira issue key like DSPX-1234 (or another [PROJECT]-[NUMBER] format) and asks for a reproducer. -allowed-tools: Bash, Read, Write, Grep, Glob ---- - -# scenario-from-bug-report - -Bugs are tracked in Jira. The user will reference an issue by its key in the form `[PROJECT]-[NUMBER]` — examples: `DSPX-3302`, `DSPX-1234`. `DSPX` is the current project's prefix but the prefix can change (e.g. `OPS-`, `SDK-`); accept any short uppercase prefix. - -You produce two artifacts the rest of the toolchain consumes: - -1. `xtest/scenarios/.yaml` — validated against `otdf_sdk_mgr.schema.Scenario`. -2. (Optional) `xtest/bug__test.py` — only if no existing xtest pytest already exercises the bug. - -The Jira key also becomes the working **branch name** (`-repro` if a fresh branch is needed) and the scenario file's `metadata.id`. - -## Step 1 — Pull the Jira issue into context - -Always start by fetching the full issue content. Don't proceed on the user's free-text summary alone — the issue body has the version pins and reproduction details you need. - -```bash -acli jira workitem view --fields '*all' --json -acli jira workitem comment list -``` - -The first command's JSON output includes `summary`, `description`, `status`, and labels. The second lists comments. Extract: - -- The **summary** (becomes scenario `metadata.title`). -- The **description** (read carefully — version numbers, KAS topology, container types, and feature flags typically live here). -- Recent **comments** — reproductions and "what changed" notes often appear in comments rather than the original description. - -If the issue references attached logs, screenshots, or linked PRs, list them via `acli jira workitem attachment list ` and `acli jira workitem link list ` and mention them in your reply. - -**Permitted Jira writes**: only `acli jira workitem comment create ...` (to post a reproduction-status update if the user asks). Everything else — `edit`, `transition`, `assign`, `archive`, `delete`, `link create`, `watcher add` — is explicitly disallowed by the plugin's permissions; if the user wants those actions, instruct them to run the command themselves. - -## Step 2 — Identify the scenario inputs - -From the issue text, extract: - -- **Encrypt-side SDKs** — which SDKs *create* the TDF? (`go`, `java`, `js`). Pin versions. -- **Decrypt-side SDKs** — which SDKs *consume* the TDF? Pin versions. -- **Platform version** — git tag like `v0.9.0` (resolves to the `service/v0.9.0` tag in `opentdf/platform`). -- **KAS topology** — which KAS instances must be running (`alpha`, `beta`, `gamma`, `delta`, `km1`, `km2`) and whether any need a different pinned version than the platform. -- **Container type** — `ztdf`, `ztdf-ecwrap`, `nano`, or `nano-with-policy`. -- **Feature flags** — e.g. `ec_tdf_enabled`. -- **Expected vs actual behavior** — copy concise prose from the issue. - -If anything is ambiguous in the Jira issue, ask the user — don't guess at versions. - -## Step 3 — Pick the id and (optionally) the branch - -- `metadata.id = ` — e.g. `DSPX-3302` → `dspx-3302`. -- Scenario file path: `xtest/scenarios/.yaml`. -- If you need a new git branch, propose `-repro` (e.g. `DSPX-3302-repro`) and let the user confirm before switching. - -## Step 4 — Search for an existing pytest - -```bash -grep -rn "" xtest/test_*.py -``` - -Likely candidates: `test_tdfs.py` (roundtrip), `test_abac.py` (ABAC), `test_legacy.py` (golden), `test_pqc.py`. If a test already asserts the relevant behavior, reuse it — only the scenario changes, not the code. - -## Step 5 — Write `xtest/scenarios/.yaml` - -Exact field shape (the schema rejects unknown fields): - -```yaml -apiVersion: opentdf.io/v1alpha1 -kind: Scenario -metadata: - id: - title: "" - created: -instance: - metadata: { name: } - platform: { dist: } - ports: { base: } - kas: - : { dist: , mode: standard } # or mode: key_management -sdks: - encrypt: - : { version: } - decrypt: - : { version: } -suite: - select: "" - containers: - # markers: "not slow" - # extra_args: ["--no-audit-logs"] -expected: "" -actual: "" -``` - -Validate before reporting success: - -```bash -uv run python -m otdf_sdk_mgr.schema validate xtest/scenarios/.yaml -``` - -## Step 6 — If no existing test fits - -Draft `xtest/bug__test.py` using the `encrypt_sdk` / `decrypt_sdk` fixtures (pattern: `xtest/test_tdfs.py`). Surface the new file in your reply for the user to review — never silently land assertions. - -## Notes - -- `sdks.encrypt` and `sdks.decrypt` map to xtest's `--sdks-encrypt` / `--sdks-decrypt`. After PR #446 those pytest options take `sdk@version` specifiers like `go@v0.24.0`, `go@main`, or `go@*`. **Do NOT write those tokens in the YAML** — write a normal `{ version: lts }` (or any version string `otdf-sdk-mgr resolve` accepts: `v0.24.0`, `main`, an SDK-specific SHA, etc.). The `scenario-up` skill runs `otdf-sdk-mgr install scenario`, which records the resolved dist directory names in `xtest/scenarios/.installed.json`; the bridge layers (`otdf-local scenario run` and pytest's `--scenario` default in `xtest/conftest.py`) read that file to emit the right `sdk@` tokens. If you forget the install step, those commands fail with `.installed.json not found — run otdf-sdk-mgr install scenario first`. -- List the same SDK in both `encrypt` and `decrypt` maps to reproduce xtest's legacy "all pairs" mode. Listing it on only one side keeps the scenario focused (a→b without b→a). -- `instance.platform.dist` and each `kas..dist` need `otdf-sdk-mgr install scenario ` (or `install release platform:`) to have built the binary first. `scenario-up` handles that downstream. -- One-line summary when done: report the scenario path, the new test file (if any), and the Jira link `https://virtru.atlassian.net/browse/` so the user can cross-reference. diff --git a/.claude/skills/scenario-from-ticket/SKILL.md b/.claude/skills/scenario-from-ticket/SKILL.md new file mode 100644 index 00000000..1f573b59 --- /dev/null +++ b/.claude/skills/scenario-from-ticket/SKILL.md @@ -0,0 +1,159 @@ +--- +name: scenario-from-ticket +description: Pull a Jira ticket of any type (Bug, Story, Task, Spike) into context via `acli jira workitem view` + `acli jira workitem comment list`, then turn it into an xtest/scenarios/.yaml manifest. Pins platform/KAS/SDKs to a released version (`dist:`), a branch or SHA (`source.ref:`), or the head of a PR — whichever matches the ticket. Optionally drafts xtest/bug__test.py when no existing pytest covers the behavior. Use when the user mentions a Jira key like DSPX-1234 (or any [PROJECT]-[NUMBER]) and wants a runnable scenario — reproducing a bug, writing a TDD test for a new feature, or validating behavior at a specific ref. +allowed-tools: Bash, Read, Write, Grep, Glob +--- + +# scenario-from-ticket + +You produce a `xtest/scenarios/.yaml` manifest from a Jira ticket. The same skill handles bugs, features (TDD), and exploratory work — the *Issue Type* field on the ticket selects which way the rest of this skill behaves. + +Two artifacts: + +1. `xtest/scenarios/.yaml` — validated against `otdf_sdk_mgr.schema.Scenario`. +2. (Optional) `xtest/bug__test.py` — only if no existing xtest pytest already exercises the behavior. The `bug_` prefix is a slug, not a type marker: feature-driven tests use it too. + +The Jira key also becomes the working **branch name** (`-repro` for Bugs, `-tdd` for Stories/Tasks) and the scenario file's `metadata.id`. + +## Step 1 — Pull the Jira ticket into context + +**Always run BOTH commands**. Don't skip the comment list — comments often carry the most recent reproduction status, "what changed" notes, or "fixed by PR #N" pointers that aren't in the original description: + +```bash +acli jira workitem view --fields '*all' --json +acli jira workitem comment list +``` + +From the JSON output of the first command, extract: + +- **Issue Type** (Bug, Story, Task, Spike) — load-bearing; selects which Step 2 branch to follow. +- **Summary** — becomes scenario `metadata.title`. +- **Description** — version numbers, KAS topology, container types, feature flags, acceptance criteria typically live here. +- **Status** — Backlog / In Progress / Done affects whether the scenario is forward-looking (TDD on Backlog) or retroactive (regression gate on Done). + +From the comments, pull any "tested at version X" / "reproduces on platform Y" / "fixed by PR #N" annotations into your mental model. + +If the ticket references attached logs, screenshots, or linked PRs, list them via `acli jira workitem attachment list ` and `acli jira workitem link list ` and call them out in your reply. + +**Permitted Jira writes**: only `acli jira workitem comment create ...` (to post a reproduction-status update if the user asks). Everything else — `edit`, `transition`, `assign`, `archive`, `delete`, `link create`, `watcher add` — is explicitly disallowed by the plugin's permissions; if the user wants those actions, instruct them to run the command themselves. + +## Step 2 — Branch on Issue Type + +### Bug + +The ticket describes a behavior that should work but doesn't. + +- `expected:` — what should happen (copy from the description's "expected behavior" section or rephrase the summary). +- `actual:` — what actually happens, including the exact error message if the ticket quotes one. +- Pin platform / KAS / SDKs to the **versions where the bug reproduces**. Usually `dist:` against a released version. Mixed-version topologies (e.g. platform `v0.9.0` + km1 `v0.9.0-rc.2`) are common and the schema supports them. + +If the description doesn't name versions, ask the user. (A headless agent has no user — in that case default to `dist: lts` everywhere and call out the assumption in `actual:`.) + +### Story / Task (feature work, TDD-style) + +The ticket describes a behavior the user wants to *add*. The scenario you produce is a forward-looking regression gate, not a bug reproducer. + +- `expected:` — the new behavior the feature should provide, paraphrased from acceptance criteria. +- `actual:` — the current state, e.g. "feature not implemented; tests skip via `.supports('')` until the supports entry lands." The scenario's `actual:` is what `scenario-run`'s "expected outcome" classifier compares against: a real failure means progress was made; a uniform skip means the prereq SDK plumbing is still pending. +- Pin platform / KAS / SDKs to the **ref where the feature will land**: + - HEAD of mainline: `platform: { source: { ref: main } }`, `sdks..version: main`. + - Feature branch: `platform: { source: { ref: feature/ecdsa-binding } }`. + - Draft PR under review: resolve to its head SHA with `gh pr view --json headRefOid` and pin `platform: { source: { ref: <40-char-SHA> } }`. SHAs are reproducible; branch names move every push. +- Only pin the component(s) the feature actually touches. Leave the rest on `lts` / `stable`. + +### Spike / unclear + +The ticket asks an open question or lacks enough concrete behavior to encode. Don't fabricate a scenario. Emit: + +``` + is a Spike (or has no specific behavior / version pins yet). Add either: + (a) the version or ref where you want behavior exercised, or + (b) a concrete pass/fail criterion (what should the test assert?) +…and re-invoke this skill. +``` + +…and stop. + +## Step 3 — Pick the id and (optionally) the branch + +- `metadata.id = ` — e.g. `DSPX-3302` → `dspx-3302`. +- Scenario file path: `xtest/scenarios/.yaml`. +- If you need a new git branch, propose `-repro` for Bugs and `-tdd` for Stories/Tasks; let the user confirm before switching. + +## Step 4 — Search for an existing pytest + +```bash +grep -rn "" xtest/test_*.py xtest/tdfs.py +``` + +Likely candidates: `test_tdfs.py` (roundtrip), `test_abac.py` (ABAC), `test_legacy.py` (golden), `test_pqc.py`. If a test already asserts the relevant behavior, reuse it via `suite.select` — no draft test needed. + +**Don't grep `xtest/sdk//cli.sh`.** Those wrappers are reusable infrastructure (versioned alongside each SDK dist) and their contents have nothing to do with scenario YAML fields. The scenario YAML doesn't need to know HOW a feature is plumbed — only WHICH pytest suite exercises it. Reading the wrappers is a waste of turns. If a feature's `supports("")` gate isn't in `tdfs.py` yet, that's a signal that supporting infrastructure has to land separately from the scenario — note it in `actual:` and move on. + +## Step 5 — Write `xtest/scenarios/.yaml` + +The schema (`otdf_sdk_mgr.schema.Scenario`) rejects unknown fields. Each pin (`PlatformPin`, `KasPin`) requires **exactly one** of `dist:`, `source:`, or `image:`. `image:` is reserved for forward-compat and rejected today — pick `dist:` or `source:`. + +Released-version pin (typical Bug scenario): + +```yaml +apiVersion: opentdf.io/v1alpha1 +kind: Scenario +metadata: + id: + title: "" + created: +instance: + metadata: { name: } + platform: { dist: v0.9.0 } + ports: { base: } + kas: + alpha: { dist: v0.9.0, mode: standard } +sdks: + encrypt: + go: { version: lts } + decrypt: + java: { version: "0.7.8" } +suite: + select: "xtest/test_tdfs.py::test_tdf_roundtrip" + containers: ztdf +expected: "..." +actual: "..." +``` + +Ref pin (TDD / HEAD / branch / PR): + +```yaml +instance: + platform: + source: { ref: main } # branch, tag, or 40-char SHA + kas: + alpha: + source: { ref: feature/ecdsa-binding } + mode: standard +sdks: + encrypt: + go: { version: main } # SdkPin.version accepts the same range of strings +``` + +Mix-and-match is fine — `platform` on `main`, `kas.alpha` on a released `dist:`, SDKs on different refs. + +Validate before reporting success: + +```bash +uv run python -m otdf_sdk_mgr.schema validate xtest/scenarios/.yaml +``` + +## Step 6 — If no existing test fits + +Draft `xtest/bug__test.py` using the `encrypt_sdk` / `decrypt_sdk` fixtures (pattern: `xtest/test_tdfs.py`). The `bug_` prefix is a historical slug applied to every scenario-tied test — feature/TDD ones use it too; don't let the name confuse you. Surface the new file in your reply for the user to review — never silently land assertions. + +For TDD tests where the underlying feature isn't yet implemented, gate participation behind `.supports("")` and call `pytest.skip(...)` when the gate fails. The scenario then runs as "all skipped" until the SDK supports entry lands, at which point the test becomes a real assertion. + +## Notes + +- `sdks.encrypt` and `sdks.decrypt` map to xtest's `--sdks-encrypt` / `--sdks-decrypt`. After PR #446 those pytest options take `sdk@version` specifiers like `go@v0.24.0`, `go@main`, or `go@*`. **Do NOT write those tokens in the YAML** — write a normal `{ version: lts }` (or any version string `otdf-sdk-mgr resolve` accepts: `v0.24.0`, `main`, an SDK-specific SHA, etc.). The `scenario-up` skill runs `otdf-sdk-mgr install scenario`, which records the resolved dist directory names in `xtest/scenarios/.installed.json`; the bridge layers (`otdf-local scenario run` and pytest's `--scenario` default in `xtest/conftest.py`) read that file to emit the right `sdk@` tokens. If you forget the install step, those commands fail with `.installed.json not found — run otdf-sdk-mgr install scenario first`. +- List the same SDK in both `encrypt` and `decrypt` maps to reproduce xtest's legacy "all pairs" mode. Listing it on only one side keeps the scenario focused (a→b without b→a). +- `instance.platform.dist` / `source.ref` and each `kas..dist` / `source.ref` need `otdf-sdk-mgr install scenario ` to have built the binary first. `scenario-up` handles that downstream. +- For matrix runs (same suite × N refs), don't author N scenarios by hand — invoke the `scenario-matrix` skill against this scenario as the base. +- One-line summary when done: report the scenario path, the new test file (if any), and the Jira link `https://virtru.atlassian.net/browse/` so the user can cross-reference. diff --git a/.claude/skills/scenario-matrix/SKILL.md b/.claude/skills/scenario-matrix/SKILL.md new file mode 100644 index 00000000..d2863749 --- /dev/null +++ b/.claude/skills/scenario-matrix/SKILL.md @@ -0,0 +1,91 @@ +--- +name: scenario-matrix +description: Given a base scenario (or a Jira ticket) plus a list of refs (PRs, branches, released versions), write one scenario file per ref so the same pytest suite runs across all of them. Use to bisect a regression across releases, validate a fix across multiple PRs, or check feature compatibility between versions. Generates files only — does not install or run them. +allowed-tools: Bash, Read, Write, Grep, Glob +--- + +# scenario-matrix + +You produce N scenario files from one base scenario, where N = the number of refs the user wants exercised. Each output scenario differs only in `instance.platform` (and optionally any KAS pins the user says should track the same ref). SDK pins are preserved unless explicitly told to vary. + +## Inputs + +- A **base**, either: + - Path to an existing `xtest/scenarios/.yaml`, OR + - A Jira ticket key — in which case invoke `scenario-from-ticket` first to produce the base, then proceed. +- A **ref list** — any combination of: + - Released versions: `v0.9.0`, `v0.8.5` + - Branch names: `main`, `feature/ecdsa-binding` + - PR numbers: `1234`, `1235` (resolved to head SHAs for reproducibility) +- (Optional) which KAS instances should track the same ref as `platform`. Default: every KAS instance in the base also tracks the ref. + +## Process + +### Step 1 — Resolve the base scenario + +- If given a path: `Read` it. +- If given a ticket key: invoke `scenario-from-ticket` against the ticket first, then `Read` the produced file. + +The base scenario provides everything except `instance.platform` (and tracked KAS pins): metadata.title becomes the title prefix, `suite` is shared across all cells, `sdks` is preserved. + +### Step 2 — Resolve each ref to a concrete value + +- Released version → use verbatim under `dist:`. Example: `v0.9.0` → `platform: { dist: v0.9.0 }`. +- Branch name → use under `source.ref:`. Example: `main` → `platform: { source: { ref: main } }`. +- PR number `N` → fetch: + + ```bash + gh pr view --json number,headRefName,headRefOid + ``` + + …and pin under `source.ref:` to the **`headRefOid`** (40-char SHA), **not** `headRefName`. Reason: branch names move on every push, SHAs don't. Record `headRefName` in the scenario title for human readability. + +### Step 3 — Emit one scenario file per ref + +Naming: `xtest/scenarios/-.yaml`. Tokens: + +- Released version: strip `v` and dots — `v0.9.0` → `v090`. +- Branch: replace `/` with `-` — `feature/ecdsa-binding` → `feature-ecdsa-binding`. +- PR: `pr` — `1234` → `pr1234`. The SHA still lives inside the file. + +Each cell scenario gets: + +- A unique `metadata.id` (`-`) matching the file basename. +- A unique `instance.metadata.name` (same as `metadata.id`). +- A unique `instance.ports.base` — start from the base's value and add `+1000` per additional cell. `scenario-up` rejects overlapping port bases between concurrent instances. +- `metadata.title` gets a ` []` suffix for at-a-glance identification. +- `instance.platform` rewritten to the resolved ref. For KAS pins that should track the same ref (default: all of them), rewrite their pin too. KAS pins the user explicitly excluded keep the base's value. +- `suite`, `sdks`, `expected`, `actual` — unchanged from the base. + +### Step 4 — Validate every file + +```bash +for f in xtest/scenarios/-*.yaml; do + uv run python -m otdf_sdk_mgr.schema validate "$f" +done +``` + +Bail (delete the just-written files) if any cell fails validation — partial matrices are confusing. + +### Step 5 — Report + +- The list of files written. +- The exact `scenario-up` / `scenario-run` chain the user can run per cell (or in a loop): + + ```bash + for f in xtest/scenarios/-*.yaml; do + name="$(basename "$f" .yaml)" + uv run otdf-sdk-mgr install scenario "$f" + uv run otdf-local instance init "$name" --from-scenario "$f" + uv run otdf-local --instance "$name" up + uv run otdf-local scenario run "$f" + uv run otdf-local --instance "$name" down + done + ``` + +## Notes + +- This skill **writes scenario files only**. It does not install artifacts, scaffold instances, or run pytest. Hand the resulting files to `scenario-up` and `scenario-run` per cell. +- For two PRs that differ in *SDK* (not platform), vary `sdks...version` instead of `platform`. Same pattern, different field — `SdkPin.version` accepts the same range of refs (`v0.24.0`, `main`, SHA). +- For a full platform × SDK matrix, generate N×M scenarios. Be prepared for long install times — each new platform ref triggers a `go build` (~30-60s first time per version); subsequent runs reuse the cached binary. +- Don't update `expected:` / `actual:` per cell unless the user specifies that one of the refs is the "known good" or "known broken" baseline. diff --git a/.claude/skills/scenario-run/SKILL.md b/.claude/skills/scenario-run/SKILL.md index 633846cf..c9a73eef 100644 --- a/.claude/skills/scenario-run/SKILL.md +++ b/.claude/skills/scenario-run/SKILL.md @@ -1,12 +1,12 @@ --- name: scenario-run -description: Execute the pytest suite declared by a scenarios.yaml against the running instance, then classify the result as "bug reproduced", "not reproduced", or "unrelated failure". Use after `scenario-up` has confirmed the instance is healthy. +description: Execute the pytest suite declared by a scenarios.yaml against the running instance, then classify the result as "expected outcome", "unexpected outcome", or "unrelated failure" against the scenario's `expected:` / `actual:` fields. Works for bug-repro scenarios, TDD/feature scenarios, and matrix runs. Use after `scenario-up` has confirmed the instance is healthy. allowed-tools: Bash, Read --- # scenario-run -You run the pytest selection declared by the scenario's `suite` block against the running instance and interpret the result in terms of the bug being investigated. +You run the pytest selection declared by the scenario's `suite` block against the running instance and interpret the result in terms of the ticket the scenario was authored for. The same three-bucket classification works for bug-repros (where "expected" means *failure that matches `actual:`*) and for TDD scenarios (where "expected" means *skip-until-feature-lands*). ## Inputs @@ -27,16 +27,21 @@ You run the pytest selection declared by the scenario's `suite` block against th 2. **Capture exit code and tail of output**. The pytest output is the source of truth; don't re-interpret. -3. **Classify**: - - **Bug reproduced** — the test failed with an assertion or stderr that matches the scenario's `actual:` field. Cite the matching line. - - **Bug NOT reproduced** — the test passed. This is meaningful: either the bug is fixed at this version combination, or the scenario doesn't capture it precisely yet. Suggest the user widen the assertion or pick a different version pin. - - **Unrelated failure** — pytest errored out (collection error, environment issue, import error, timeout). Don't claim repro success or failure; report the error and recommend a next diagnostic step. +3. **Classify** against the scenario's `expected:` and `actual:` fields: + - **Expected outcome** — the test result matches what `expected:` (or, for a bug, `actual:`) predicts. + - Bug scenario: pytest FAILED with an assertion/stderr matching `actual:`. Bug reproduced. Cite the matching line. + - TDD/feature scenario on a ref where the feature isn't landed yet: tests SKIPPED via `supports("")`. Feature gate is still pending as predicted. + - TDD/feature scenario on a ref where the feature is landed: tests PASSED. Feature works; the scenario is now a regression gate. + - **Unexpected outcome** — the test result is *not* what the scenario predicted. + - Bug scenario: pytest PASSED. Either the bug is fixed at this pin, or the scenario doesn't capture it tightly enough. Suggest widening the assertion, pinning a different ref, or marking the bug closed. + - TDD/feature scenario: tests FAILED for a reason that doesn't match `actual:`. A real bug surfaced, OR the prereq implementation work landed and the test now needs a real assertion (not a skip). Surface the actual failure to the user. + - **Unrelated failure** — pytest errored out (collection error, environment issue, import error, timeout). Don't claim outcome match either way; report the error and recommend a next diagnostic step. 4. **Record artifacts**. The pytest run leaves logs under `tests/instances//logs/`. List the relevant log files in your reply so the user can attach them to the Jira ticket. ## Output format -One-line headline (`bug reproduced` / `not reproduced` / `unrelated failure`), then a short bulleted summary: +One-line headline (`expected outcome` / `unexpected outcome` / `unrelated failure`), then a short bulleted summary: - `select:` the pytest selector - `exit_code:` the return value - `evidence:` 1-2 lines from the output that justify the classification diff --git a/.claude/skills/scenario-up/SKILL.md b/.claude/skills/scenario-up/SKILL.md index 955fce8b..8254dd61 100644 --- a/.claude/skills/scenario-up/SKILL.md +++ b/.claude/skills/scenario-up/SKILL.md @@ -1,6 +1,6 @@ --- name: scenario-up -description: Provision artifacts, scaffold the instance directory, and start the test environment for a given xtest/scenarios/.yaml. Use after `scenario-from-bug-report` (or when the user already has a scenario YAML) and wants the environment running. +description: Provision artifacts, scaffold the instance directory, and start the test environment for a given xtest/scenarios/.yaml. Use after `scenario-from-ticket` (or `scenario-matrix`, or when the user already has a scenario YAML) and wants the environment running. allowed-tools: Bash, Read --- From 6a680239d0dd0510944a359c45390f4a35eb5760 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Fri, 15 May 2026 21:24:46 -0400 Subject: [PATCH 3/5] feat(.claude): feature-design skill for cross-repo features (DSPX-3302) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For features (or bugs) that touch more than one OpenTDF repo — platform plus the Go / Java / JS SDKs — feature-design captures the work as a single spec at xtest/features/.yaml plus the tests-side artifacts that land first (feature_type entry in tdfs.py, scenario, draft test). The model matches the team's existing pattern: tests-side artifacts merge first, dormant under a `supports("")` gate, and each per-repo PR activates the gate by adding `supports ` to its cli.sh. PRs land async, in any order; no cross-PR lockstep needed. - `feature-design` SKILL: propose-then-iterate authoring from a Jira ticket (or free-form description). Drafts a complete spec on the first pass, asks one composite redirect question, then writes the spec + patches tdfs.py + invokes scenario-from-ticket internally to produce the dormant scenario and draft test. Bails on Spike or unclear tickets rather than fabricating. - `xtest/features/{README,CLAUDE}.md`: progressive-disclosure docs — human-facing README and agent-facing CLAUDE.md. - `xtest/README.md` gains a brief "Test artifact directories" section pointing at scenarios/ and features/. - `settings.json` + `plugin.json`: Write(xtest/features/**) allowlist, feature-design added to plugin skills array. The complementary feature-orchestrate skill (fanning out per-repo subagents to draft impl PRs in each touched repo) is a follow-up. Co-Authored-By: Claude Opus 4.7 (1M context) --- .claude/plugin/plugin.json | 2 + .claude/settings.json | 1 + .claude/skills/feature-design/SKILL.md | 118 +++++++++++++++++++++++++ xtest/README.md | 7 ++ xtest/features/CLAUDE.md | 13 +++ xtest/features/README.md | 14 +++ 6 files changed, 155 insertions(+) create mode 100644 .claude/skills/feature-design/SKILL.md create mode 100644 xtest/features/CLAUDE.md create mode 100644 xtest/features/README.md diff --git a/.claude/plugin/plugin.json b/.claude/plugin/plugin.json index bb4c1311..952feaea 100644 --- a/.claude/plugin/plugin.json +++ b/.claude/plugin/plugin.json @@ -4,6 +4,7 @@ "description": "Jira-ticket-driven scenarios for the OpenTDF test harness. Pulls ticket context from Jira (acli) — any ticket type, including bugs, feature stories, and PR-driven work — provisions pinned platform/KAS/SDK versions or refs (released versions, main, feature branches, PR SHAs), runs the xtest pytest suite, and tears down. Useful for QA, platform/SDK developers writing tests for new features first, and downstream first/third-party integrators.", "skills_dir": "../skills", "skills": [ + "feature-design", "scenario-from-ticket", "scenario-matrix", "scenario-up", @@ -32,6 +33,7 @@ "Bash(acli jira workitem link list *)", "Bash(acli jira project view *)", "Write(xtest/scenarios/**)", + "Write(xtest/features/**)", "Write(xtest/bug_*_test.py)", "Write(tests/instances/**)" ] diff --git a/.claude/settings.json b/.claude/settings.json index a1dba2d8..9fd70f3c 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -24,6 +24,7 @@ "Bash(acli jira board view *)", "Bash(acli jira sprint view *)", "Write(xtest/scenarios/**)", + "Write(xtest/features/**)", "Write(xtest/bug_*_test.py)", "Write(tests/instances/**)", "Write(.claude/tmp/**)" diff --git a/.claude/skills/feature-design/SKILL.md b/.claude/skills/feature-design/SKILL.md new file mode 100644 index 00000000..ebb2eed9 --- /dev/null +++ b/.claude/skills/feature-design/SKILL.md @@ -0,0 +1,118 @@ +--- +name: feature-design +description: Turn a multi-repo feature (or cross-repo bug fix) into a concrete spec at xtest/features/.yaml plus the tests-side artifacts that have to land first (scenario, draft pytest, feature_type entry in tdfs.py). Pulls Jira context, drafts a complete spec from the ticket, then iterates with the user. Use when a feature touches more than one repo (e.g. platform + Go SDK + Java SDK + JS SDK) and you want to set up the cross-repo work in one go without manually authoring each piece. +allowed-tools: Bash, Read, Write, Edit, Grep, Glob, Skill +--- + +# feature-design + +You turn a fuzzy "let's build X across the OpenTDF repos" into a concrete bundle of artifacts that pin down the tests-side work first and stage the cross-repo work for handoff to `feature-orchestrate`. + +Two ideas to internalize before reading the steps: + +1. **Tests-side artifacts land first, dormant.** The scenario + draft test + `feature_type` entry merge to `tests/main` as a regular PR. They stay "all skipped" until each SDK opens its own PR adding a `supports ` case to its `cli.sh` source — that PR's CI activates the test for that SDK. This means no cross-PR lockstep coordination; per-repo PRs land async, in any order. +2. **Propose, don't ask.** Draft a complete spec from the Jira ticket on the first pass and let the user redirect what's wrong in a single revision. Only ask one composite question. If you're missing information you can't fill in (no Jira ticket, ambiguous scope, unclear feature name), bail — don't fabricate. + +## Inputs + +- Jira key (Story/Task usually; Bug works the same way), OR a free-text description of the feature. +- (Optional) explicit list of repos to scope to, if the user wants something tighter than the default. + +## Steps + +### Step 1 — Pull the Jira context + +If a Jira key was given, run both — comments often carry scope refinements that aren't in the description: + +```bash +acli jira workitem view --fields '*all' --json +acli jira workitem comment list +``` + +Extract Issue Type, summary, description, status, and any comments about scope or implementation notes. If no Jira key, the user's description IS the spec input. + +### Step 2 — Propose a complete draft + +Draft the full spec body and the per-repo todo lists inline in your reply. Don't ask the user one field at a time — produce a complete first draft they can react to: + +- **Feature flag name** — snake_case identifier derived from the Jira summary. Becomes the `supports("")` gate string AND the `feature_type` entry in `xtest/tdfs.py`. Validate it's a valid Python identifier and doesn't collide with an existing `feature_type` member. +- **Touched repos** — default set is `tests, platform, sdk-go, sdk-java, sdk-web`. Trim or expand based on what the ticket says. Pure platform features skip the SDK repos; pure SDK-only features skip platform; `tests` is always present (the dormant scenario + tdfs.py entry has to live there). +- **Per-repo todo lists** — 2-4 bullets per repo, derived from the description plus each repo's known role: + - `tests` — register the feature in `feature_type`, author the scenario, draft the test gated on `supports("")`. + - `platform` — service-side implementation (KAS path, policy plumbing, etc.) and any env-var handling in the dev harness (e.g. honoring `XT_WITH_`). + - `sdk-go` / `sdk-java` / `sdk-web` — encrypt/decrypt path implementation, plus a `supports ` case in that SDK's `cli.sh` source. **Don't pin the version bound in the spec** — the implementing engineer sets the `awk` predicate at PR time, since the bound depends on which release will ship the impl. +- **Branch name** — `-`, the same string across every touched repo so `feature-orchestrate` (and the user) can find each repo's PR by branch alone. + +Present the draft, then ask exactly one composite question: "Anything to redirect — feature name, touched repos, todo items, branch?" Apply edits in a single revision rather than turn-by-turn. The user can always drop into plain chat if they want to think out loud — just answer them and re-invoke this skill once the design firms up. + +If no Jira key was given AND the user's description doesn't pin down a clear scope (feature flag name, touched repos, intended behavior), bail rather than fabricate: + +``` +I need either (a) a Jira Story/Task/Bug key, or (b) a description that names +the feature flag, the repos it touches, and the intended behavior. Add either +and re-invoke this skill. +``` + +### Step 3 — Write the spec + +Write `xtest/features/.yaml`. Shape (still informal — no Pydantic model yet): + +```yaml +apiVersion: opentdf.io/v1alpha1 +kind: Feature +metadata: + name: # supports() string + feature_type entry, snake_case + jira: # omit if no ticket + title: "" + created: +repos: + tests: + branch: - + todo: + - Register "" in xtest/tdfs.py feature_type + - Author scenario + draft test (via scenario-from-ticket) + platform: + branch: - + todo: [ ... ] + sdk-go: + branch: - + todo: + - Implement in the encrypt/decrypt path + - Add `supports ` case to cli.sh with version-bound awk predicate + sdk-java: { branch: ..., todo: [ ... ] } + sdk-web: { branch: ..., todo: [ ... ] } +scenarios: + - xtest/scenarios/.yaml +``` + +PR status (open/merged/CI passing) deliberately is NOT in the spec — it's auto-discovered from `gh pr list --search "head:"` per repo whenever something asks "where are we?" The spec is a declaration of intent. + +### Step 4 — Drive the tests-side artifacts + +In this order, so each step's output feeds the next: + +1. **Add the feature flag to `xtest/tdfs.py`**. Find the `feature_type` Literal alias near the top of the file. Insert the new entry alphabetically. Don't touch any `cli.sh` files — `supports ` cases land per-SDK in their own PRs. + +2. **Invoke `scenario-from-ticket`** via the Skill tool (`skill: scenario-from-ticket`, `args: `). It runs its Story/Task branch and produces the scenario + draft test gated on `supports("")` — pinning the feature-introducing components to `main` via `source.ref:`. If no Jira key was given, draft the scenario directly using the same shape (`xtest/scenarios/.yaml`). + +3. **Validate the scenario**: + + ```bash + uv run python -m otdf_sdk_mgr.schema validate xtest/scenarios/.yaml + ``` + +### Step 5 — Report + +One block summarizing: + +- The spec path (`xtest/features/.yaml`). +- The scenario + draft test paths. +- The line(s) added to `xtest/tdfs.py`. +- A one-liner suggesting the next step: `feature-orchestrate xtest/features/.yaml`. + +## Notes + +- This skill produces **tests-side artifacts only**. It does NOT create branches in other repos, does NOT open PRs, does NOT install platform/SDK builds. That's `feature-orchestrate`'s job. +- Bugs that span repos use the same shape — pass the Bug ticket key and `scenario-from-ticket`'s Bug branch fills `expected:` / `actual:` from the reproduction prose. The cross-repo gating still works: tests land dormant, each per-repo PR activates them by adding the supports case as part of the fix. +- For an existing spec being revised, read it first and propose a diff rather than a full rewrite. The tests-side artifacts (scenario, tdfs.py entry) usually shouldn't be regenerated — just edit them surgically. +- If the user starts the conversation by describing the feature in plain chat rather than invoking this skill, answer normally — re-invoke the skill once the scope firms up. Don't gatekeep. diff --git a/xtest/README.md b/xtest/README.md index 6bdfcc40..5de98942 100644 --- a/xtest/README.md +++ b/xtest/README.md @@ -122,3 +122,10 @@ pytest rm -rf tmp pytest test_tdfs.py ``` + +## Test artifact directories + +- **`scenarios/`** — Per-ticket scenario YAMLs that pin a platform / KAS / SDK topology to a specific pytest selection. Consumed by `otdf-local scenario run`. +- **`features/`** — Multi-repo feature specs: features that touch more than one OpenTDF repo (platform + SDKs) authored as a single declaration of intent. See `features/README.md`. + +Both are produced by the Claude Code skills under `tests/.claude/skills/` (`scenario-from-ticket`, `feature-design`, etc.) and can also be hand-authored. diff --git a/xtest/features/CLAUDE.md b/xtest/features/CLAUDE.md new file mode 100644 index 00000000..9f5e9a7e --- /dev/null +++ b/xtest/features/CLAUDE.md @@ -0,0 +1,13 @@ +# Agent guidance for xtest/features + +This directory is owned by two skills: + +- **`feature-design`** drafts new spec files here from a Jira ticket (or free-form description) using propose-then-iterate authoring. It also writes the tests-side artifacts that have to land first: the `feature_type` entry in `xtest/tdfs.py`, the scenario under `xtest/scenarios/`, and (if needed) a draft pytest. +- **`feature-orchestrate`** reads spec files and fans out per-repo subagents that implement the feature in each touched repo and open draft PRs. + +When you see a `xtest/features/.yaml` referenced: + +- It is canonical for the feature's flag name, scope, and per-repo todos. +- It is NOT canonical for status — query `gh pr list --search "head:"` per repo. + +Don't hand-author spec files in this directory unless you've also done what `feature-design` would do (add the entry to `feature_type` in `xtest/tdfs.py`, generate the scenario + draft test). Those side effects keep the spec consistent with the tests it depends on. diff --git a/xtest/features/README.md b/xtest/features/README.md new file mode 100644 index 00000000..2a1f5551 --- /dev/null +++ b/xtest/features/README.md @@ -0,0 +1,14 @@ +# xtest/features + +Specs for features that touch more than one OpenTDF repo (e.g. platform + Go SDK + Java SDK + JS SDK). + +Each `.yaml` captures: + +- The feature flag name — the `supports("")` gate string in `xtest/tdfs.py`. +- The Jira ticket driving the work, if any. +- Per-repo todo lists and the shared branch name to use across them. +- The scenario(s) under `xtest/scenarios/` that exercise the feature once each repo's PR lands. + +Specs are declarative — they describe intent, not status. PR state (open / merged / CI passing) is auto-discovered from `gh pr list --search "head:"` per repo, not stored here. + +See `CLAUDE.md` in this directory for how Claude Code skills produce and consume these files. From 42251871ec8b4fdb5ef5471beedccaea399146b9 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Fri, 15 May 2026 21:45:00 -0400 Subject: [PATCH 4/5] fix(.claude): allow Skill tool + correct acli comment list syntax (DSPX-3302) Headless dogfooding (runs 1 and 2 of scenario-from-ticket on DSPX-2719) surfaced two real gaps: - The `Skill` tool was denied on both runs because the allowlist didn't cover it, so the body of SKILL.md wasn't injected on invocation; the agent had to manually `Read` the skill file ~25 turns in, wasting time and biasing exploration toward grepping unrelated files first. Add `Skill(*)` to settings.json and per-skill `Skill()` entries to plugin.json (the latter enumerates exactly what downstream installs get, since they shouldn't inherit a wildcard). - `acli jira workitem comment list` requires `--key ` (the subcommand differs from `view`, which takes the key positionally). Both scenario-from-ticket and feature-design had the wrong form; corrected, with a one-line note about the asymmetry so the next agent doesn't paraphrase. Verified via run-3 on DSPX-2719: 41 turns / 5m16s / $1.07 (vs run-1's 48 turns / 6m44s / $1.27). Skill tool returned success on first call, both acli commands ran cleanly, the Story/Task branch produced `source.ref: main` pins correctly (no more incorrectly defaulting to `dist: lts`), and the agent's `actual:` field correctly enumerated all three test-infrastructure prerequisites including a `with_ecdsa_binding` parameter that run-1's scenario missed. Co-Authored-By: Claude Opus 4.7 (1M context) --- .claude/plugin/plugin.json | 7 +++++++ .claude/settings.json | 1 + .claude/skills/feature-design/SKILL.md | 4 ++-- .claude/skills/scenario-from-ticket/SKILL.md | 4 ++-- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.claude/plugin/plugin.json b/.claude/plugin/plugin.json index 952feaea..cbf554ef 100644 --- a/.claude/plugin/plugin.json +++ b/.claude/plugin/plugin.json @@ -32,6 +32,13 @@ "Bash(acli jira workitem attachment list *)", "Bash(acli jira workitem link list *)", "Bash(acli jira project view *)", + "Skill(feature-design)", + "Skill(scenario-from-ticket)", + "Skill(scenario-matrix)", + "Skill(scenario-up)", + "Skill(scenario-run)", + "Skill(scenario-tear-down)", + "Skill(instance-status)", "Write(xtest/scenarios/**)", "Write(xtest/features/**)", "Write(xtest/bug_*_test.py)", diff --git a/.claude/settings.json b/.claude/settings.json index 9fd70f3c..0f4e65da 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -23,6 +23,7 @@ "Bash(acli jira project view *)", "Bash(acli jira board view *)", "Bash(acli jira sprint view *)", + "Skill(*)", "Write(xtest/scenarios/**)", "Write(xtest/features/**)", "Write(xtest/bug_*_test.py)", diff --git a/.claude/skills/feature-design/SKILL.md b/.claude/skills/feature-design/SKILL.md index ebb2eed9..170e650a 100644 --- a/.claude/skills/feature-design/SKILL.md +++ b/.claude/skills/feature-design/SKILL.md @@ -22,11 +22,11 @@ Two ideas to internalize before reading the steps: ### Step 1 — Pull the Jira context -If a Jira key was given, run both — comments often carry scope refinements that aren't in the description: +If a Jira key was given, run both — `view` takes the key positionally, `comment list` requires `--key`; comments often carry scope refinements that aren't in the description: ```bash acli jira workitem view --fields '*all' --json -acli jira workitem comment list +acli jira workitem comment list --key ``` Extract Issue Type, summary, description, status, and any comments about scope or implementation notes. If no Jira key, the user's description IS the spec input. diff --git a/.claude/skills/scenario-from-ticket/SKILL.md b/.claude/skills/scenario-from-ticket/SKILL.md index 1f573b59..7f154ded 100644 --- a/.claude/skills/scenario-from-ticket/SKILL.md +++ b/.claude/skills/scenario-from-ticket/SKILL.md @@ -17,11 +17,11 @@ The Jira key also becomes the working **branch name** (`-repro` for Bu ## Step 1 — Pull the Jira ticket into context -**Always run BOTH commands**. Don't skip the comment list — comments often carry the most recent reproduction status, "what changed" notes, or "fixed by PR #N" pointers that aren't in the original description: +**Always run BOTH commands** — exactly as shown; the two subcommands take the key differently (`view` is positional, `comment list` requires `--key`). Don't skip the comment list — comments often carry the most recent reproduction status, "what changed" notes, or "fixed by PR #N" pointers that aren't in the original description: ```bash acli jira workitem view --fields '*all' --json -acli jira workitem comment list +acli jira workitem comment list --key ``` From the JSON output of the first command, extract: From 5197c2195b463f4a19043030650f1e23f9822517 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Fri, 15 May 2026 21:53:41 -0400 Subject: [PATCH 5/5] feat(otdf-sdk-mgr): schema dump CLI + xtest/schema canonical JSON Schemas (DSPX-3302) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Headless runs of scenario-from-ticket kept trying `python3 -c "from otdf_sdk_mgr.schema import Scenario; ..."` to introspect Pydantic model shape while authoring scenarios. That form isn't in the plugin's Bash allowlist (deliberately — it's arbitrary code execution), so the agent fell back to Reading schema.py source. Static, committed JSON Schemas give the same information declaratively without needing a python verb in the allowlist at all. - `otdf-sdk-mgr schema dump [--out-dir]`: writes `xtest/schema/{scenario,instance}.schema.json` from `Model.model_json_schema()`, sorted-keys + trailing newline so output is byte-stable. Add new models to `SCHEMAS` in cli_schema.py and they get picked up automatically. - `xtest/schema/` is committed with the generated files plus brief README/CLAUDE.md (progressive-disclosure, mirroring xtest/features/). - `test_schema_sync.py` parametrizes over `SCHEMAS` and fails if any committed file drifts from the live model — the safety net for "someone edited a Pydantic model without regenerating." - `scenario-from-ticket` SKILL.md Step 5 now points at `xtest/schema/scenario.schema.json` as the canonical field list. - `xtest/README.md` lists the new directory alongside `scenarios/` and `features/`. No allowlist changes needed — `Bash(uv run otdf-sdk-mgr *)` already covers the dump subcommand, and `Read` is unrestricted. Co-Authored-By: Claude Opus 4.7 (1M context) --- .claude/skills/scenario-from-ticket/SKILL.md | 2 +- otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py | 2 + otdf-sdk-mgr/src/otdf_sdk_mgr/cli_schema.py | 57 +++ otdf-sdk-mgr/tests/test_schema_sync.py | 38 ++ xtest/README.md | 3 +- xtest/schema/CLAUDE.md | 8 + xtest/schema/README.md | 16 + xtest/schema/instance.schema.json | 285 ++++++++++++ xtest/schema/scenario.schema.json | 465 +++++++++++++++++++ 9 files changed, 874 insertions(+), 2 deletions(-) create mode 100644 otdf-sdk-mgr/src/otdf_sdk_mgr/cli_schema.py create mode 100644 otdf-sdk-mgr/tests/test_schema_sync.py create mode 100644 xtest/schema/CLAUDE.md create mode 100644 xtest/schema/README.md create mode 100644 xtest/schema/instance.schema.json create mode 100644 xtest/schema/scenario.schema.json diff --git a/.claude/skills/scenario-from-ticket/SKILL.md b/.claude/skills/scenario-from-ticket/SKILL.md index 7f154ded..12e96067 100644 --- a/.claude/skills/scenario-from-ticket/SKILL.md +++ b/.claude/skills/scenario-from-ticket/SKILL.md @@ -92,7 +92,7 @@ Likely candidates: `test_tdfs.py` (roundtrip), `test_abac.py` (ABAC), `test_lega ## Step 5 — Write `xtest/scenarios/.yaml` -The schema (`otdf_sdk_mgr.schema.Scenario`) rejects unknown fields. Each pin (`PlatformPin`, `KasPin`) requires **exactly one** of `dist:`, `source:`, or `image:`. `image:` is reserved for forward-compat and rejected today — pick `dist:` or `source:`. +The canonical field list (titles, types, defaults, `anyOf` branches) lives in `xtest/schema/scenario.schema.json` — `Read` it whenever you need to know what's allowed. Each pin (`PlatformPin`, `KasPin`) requires **exactly one** of `dist:`, `source:`, or `image:`. `image:` is reserved for forward-compat and rejected today — pick `dist:` or `source:`. Released-version pin (typical Bug scenario): diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py index 24148bdd..78b137c9 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py @@ -10,6 +10,7 @@ import typer from otdf_sdk_mgr.cli_install import install_app +from otdf_sdk_mgr.cli_schema import schema_app from otdf_sdk_mgr.cli_versions import versions_app from otdf_sdk_mgr.config import ALL_SDKS, get_sdk_dirs @@ -20,6 +21,7 @@ ) app.add_typer(install_app, name="install") +app.add_typer(schema_app, name="schema") app.add_typer(versions_app, name="versions") diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_schema.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_schema.py new file mode 100644 index 00000000..b3fb17b7 --- /dev/null +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_schema.py @@ -0,0 +1,57 @@ +"""`otdf-sdk-mgr schema` subcommands. + +Emit canonical JSON Schemas for the Pydantic models in `otdf_sdk_mgr.schema` +so agents (and humans) can introspect the on-disk YAML formats without +running `python -c` against the package. The generated files live under +`xtest/schema/` and are kept in sync via `tests/test_schema_sync.py`. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Annotated + +import typer +from otdf_sdk_mgr.schema import Instance, Scenario + +schema_app = typer.Typer(help="Emit JSON Schemas for the scenario/instance models.") + +# (model_class, output_filename). Add new models here and `schema dump` +# will pick them up automatically. +SCHEMAS: tuple[tuple[type, str], ...] = ( + (Scenario, "scenario.schema.json"), + (Instance, "instance.schema.json"), +) + + +def render(model: type) -> str: + """Render `model.model_json_schema()` as a deterministic JSON string. + + Sorted keys and a trailing newline so byte-equality comparisons in the + sync test are stable. + """ + return json.dumps(model.model_json_schema(), indent=2, sort_keys=True) + "\n" + + +@schema_app.command("dump") +def dump( + out_dir: Annotated[ + Path, + typer.Option( + "--out-dir", + help="Directory to write *.schema.json files into.", + ), + ] = Path("xtest/schema"), +) -> None: + """Write JSON Schemas for every canonical scenario/instance model. + + Overwrites existing files. Re-run whenever a Pydantic model changes; + the committed schemas in xtest/schema/ are otherwise the source of + truth that the scenario-authoring skills read. + """ + out_dir.mkdir(parents=True, exist_ok=True) + for model, filename in SCHEMAS: + path = out_dir / filename + path.write_text(render(model), encoding="utf-8") + typer.echo(f" wrote {path}") diff --git a/otdf-sdk-mgr/tests/test_schema_sync.py b/otdf-sdk-mgr/tests/test_schema_sync.py new file mode 100644 index 00000000..addeaf8a --- /dev/null +++ b/otdf-sdk-mgr/tests/test_schema_sync.py @@ -0,0 +1,38 @@ +"""Guard that the committed JSON Schemas under xtest/schema/ stay in sync +with the live Pydantic models. + +The skills authoring scenarios read those JSON files directly to know what +fields are allowed; if a Pydantic model gains, loses, or renames a field +without a corresponding `uv run otdf-sdk-mgr schema dump`, the skills will +silently rely on a stale schema. This test makes that drift loud. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from otdf_sdk_mgr.cli_schema import SCHEMAS, render + + +def _xtest_schema_dir() -> Path: + """Locate xtest/schema/ relative to this test file. + + The repo layout puts otdf-sdk-mgr/tests/ next to xtest/, so two parents + up from this file is the tests/ root. + """ + return Path(__file__).resolve().parents[2] / "xtest" / "schema" + + +@pytest.mark.parametrize(("model", "filename"), SCHEMAS, ids=lambda v: getattr(v, "__name__", v)) +def test_committed_schema_matches_model(model: type, filename: str) -> None: + path = _xtest_schema_dir() / filename + assert path.is_file(), ( + f"Missing {path}. Run `uv run otdf-sdk-mgr schema dump` to regenerate." + ) + expected = render(model) + actual = path.read_text(encoding="utf-8") + assert actual == expected, ( + f"{path} is out of sync with {model.__name__}. " + f"Run `uv run otdf-sdk-mgr schema dump` to regenerate." + ) diff --git a/xtest/README.md b/xtest/README.md index 5de98942..0c7400fa 100644 --- a/xtest/README.md +++ b/xtest/README.md @@ -127,5 +127,6 @@ pytest test_tdfs.py - **`scenarios/`** — Per-ticket scenario YAMLs that pin a platform / KAS / SDK topology to a specific pytest selection. Consumed by `otdf-local scenario run`. - **`features/`** — Multi-repo feature specs: features that touch more than one OpenTDF repo (platform + SDKs) authored as a single declaration of intent. See `features/README.md`. +- **`schema/`** — Generated JSON Schemas for the canonical scenario / instance models. Regenerate via `uv run otdf-sdk-mgr schema dump` after editing the Pydantic models in `otdf-sdk-mgr/src/otdf_sdk_mgr/schema.py`. See `schema/README.md`. -Both are produced by the Claude Code skills under `tests/.claude/skills/` (`scenario-from-ticket`, `feature-design`, etc.) and can also be hand-authored. +The first two are produced by the Claude Code skills under `tests/.claude/skills/` (`scenario-from-ticket`, `feature-design`, etc.) and can also be hand-authored. diff --git a/xtest/schema/CLAUDE.md b/xtest/schema/CLAUDE.md new file mode 100644 index 00000000..7b215459 --- /dev/null +++ b/xtest/schema/CLAUDE.md @@ -0,0 +1,8 @@ +# Agent guidance for xtest/schema + +These JSON Schemas are the canonical reference for the on-disk YAML formats. When you need to know what fields a scenario or instance accepts: + +- **Read these files**. Don't run `python -c "from otdf_sdk_mgr.schema import ..."` to introspect — those forms aren't in the plugin's allowlist, and the JSON Schemas have the same information in declarative form (titles, types, `anyOf` for ref-vs-version pins, `additionalProperties: false`, default values, etc.). +- The files are byte-stable and sorted; safe to grep, diff, or quote. + +If a Pydantic model changes and these files drift, the user (or CI) will regenerate them via `uv run otdf-sdk-mgr schema dump`. Don't try to regenerate them yourself unless you're explicitly fixing the drift in a schema-editing PR. diff --git a/xtest/schema/README.md b/xtest/schema/README.md new file mode 100644 index 00000000..c292457a --- /dev/null +++ b/xtest/schema/README.md @@ -0,0 +1,16 @@ +# xtest/schema + +JSON Schemas for the canonical scenario / instance YAML formats. One file per Pydantic model in `otdf-sdk-mgr/src/otdf_sdk_mgr/schema.py`: + +- `scenario.schema.json` — the shape that `xtest/scenarios/.yaml` validates against. +- `instance.schema.json` — the shape of `tests/instances//instance.yaml`. + +These files are generated artifacts. To refresh them after editing a Pydantic model: + +```bash +uv run --project otdf-sdk-mgr otdf-sdk-mgr schema dump +``` + +A pytest in `otdf-sdk-mgr/tests/test_schema_sync.py` fails CI if the committed files drift from what the live models would produce. + +See `CLAUDE.md` for how Claude Code skills consume these files. diff --git a/xtest/schema/instance.schema.json b/xtest/schema/instance.schema.json new file mode 100644 index 00000000..cc4b40c0 --- /dev/null +++ b/xtest/schema/instance.schema.json @@ -0,0 +1,285 @@ +{ + "$defs": { + "Fixtures": { + "additionalProperties": false, + "properties": { + "attributes": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Attributes" + }, + "policy": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Policy" + } + }, + "title": "Fixtures", + "type": "object" + }, + "KasPin": { + "additionalProperties": false, + "description": "Per-KAS-instance version + mode pin.", + "properties": { + "dist": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Dist" + }, + "features": { + "additionalProperties": { + "type": "boolean" + }, + "title": "Features", + "type": "object" + }, + "image": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Image" + }, + "mode": { + "default": "standard", + "enum": [ + "standard", + "key_management" + ], + "title": "Mode", + "type": "string" + }, + "source": { + "anyOf": [ + { + "$ref": "#/$defs/SourceRef" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "title": "KasPin", + "type": "object" + }, + "Metadata": { + "additionalProperties": false, + "properties": { + "created": { + "anyOf": [ + { + "format": "date", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Created" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Id" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Title" + } + }, + "title": "Metadata", + "type": "object" + }, + "PlatformPin": { + "additionalProperties": false, + "description": "Version pin for the platform service.\n\n`dist` references a built binary at `xtest/platform/dist//service`\nproduced by `otdf-sdk-mgr install platform:`. `source.ref` is a\ngit ref to build from on demand. `image` is reserved for forward-compat\nonce container images are published; rejected at run time today.", + "properties": { + "dist": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Dist" + }, + "image": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Image" + }, + "source": { + "anyOf": [ + { + "$ref": "#/$defs/SourceRef" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "title": "PlatformPin", + "type": "object" + }, + "PortsConfig": { + "additionalProperties": false, + "properties": { + "base": { + "default": 8080, + "maximum": 60000, + "minimum": 1024, + "title": "Base", + "type": "integer" + } + }, + "title": "PortsConfig", + "type": "object" + }, + "SourceRef": { + "additionalProperties": false, + "properties": { + "path": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional local checkout path", + "title": "Path" + }, + "ref": { + "description": "Git tag, branch, or SHA", + "title": "Ref", + "type": "string" + } + }, + "required": [ + "ref" + ], + "title": "SourceRef", + "type": "object" + } + }, + "additionalProperties": false, + "description": "Standalone instance definition (one platform + N KAS).\n\nPersisted to `tests/instances//instance.yaml`. Also embedded inside\nScenario to keep the \"describe a bug-repro environment\" entry point a\nsingle file.", + "properties": { + "apiVersion": { + "const": "opentdf.io/v1alpha1", + "default": "opentdf.io/v1alpha1", + "title": "Apiversion", + "type": "string" + }, + "features": { + "additionalProperties": { + "type": "boolean" + }, + "title": "Features", + "type": "object" + }, + "fixtures": { + "$ref": "#/$defs/Fixtures" + }, + "kas": { + "additionalProperties": { + "$ref": "#/$defs/KasPin" + }, + "title": "Kas", + "type": "object" + }, + "kind": { + "const": "Instance", + "default": "Instance", + "title": "Kind", + "type": "string" + }, + "metadata": { + "$ref": "#/$defs/Metadata" + }, + "platform": { + "$ref": "#/$defs/PlatformPin" + }, + "ports": { + "$ref": "#/$defs/PortsConfig" + } + }, + "required": [ + "platform" + ], + "title": "Instance", + "type": "object" +} diff --git a/xtest/schema/scenario.schema.json b/xtest/schema/scenario.schema.json new file mode 100644 index 00000000..d7fc6adc --- /dev/null +++ b/xtest/schema/scenario.schema.json @@ -0,0 +1,465 @@ +{ + "$defs": { + "Fixtures": { + "additionalProperties": false, + "properties": { + "attributes": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Attributes" + }, + "policy": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Policy" + } + }, + "title": "Fixtures", + "type": "object" + }, + "Instance": { + "additionalProperties": false, + "description": "Standalone instance definition (one platform + N KAS).\n\nPersisted to `tests/instances//instance.yaml`. Also embedded inside\nScenario to keep the \"describe a bug-repro environment\" entry point a\nsingle file.", + "properties": { + "apiVersion": { + "const": "opentdf.io/v1alpha1", + "default": "opentdf.io/v1alpha1", + "title": "Apiversion", + "type": "string" + }, + "features": { + "additionalProperties": { + "type": "boolean" + }, + "title": "Features", + "type": "object" + }, + "fixtures": { + "$ref": "#/$defs/Fixtures" + }, + "kas": { + "additionalProperties": { + "$ref": "#/$defs/KasPin" + }, + "title": "Kas", + "type": "object" + }, + "kind": { + "const": "Instance", + "default": "Instance", + "title": "Kind", + "type": "string" + }, + "metadata": { + "$ref": "#/$defs/Metadata" + }, + "platform": { + "$ref": "#/$defs/PlatformPin" + }, + "ports": { + "$ref": "#/$defs/PortsConfig" + } + }, + "required": [ + "platform" + ], + "title": "Instance", + "type": "object" + }, + "KasPin": { + "additionalProperties": false, + "description": "Per-KAS-instance version + mode pin.", + "properties": { + "dist": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Dist" + }, + "features": { + "additionalProperties": { + "type": "boolean" + }, + "title": "Features", + "type": "object" + }, + "image": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Image" + }, + "mode": { + "default": "standard", + "enum": [ + "standard", + "key_management" + ], + "title": "Mode", + "type": "string" + }, + "source": { + "anyOf": [ + { + "$ref": "#/$defs/SourceRef" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "title": "KasPin", + "type": "object" + }, + "Metadata": { + "additionalProperties": false, + "properties": { + "created": { + "anyOf": [ + { + "format": "date", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Created" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Id" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Title" + } + }, + "title": "Metadata", + "type": "object" + }, + "PlatformPin": { + "additionalProperties": false, + "description": "Version pin for the platform service.\n\n`dist` references a built binary at `xtest/platform/dist//service`\nproduced by `otdf-sdk-mgr install platform:`. `source.ref` is a\ngit ref to build from on demand. `image` is reserved for forward-compat\nonce container images are published; rejected at run time today.", + "properties": { + "dist": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Dist" + }, + "image": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Image" + }, + "source": { + "anyOf": [ + { + "$ref": "#/$defs/SourceRef" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "title": "PlatformPin", + "type": "object" + }, + "PortsConfig": { + "additionalProperties": false, + "properties": { + "base": { + "default": 8080, + "maximum": 60000, + "minimum": 1024, + "title": "Base", + "type": "integer" + } + }, + "title": "PortsConfig", + "type": "object" + }, + "ScenarioSdks": { + "additionalProperties": false, + "description": "Encrypt/decrypt split mirrors xtest's --sdks-encrypt/--sdks-decrypt.\n\nListing the same SDK in both maps reproduces the legacy \"all pairs\" mode.", + "properties": { + "decrypt": { + "additionalProperties": { + "$ref": "#/$defs/SdkPin" + }, + "propertyNames": { + "enum": [ + "go", + "java", + "js" + ] + }, + "title": "Decrypt", + "type": "object" + }, + "encrypt": { + "additionalProperties": { + "$ref": "#/$defs/SdkPin" + }, + "propertyNames": { + "enum": [ + "go", + "java", + "js" + ] + }, + "title": "Encrypt", + "type": "object" + } + }, + "title": "ScenarioSdks", + "type": "object" + }, + "SdkPin": { + "additionalProperties": false, + "description": "SDK version pin (forwarded to otdf-sdk-mgr's existing resolve()).", + "properties": { + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "For Go: \"platform\" to use the monorepo module path", + "title": "Source" + }, + "version": { + "title": "Version", + "type": "string" + } + }, + "required": [ + "version" + ], + "title": "SdkPin", + "type": "object" + }, + "SourceRef": { + "additionalProperties": false, + "properties": { + "path": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional local checkout path", + "title": "Path" + }, + "ref": { + "description": "Git tag, branch, or SHA", + "title": "Ref", + "type": "string" + } + }, + "required": [ + "ref" + ], + "title": "SourceRef", + "type": "object" + }, + "Suite": { + "additionalProperties": false, + "description": "Pytest selection + flags.", + "properties": { + "containers": { + "anyOf": [ + { + "enum": [ + "ztdf", + "ztdf-ecwrap", + "nano", + "nano-with-policy" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Forwarded to --containers", + "title": "Containers" + }, + "extra_args": { + "items": { + "type": "string" + }, + "title": "Extra Args", + "type": "array" + }, + "markers": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Forwarded to -m", + "title": "Markers" + }, + "select": { + "description": "Pytest -k or path::node selector", + "title": "Select", + "type": "string" + } + }, + "required": [ + "select" + ], + "title": "Suite", + "type": "object" + } + }, + "additionalProperties": false, + "description": "Top-level scenarios.yaml model.\n\nComposes an Instance with SDK pins and a pytest Suite selection.", + "properties": { + "actual": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Actual" + }, + "apiVersion": { + "const": "opentdf.io/v1alpha1", + "default": "opentdf.io/v1alpha1", + "title": "Apiversion", + "type": "string" + }, + "expected": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected" + }, + "instance": { + "$ref": "#/$defs/Instance", + "description": "Inline instance definition" + }, + "kind": { + "const": "Scenario", + "default": "Scenario", + "title": "Kind", + "type": "string" + }, + "metadata": { + "$ref": "#/$defs/Metadata" + }, + "sdks": { + "$ref": "#/$defs/ScenarioSdks" + }, + "suite": { + "$ref": "#/$defs/Suite" + } + }, + "required": [ + "instance", + "suite" + ], + "title": "Scenario", + "type": "object" +}