Skip to content

scenario-suite

scenario-suite #98

name: scenario-suite
# Drives the prod release lifecycle end to end and asserts on the
# resulting tags, GitHub releases, and cascade state pointer.
#
# This suite never runs the lifecycle inline. It fires the existing
# generated workflows (orchestrate on merge, Release on dispatch) and
# polls for the expected shapes. Assertions check patterns and flags,
# never hardcoded SHAs or versions.
on:
workflow_dispatch:
inputs:
cascade_version:
description: 'cascade rc tag to self-repin to (e.g. v0.13.0-dryrun.46). Empty runs committed defaults.'
required: false
default: ''
cascade_version_sha:
description: 'Peeled commit SHA paired with cascade_version. Empty runs committed defaults.'
required: false
default: ''
schedule:
- cron: '0 6 * * 1'
permissions:
contents: write
pull-requests: write
actions: write
concurrency:
group: scenario-suite
cancel-in-progress: false
env:
GH_TOKEN: ${{ secrets.CASCADE_STATE_TOKEN }}
MANIFEST: .github/manifest.yaml
MANIFEST_KEY: ci
RELEASE_WORKFLOW: promote.yaml
jobs:
# A single declared environment is below cascade's two-env hotfix gate, so the
# generator must NOT emit cascade-hotfix.yaml for this repo. This is a pure
# generation-shape assertion against the checked-out (fleet-repinned) workflow
# set: it causes no run, so there is nothing to register; the reconcile gate is
# unaffected. It fails hard if the hotfix workflow is ever present here.
hotfix-omission:
name: 'Stage 0: assert hotfix workflow omitted'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- name: Assert cascade-hotfix.yaml is absent for single-env
shell: bash
run: |
set -euo pipefail
HOTFIX=".github/workflows/cascade-hotfix.yaml"
if [ -e "$HOTFIX" ]; then
echo "::error::single-env must not emit a hotfix workflow, but $HOTFIX exists"
ls -l "$HOTFIX" || true
exit 1
fi
# Defensive: no other hotfix-named workflow either.
STRAY="$(find .github/workflows -maxdepth 1 -iname '*hotfix*' 2>/dev/null || true)"
if [ -n "$STRAY" ]; then
echo "::error::single-env must not emit any hotfix workflow, found: $STRAY"
exit 1
fi
echo "OK: no hotfix workflow generated for the single-env topology"
{
echo "## Stage 0: hotfix omission"
echo "- cascade-hotfix.yaml absent for single-env: yes"
} >> "$GITHUB_STEP_SUMMARY"
reset:
name: 'Stage 1: merge create-draft'
runs-on: ubuntu-latest
outputs:
merge_sha: ${{ steps.merge.outputs.merge_sha }}
# Captured before anything is dispatched. The reconcile job enumerates
# every run this repo produced at or after this instant and fails if any
# is unaccounted for in the ledger. Recorded first so no run the suite
# causes can fall outside the window.
window_start: ${{ steps.window.outputs.window_start }}
steps:
- name: Log cascade version mode
run: |
if [ -n "${{ inputs.cascade_version }}" ]; then
echo "Running against dispatched rc: ${{ inputs.cascade_version }} (sha ${{ inputs.cascade_version_sha }})"
else
echo "Running against committed defaults (no rc dispatched)"
fi
- name: Install gh transient-retry wrapper
run: |
cat > "$RUNNER_TEMP/gh-retry.sh" <<'GHRETRY'
_gh_is_transient() {
local out="$1"
if printf '%s' "$out" | grep -qiE 'HTTP 5[0-9][0-9]|HTTP 429|HTTP 401|Bad credentials|was submitted too quickly|secondary rate limit'; then
return 0
fi
if printf '%s' "$out" | grep -qiE 'HTTP 403'; then
if printf '%s' "$out" | grep -qiE 'rate limit|secondary|abuse|too quickly'; then
return 0
fi
fi
return 1
}
gh() {
local attempt=1 max="${GH_RETRY_MAX:-5}" delay="${GH_RETRY_BASE_DELAY:-3}" out rc
while :; do
out="$(command gh "$@" 2>&1)" && rc=0 || rc=$?
if [ "$rc" -eq 0 ]; then
printf '%s\n' "$out"
return 0
fi
if [ "$attempt" -ge "$max" ] || ! _gh_is_transient "$out"; then
printf '%s\n' "$out" >&2
return "$rc"
fi
printf 'gh: transient error on attempt %d/%d, retrying in %ds\n%s\n' "$attempt" "$max" "$delay" "$out" >&2
sleep "$delay"
attempt=$((attempt + 1))
delay=$((delay * 2))
done
}
GHRETRY
echo "BASH_ENV=$RUNNER_TEMP/gh-retry.sh" >> "$GITHUB_ENV"
# The reconcile window opens here, before the first PR merge. Every run
# the suite goes on to cause (orchestrate on merge, both Release
# dispatches) lands at or after this timestamp, so reconcile sees them.
- name: Open reconcile window
id: window
shell: bash
run: |
set -euo pipefail
WINDOW_START="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "window_start=$WINDOW_START" >> "$GITHUB_OUTPUT"
echo "reconcile window opened at $WINDOW_START"
- uses: actions/checkout@v6
with:
token: ${{ secrets.CASCADE_STATE_TOKEN }}
fetch-depth: 0
- name: Self-repin manifest to the dispatched rc
uses: stablekernel/cascade/.github/actions/fleet-repin@main
with:
cascade_version: ${{ inputs.cascade_version }}
cascade_version_sha: ${{ inputs.cascade_version_sha }}
token: ${{ secrets.CASCADE_STATE_TOKEN }}
- name: Clean slate - delete leftover releases and tags
env:
GH_TOKEN: ${{ secrets.CASCADE_STATE_TOKEN }}
run: |
gh release list --repo "$GITHUB_REPOSITORY" --limit 200 --json tagName --jq '.[].tagName' \
| while read -r t; do gh release delete "$t" --repo "$GITHUB_REPOSITORY" --yes --cleanup-tag 2>/dev/null || true; done
git fetch --tags --quiet || true
for t in $(git tag -l 'v*' 'rel-*'); do git push origin --delete "$t" 2>/dev/null || true; done
- name: Open and merge a source change
id: merge
shell: bash
env:
TRUNK: main
run: |
set -euo pipefail
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
BRANCH="scenario/src-$(date +%s)"
git checkout -b "$BRANCH"
mkdir -p src
echo "touched at $(date -u +%FT%TZ)" >> src/scenario.txt
git add src/scenario.txt
git commit --no-gpg-sign -m "feat: scenario source change"
git push origin "$BRANCH"
PR_URL="$(gh pr create --base "$TRUNK" --head "$BRANCH" \
--title "feat: scenario source change" \
--body "Automated scenario run. Drives orchestrate on merge.")"
echo "Opened $PR_URL"
gh pr merge "$BRANCH" --rebase --delete-branch
git fetch origin "$TRUNK"
MERGE_SHA="$(git rev-parse "origin/$TRUNK")"
echo "merge_sha=$MERGE_SHA" >> "$GITHUB_OUTPUT"
- name: Wait for orchestrate run on merge SHA
id: orchestrate
shell: bash
env:
MERGE_SHA: ${{ steps.merge.outputs.merge_sha }}
run: |
set -euo pipefail
RUN_ID=""
for i in $(seq 1 6); do
RUN_ID="$(gh run list --workflow=orchestrate.yaml --branch=main \
--json databaseId,headSha,status \
--jq ".[] | select(.headSha==\"$MERGE_SHA\") | .databaseId" | head -n1 || true)"
[ -n "$RUN_ID" ] && break
echo "attempt $i: no orchestrate run for $MERGE_SHA yet"
sleep 60
done
[ -n "$RUN_ID" ] || { echo "::error::no orchestrate run found for $MERGE_SHA"; exit 1; }
# Surface the resolved run id so the next step can register it into the
# fleet ledger before we block on it.
echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT"
gh run watch "$RUN_ID" --exit-status --interval 60
# Register the orchestrate run into the fleet ledger. upload: true so this
# job's ledger reaches the reconcile job as an artifact (multi-job suite).
- name: Register orchestrate run
uses: stablekernel/cascade/.github/actions/register-run@main
with:
run-id: ${{ steps.orchestrate.outputs.run_id }}
expected-conclusion: success
reason: stage1-orchestrate-on-merge
upload: 'true'
- name: Assert draft release and state rc.0
shell: bash
run: |
set -euo pipefail
ok=""
for i in $(seq 1 10); do
tags="$(git ls-remote --tags origin 2>/dev/null | sed 's#.*refs/tags/##' || true)"
rc_tag="$(echo "$tags" | grep -E '.*-rc\.[0-9]+$' | head -n1 || true)"
draft="$(gh release list --limit 20 --json tagName,isDraft \
--jq '.[] | select(.isDraft == true) | .tagName' 2>/dev/null | head -n1 || true)"
if [ -n "$rc_tag" ] && [ -n "$draft" ]; then
echo "Found rc tag '$rc_tag' and draft release '$draft'"
ok="yes"
break
fi
echo "attempt $i: waiting for draft + rc tag (rc='$rc_tag' draft='$draft')"
sleep 60
done
if [ -z "$ok" ]; then
echo "::error::Stage 1 timed out waiting for draft release and rc tag"
exit 1
fi
{
echo "## Stage 1: create-draft"
echo "- rc tag present: yes (pattern *-rc.N)"
echo "- draft release present: yes"
} >> "$GITHUB_STEP_SUMMARY"
prerelease:
name: 'Stage 2: dispatch prerelease'
runs-on: ubuntu-latest
needs: reset
steps:
- name: Install gh transient-retry wrapper
run: |
cat > "$RUNNER_TEMP/gh-retry.sh" <<'GHRETRY'
_gh_is_transient() {
local out="$1"
if printf '%s' "$out" | grep -qiE 'HTTP 5[0-9][0-9]|HTTP 429|HTTP 401|Bad credentials|was submitted too quickly|secondary rate limit'; then
return 0
fi
if printf '%s' "$out" | grep -qiE 'HTTP 403'; then
if printf '%s' "$out" | grep -qiE 'rate limit|secondary|abuse|too quickly'; then
return 0
fi
fi
return 1
}
gh() {
local attempt=1 max="${GH_RETRY_MAX:-5}" delay="${GH_RETRY_BASE_DELAY:-3}" out rc
while :; do
out="$(command gh "$@" 2>&1)" && rc=0 || rc=$?
if [ "$rc" -eq 0 ]; then
printf '%s\n' "$out"
return 0
fi
if [ "$attempt" -ge "$max" ] || ! _gh_is_transient "$out"; then
printf '%s\n' "$out" >&2
return "$rc"
fi
printf 'gh: transient error on attempt %d/%d, retrying in %ds\n%s\n' "$attempt" "$max" "$delay" "$out" >&2
sleep "$delay"
attempt=$((attempt + 1))
delay=$((delay * 2))
done
}
GHRETRY
echo "BASH_ENV=$RUNNER_TEMP/gh-retry.sh" >> "$GITHUB_ENV"
- uses: actions/checkout@v6
with:
token: ${{ secrets.CASCADE_STATE_TOKEN }}
- name: Dispatch Release with release_action=prerelease
id: dispatch
shell: bash
run: |
set -euo pipefail
TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
gh workflow run "${RELEASE_WORKFLOW}" \
-f release_action=prerelease
RUN_ID=""
for i in $(seq 1 6); do
RUN_ID="$(gh run list --workflow="${RELEASE_WORKFLOW}" --created ">=$TS" \
--json databaseId --jq '.[0].databaseId')"
[ -n "$RUN_ID" ] && break
echo "attempt $i: no ${RELEASE_WORKFLOW} run dispatched since $TS yet"
sleep 60
done
[ -n "$RUN_ID" ] || { echo "::error::no ${RELEASE_WORKFLOW} run found after prerelease dispatch"; exit 1; }
# Surface the resolved run id so the next step can register it into the
# fleet ledger before we block on it.
echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT"
gh run watch "$RUN_ID" --exit-status --interval 60
# Register the prerelease Release run into the fleet ledger.
- name: Register prerelease Release run
uses: stablekernel/cascade/.github/actions/register-run@main
with:
run-id: ${{ steps.dispatch.outputs.run_id }}
expected-conclusion: success
reason: stage2-release-prerelease
upload: 'true'
- name: Assert release becomes prerelease
shell: bash
run: |
set -euo pipefail
ok=""
for i in $(seq 1 10); do
pre="$(gh release list --limit 20 --json tagName,isDraft,isPrerelease \
--jq '.[] | select(.isPrerelease == true and .isDraft == false) | .tagName' \
2>/dev/null | head -n1 || true)"
if [ -n "$pre" ]; then
echo "Found prerelease '$pre' (prerelease=true draft=false)"
ok="yes"
break
fi
echo "attempt $i: waiting for prerelease flag"
sleep 60
done
if [ -z "$ok" ]; then
echo "::error::Stage 2 timed out waiting for prerelease state"
exit 1
fi
{
echo "## Stage 2: prerelease"
echo "- release prerelease=true, draft=false: yes"
} >> "$GITHUB_STEP_SUMMARY"
publish:
name: 'Stage 3: dispatch release'
runs-on: ubuntu-latest
needs: [reset, prerelease]
steps:
- name: Install gh transient-retry wrapper
run: |
cat > "$RUNNER_TEMP/gh-retry.sh" <<'GHRETRY'
_gh_is_transient() {
local out="$1"
if printf '%s' "$out" | grep -qiE 'HTTP 5[0-9][0-9]|HTTP 429|HTTP 401|Bad credentials|was submitted too quickly|secondary rate limit'; then
return 0
fi
if printf '%s' "$out" | grep -qiE 'HTTP 403'; then
if printf '%s' "$out" | grep -qiE 'rate limit|secondary|abuse|too quickly'; then
return 0
fi
fi
return 1
}
gh() {
local attempt=1 max="${GH_RETRY_MAX:-5}" delay="${GH_RETRY_BASE_DELAY:-3}" out rc
while :; do
out="$(command gh "$@" 2>&1)" && rc=0 || rc=$?
if [ "$rc" -eq 0 ]; then
printf '%s\n' "$out"
return 0
fi
if [ "$attempt" -ge "$max" ] || ! _gh_is_transient "$out"; then
printf '%s\n' "$out" >&2
return "$rc"
fi
printf 'gh: transient error on attempt %d/%d, retrying in %ds\n%s\n' "$attempt" "$max" "$delay" "$out" >&2
sleep "$delay"
attempt=$((attempt + 1))
delay=$((delay * 2))
done
}
GHRETRY
echo "BASH_ENV=$RUNNER_TEMP/gh-retry.sh" >> "$GITHUB_ENV"
- uses: actions/checkout@v6
with:
token: ${{ secrets.CASCADE_STATE_TOKEN }}
fetch-depth: 0
- name: Setup CLI
uses: stablekernel/cascade/.github/actions/setup-cli@v0.12.0
with:
token: ${{ secrets.GITHUB_TOKEN }}
version: ${{ inputs.cascade_version || 'v0.8.0' }}
- name: Dispatch Release with release_action=release
id: dispatch
shell: bash
run: |
set -euo pipefail
TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
gh workflow run "${RELEASE_WORKFLOW}" \
-f release_action=release
RUN_ID=""
for i in $(seq 1 6); do
RUN_ID="$(gh run list --workflow="${RELEASE_WORKFLOW}" --created ">=$TS" \
--json databaseId --jq '.[0].databaseId')"
[ -n "$RUN_ID" ] && break
echo "attempt $i: no ${RELEASE_WORKFLOW} run dispatched since $TS yet"
sleep 60
done
[ -n "$RUN_ID" ] || { echo "::error::no ${RELEASE_WORKFLOW} run found after release dispatch"; exit 1; }
# Surface the resolved run id so the next step can register it into the
# fleet ledger before we block on it.
echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT"
gh run watch "$RUN_ID" --exit-status --interval 60
# Register the publish Release run into the fleet ledger.
- name: Register publish Release run
uses: stablekernel/cascade/.github/actions/register-run@main
with:
run-id: ${{ steps.dispatch.outputs.run_id }}
expected-conclusion: success
reason: stage3-release-publish
upload: 'true'
- name: Assert published latest, rc cleaned, v tag present
id: published
shell: bash
run: |
set -euo pipefail
ok=""
v_tag=""
for i in $(seq 1 10); do
git fetch --tags --force origin >/dev/null 2>&1 || true
tags="$(git ls-remote --tags origin 2>/dev/null | sed 's#.*refs/tags/##' || true)"
v_tag="$(echo "$tags" | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | head -n1 || true)"
rc_left="$(echo "$tags" | grep -E '.*-rc\.[0-9]+$' | head -n1 || true)"
published="$(gh release list --limit 20 --json tagName,isDraft,isPrerelease \
--jq '.[] | select(.isDraft == false and .isPrerelease == false) | .tagName' \
2>/dev/null | head -n1 || true)"
latest="$(gh api "repos/${{ github.repository }}/releases/latest" \
--jq '.tag_name' 2>/dev/null || true)"
if [ -n "$v_tag" ] && [ -z "$rc_left" ] && [ -n "$published" ] && [ -n "$latest" ]; then
echo "Published '$published', latest '$latest', v tag '$v_tag', rc cleaned"
ok="yes"
break
fi
echo "attempt $i: v='$v_tag' rc_left='$rc_left' published='$published' latest='$latest'"
sleep 60
done
if [ -z "$ok" ]; then
echo "::error::Stage 3 timed out waiting for published release and rc cleanup"
exit 1
fi
# Carry the exact published version forward as the known value the prod
# state pointer must record.
echo "published_version=$v_tag" >> "$GITHUB_OUTPUT"
{
echo "## Stage 3: release"
echo "- v tag present (pattern vN.N.N): yes"
echo "- rc tags cleaned: yes"
echo "- release published as latest: yes"
} >> "$GITHUB_STEP_SUMMARY"
- name: Assert prod state and published pointer carry the known values
shell: bash
env:
EXPECTED_VERSION: ${{ steps.published.outputs.published_version }}
EXPECTED_SHA: ${{ needs.reset.outputs.merge_sha }}
run: |
set -euo pipefail
# In a single declared env (prod, terminal), release_action=release does
# not deploy: it publishes the prod RC as the semver release. The prod
# state pointer keeps the SHA built on merge (state.prod.sha == the Stage 1
# trunk merge SHA), while the published version lands in the latest_release
# pointer (promote.yaml writes latest_release.version = the semver tag and
# latest_release.sha = state.prod.sha). Assert both against KNOWN values,
# read through cascade status (the .state. / .latest_release. paths), not
# a grep of the manifest.
if [ -z "$EXPECTED_VERSION" ]; then
echo "::error::expected published version is empty (Stage 3 did not capture v tag)"
exit 1
fi
if [ -z "$EXPECTED_SHA" ]; then
echo "::error::expected merge SHA from Stage 1 is empty"
exit 1
fi
# finalize pushes the latest_release state commit shortly after the
# release run reports completed, so poll-pull trunk until both pointers
# report the exact known values.
ok=""
STATE_SHA=""
STATE_VERSION=""
REL_VERSION=""
REL_SHA=""
for i in $(seq 1 6); do
git pull origin main --quiet || true
ENV_JSON="$(cascade status env prod --config "$MANIFEST" --key "$MANIFEST_KEY" --json)"
ALL_JSON="$(cascade status --config "$MANIFEST" --key "$MANIFEST_KEY" --json)"
STATE_SHA="$(echo "$ENV_JSON" | jq -r '.state.sha // ""')"
STATE_VERSION="$(echo "$ENV_JSON" | jq -r '.state.version // ""')"
REL_VERSION="$(echo "$ALL_JSON" | jq -r '.latest_release.version // ""')"
REL_SHA="$(echo "$ALL_JSON" | jq -r '.latest_release.sha // ""')"
if [ "$STATE_SHA" = "$EXPECTED_SHA" ] \
&& [ "$REL_VERSION" = "$EXPECTED_VERSION" ] \
&& [ "$REL_SHA" = "$EXPECTED_SHA" ]; then
ok="yes"
break
fi
echo "attempt $i: state.sha='$STATE_SHA' latest_release.version='$REL_VERSION' latest_release.sha='$REL_SHA'"
sleep 60
done
if [ -z "$ok" ]; then
echo "::error::prod/published pointers did not record the known values"
echo "::error::state.prod.sha: got '$STATE_SHA', want '$EXPECTED_SHA'"
echo "::error::latest_release.version: got '$REL_VERSION', want '$EXPECTED_VERSION'"
echo "::error::latest_release.sha: got '$REL_SHA', want '$EXPECTED_SHA'"
exit 1
fi
# The prod state version is the RC the env was built on; assert it is a
# real, non-empty semver RC (the rc-base of the published version), not a
# placeholder. This is a knowable shape (the published version with an
# -rc.N suffix), so accept the rc family rather than a bare regex.
case "$STATE_VERSION" in
"${EXPECTED_VERSION}-rc."*) ;;
*)
echo "::error::state.prod.version '$STATE_VERSION' is not an rc of published $EXPECTED_VERSION"
exit 1
;;
esac
echo "✓ state.prod.sha == $STATE_SHA (merge SHA); state.prod.version == $STATE_VERSION"
echo "✓ latest_release.version == $REL_VERSION; latest_release.sha == $REL_SHA"
{
echo "## Stage 3: prod state pointer"
echo "- state.prod.sha == trunk merge SHA: yes"
echo "- state.prod.version == rc of $EXPECTED_VERSION: yes"
echo "- latest_release.version == published $EXPECTED_VERSION: yes"
echo "- latest_release.sha == trunk merge SHA: yes"
} >> "$GITHUB_STEP_SUMMARY"
# ---------------------------------------------------------------------
# Reconcile gate: structural coverage backstop. The per-stage asserts above
# still gate each run they wait on (every dispatched orchestrate/Release run
# is wrapped in `gh run watch --exit-status`); this job is additive. It
# enumerates EVERY run this repo produced since window-start (captured before
# the first merge) and fails if any is unaccounted for in the ledger the
# register-run steps uploaded - an unregistered non-success run, or a
# registered run that concluded other than its expected conclusion. That turns
# any future fire-and-forget run the suite forgot to gate into a hard red.
reconcile:
name: Reconcile scenario-window runs
needs: [reset, prerelease, publish]
if: always()
uses: stablekernel/cascade/.github/workflows/fleet-reconcile.yaml@main
permissions:
contents: read
actions: read
with:
window-start: ${{ needs.reset.outputs.window_start }}
# Artifact mode: each register-run step uploaded a per-job ledger under
# the default cascade-run-ledger-* name; reconcile globs and merges them.
cascade-ref: main