diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c5ddd32 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,43 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +# CI is the only reason a bot needs to read this repo — no write access. +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + smoke: + name: Smoke tests (scanner + dashboard) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "20" + cache: "npm" + + - name: Install dependencies + run: npm ci + + # Run the scanner against this repo itself. Catches scanner-side + # regressions end-to-end: scan rules, policy rendering, framework + # evaluation, and report generation all exercised on a real tree. + - name: Scanner smoke + run: npm run scan -- . + + # Dashboard render paths get their own smoke test with two fixtures: + # a current-shape manifest and a pre-Phase-8 manifest missing the + # aiSystems field. The second fixture guards against the 2026-04-18 + # outage where an older stored manifest took /api/repos and the + # homepage down with an uncaught TypeError. + - name: Dashboard render smoke + run: npx tsx scripts/smoke-dashboard.ts diff --git a/docs/implementation-checklist.md b/docs/implementation-checklist.md index 5dbc9c8..f31e8cb 100644 --- a/docs/implementation-checklist.md +++ b/docs/implementation-checklist.md @@ -479,8 +479,9 @@ These cut across all phases and should be addressed opportunistically. ### Unfixed Technical Debt - [ ] joeeftekhari.com has 1 critical + 3 high CVEs we haven't addressed. We don't use our own tool's output. - [ ] joeeftekhari.com doesn't yet serve policies at public URLs — files exist in `docs/policies/` but no Express routes to serve them. Once routes exist, user will configure `policy_urls` and Check Production will verify them. -- [ ] No tests exist anywhere in the project -- [ ] No CI beyond deploy (no lint, no type-check) +- [ ] No unit tests — smoke tests exist (scripts/smoke-dashboard.ts covers render pipeline) but no assertions beyond "did not throw / no bare 'undefined' in output". A real unit test suite (Vitest or similar) is still open. +- [x] CI on every PR — `.github/workflows/ci.yml` runs the scanner against this repo and smoke-tests every dashboard render function with both new-shape and old-shape manifest fixtures. Catches regressions like the 2026-04-18 outage. +- [ ] Lint / type-check still missing — `tsc --noEmit` fights with `verbatimModuleSyntax` + the no-@types/node config the runtime relies on (`tsx` handles this at runtime but `tsc` doesn't). Adding a lint pass is tracked separately. - [ ] Documentation in `docs/` has overlapping content across files - [ ] Deploy workflow uses `sed` for placeholder injection in wrangler.toml — works but is fragile - [ ] Monorepo support is poor (scans root only, no per-package awareness) diff --git a/scripts/smoke-dashboard.ts b/scripts/smoke-dashboard.ts new file mode 100644 index 0000000..0f95ead --- /dev/null +++ b/scripts/smoke-dashboard.ts @@ -0,0 +1,250 @@ +/** + * Smoke test for dashboard render paths. + * + * Not a replacement for real unit tests — just catches obvious runtime + * regressions (throws, null-deref, missing fields) that would take / + * down in production. Runs in CI on every PR. + * + * Strategy: construct one new-shape manifest and one old-shape manifest + * (simulating data stored in KV by an older scanner version), feed both + * through every exported dashboard render function, and ensure none + * throws. The old-shape fixture specifically exercises the hot path + * that caused the 2026-04-18 outage: `manifest.aiSystems` missing + * entirely because the manifest predates Phase 8A. + */ + +import type { Manifest } from "../scanner/types.js"; +import type { RepoSummary, HistoryEntry } from "../dashboard/worker.js"; +import { + renderDashboard, + renderRepoDetail, + renderNistView, + renderBranchComparison, + renderTrendChart, + renderAIComplianceView, + renderInventoryView, +} from "../dashboard/views/render.js"; +import { + evaluateEUAIAct, + calcAIComplianceScore, + getAIPhaseScores, +} from "../scanner/frameworks/eu-ai-act.js"; +import { evaluateFramework } from "../scanner/generators/framework-report.js"; + +function fail(label: string, err: unknown): never { + console.error(`✗ ${label}`); + console.error(err); + process.exit(1); +} + +function ok(label: string) { + console.log(`✓ ${label}`); +} + +function baseArtifacts(): Manifest["artifacts"] { + return { + privacyPolicy: "generated", + termsOfService: "generated", + securityTxt: "present", + vulnerabilityDisclosure: "present", + incidentResponsePlan: "present", + }; +} + +/** Current-shape manifest — everything the Phase 8 scanner emits. */ +function newShapeManifest(): Manifest { + return { + repo: "smoke/new", + scanDate: "2026-04-18T00:00:00Z", + branch: "main", + commit: "abc1234", + dataCollection: [], + thirdPartyServices: [], + securityHeaders: null, + https: null, + dependencies: null, + secretsScan: { detected: false, findings: [] }, + artifacts: { + ...baseArtifacts(), + aiUsagePolicy: "present", + modelCards: "not-applicable", + fria: "not-applicable", + }, + accessControls: { branchProtection: true, requiredReviews: 1, signedCommits: false }, + aiSystems: [ + { + provider: "OpenAI", + sdk: "openai", + location: "package.json", + category: "inference", + dataFlows: [], + riskTier: "limited", + riskTierSource: "heuristic", + riskReasoning: "Category-default classification.", + euMarket: true, + }, + ], + }; +} + +/** + * Pre-Phase-8 shape: no `aiSystems` field, no AI artifact fields. Mirrors + * what a scanner from before 2026-04-17 would have written into KV. The + * 2026-04-18 outage was a failure to handle this case. + */ +function oldShapeManifest(): Manifest { + return { + repo: "smoke/old", + scanDate: "2025-11-01T00:00:00Z", + branch: "main", + commit: "def5678", + dataCollection: [], + thirdPartyServices: [], + securityHeaders: null, + https: null, + dependencies: null, + secretsScan: { detected: false, findings: [] }, + artifacts: baseArtifacts(), + accessControls: { branchProtection: true, requiredReviews: 1, signedCommits: false }, + } as unknown as Manifest; // intentionally missing aiSystems +} + +function summaryFor(m: Manifest): RepoSummary { + const nistResults = evaluateFramework(m); + const applicable = nistResults.filter(r => r.status !== "not-applicable"); + const passed = applicable.filter(r => r.status === "pass").length; + const partial = applicable.filter(r => r.status === "partial").length; + const nistScore = applicable.length > 0 + ? Math.round(((passed + partial * 0.5) / applicable.length) * 100) + : 0; + return { + repo: m.repo, + branch: m.branch, + commit: m.commit, + scanDate: m.scanDate, + dataCollectionCount: m.dataCollection.length, + thirdPartyCount: m.thirdPartyServices.length, + secretsDetected: m.secretsScan.detected, + headersPresent: 0, + headersTotal: 0, + httpsEnforced: null, + certExpiry: null, + criticalVulns: 0, + highVulns: 0, + complianceScore: 75, + nistScore, + nistResults, + artifacts: m.artifacts, + aiScore: calcAIComplianceScore(evaluateEUAIAct(m)), + aiSystemCount: (m.aiSystems ?? []).length, + aiHighRiskCount: 0, + siteUrl: "https://example.com", + }; +} + +function functionScoresFor(summary: RepoSummary) { + return ["Identify", "Protect", "Detect", "Respond", "Recover"].map(name => { + const controls = summary.nistResults.filter(r => r.control.function === name); + const applicable = controls.filter(r => r.status !== "not-applicable"); + const passed = applicable.filter(r => r.status === "pass").length; + const partial = applicable.filter(r => r.status === "partial").length; + const failed = applicable.filter(r => r.status === "fail").length; + const percentage = applicable.length > 0 + ? Math.round(((passed + partial * 0.5) / applicable.length) * 100) + : 100; + return { name, percentage, passed, partial, failed }; + }); +} + +function historyFor(m: Manifest, aiCount: number): HistoryEntry[] { + return [ + { + repo: m.repo, branch: m.branch, commit: m.commit, scanDate: m.scanDate, + complianceScore: 75, nistScore: 80, criticalVulns: 0, highVulns: 0, + headersPresent: 0, headersTotal: 6, + aiScore: aiCount > 0 ? 67 : undefined, + aiSystemCount: aiCount, + }, + ]; +} + +function assertHtml(html: string, label: string): void { + if (typeof html !== "string") fail(label, `expected string, got ${typeof html}`); + if (html.length < 100) fail(label, `HTML suspiciously short (${html.length} bytes)`); + // "undefined" appearing in an HTML TEXT node (`>undefined<`) or as an + // attribute value (`="undefined"`) is almost always a bug where a template + // interpolated a missing field. We don't ban the token outright because + // inline