Automated regression test suite for validator consensus failure modes

.github/workflows/consensus-sim.yml

@@ -0,0 +1,25 @@
+name: Consensus Simulation Tests
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  consensus-sim:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '18'
+      - name: Install deps
+        run: npm ci
+      - name: Run consensus simulation
+        run: npm run test:consensus-sim
+      - name: Upload JUnit results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: consensus-sim-junit
+          path: tests-results/consensus-sim-junit.xml

TODO.md

@@ -0,0 +1,16 @@
+# TODO - Config Runtime Auditing (Drift Detection)
+
+- [ ] Implement baseline loader (read committed baseline config files; start with config.json.example)
+- [ ] Implement runtime snapshot collector (interval 5 minutes; uses getConfig())
+- [ ] Implement diff comparator (value changes + added/removed keys) using stable flattening
+- [ ] Implement snapshot storage (in-memory + JSONL on disk optional via env)
+- [ ] Implement PagerDuty notifier + alert routing policy (deployment-scoped critical drift)
+- [x] Wire auditor into index.js after config init
+
+- [x] Add dashboard endpoints: GET /debug/config-drift and history
+
+- [x] Add unit tests for flatten+diff and alert policy
+
+
+- [ ] Run tests and ensure build passes
+

index.js

@@ -56,7 +56,43 @@ async function bootstrap() {
     console.warn('[index] Config module not loaded; running with env defaults');
   }
 
+  // 1b. Start config drift auditor + expose debug endpoints
+  const driftModule = loadTsModule('config-drift/auditor');
+  const driftRoutesModule = loadTsModule('config-drift/routes');
+  if (driftModule && driftRoutesModule) {
+    try {
+      const { createConfigDriftAuditorFromEnv } = driftModule;
+      const { registerConfigDriftRoutes } = driftRoutesModule;
+
+      const auditor = createConfigDriftAuditorFromEnv({});
+      auditor.init().then(() => {
+        auditor.start();
+        registerConfigDriftRoutes(app, auditor);
+        console.log('[config-drift] Auditor started');
+      });
+
+
+      // best-effort shutdown hook
+      const shutdownHandler = () => {
+
+        try {
+          auditor.stop();
+        } catch {
+          // noop
+        }
+      };
+      process.once('SIGINT', shutdownHandler);
+      process.once('SIGTERM', shutdownHandler);
+
+    } catch (err) {
+      console.warn('[config-drift] Failed to start auditor:', (err && err.message) ? err.message : String(err));
+    }
+  } else {
+    console.warn('[config-drift] Drift modules not loaded');
+  }
+
   // 2. Initialize tracing
+
   const tracing = loadTsModule('diagnostics/tracer');
   if (tracing && typeof tracing.initTracingFromConfig === 'function') {
     const otelCfg = getConfigValue ? getConfigValue('telemetry.otel') : null;

package.json

@@ -5,7 +5,8 @@
   "main": "dist/index.js",
   "scripts": {
     "build": "tsc",
-    "test": "npx ts-node --project tsconfig.json tests/config.test.ts && npx ts-node --project tsconfig.json tests/slashing_sequencer.test.ts && npx ts-node --project tsconfig.json tests/notifications/slashingNotifier.test.ts && npx ts-node --project tsconfig.json tests/staking/bondPool.test.ts && npx ts-node --project tsconfig.json tests/uptime_queries.test.ts && npx ts-node --project tsconfig.json tests/tracer.test.ts && npx ts-node --project tsconfig.json tests/pool_isolation.test.ts && npx ts-node --project tsconfig.json tests/mtls.test.ts && npx ts-node --project tsconfig.json tests/tls_rotation.test.ts && npx ts-node --project tsconfig.json tests/reputation/scoreService.test.ts && npx ts-node --project tsconfig.json tests/rewards/distributor.test.ts && npx ts-node --project tsconfig.json tests/blockchain/preflight_analyzer.test.ts && npx ts-node --project tsconfig.json tests/blockchain/rpc_client.test.ts && npx ts-node --project tsconfig.json tests/blockchain/transaction_builder.test.ts && npx ts-node --project tsconfig.json tests/state_archival.test.ts && npx ts-node --project tsconfig.json tests/dead_letter_queue.test.ts && TS_NODE_TRANSPILE_ONLY=true npx ts-node --project tsconfig.json tests/core/attestation/engine_extra.test.ts",
+    "test": "npx ts-node --project tsconfig.json tests/config.test.ts && npx ts-node --project tsconfig.json tests/slashing_sequencer.test.ts && npx ts-node --project tsconfig.json tests/notifications/slashingNotifier.test.ts && npx ts-node --project tsconfig.json tests/staking/bondPool.test.ts && npx ts-node --project tsconfig.json tests/uptime_queries.test.ts && npx ts-node --project tsconfig.json tests/tracer.test.ts && npx ts-node --project tsconfig.json tests/pool_isolation.test.ts && npx ts-node --project tsconfig.json tests/mtls.test.ts && npx ts-node --project tsconfig.json tests/tls_rotation.test.ts && npx ts-node --project tsconfig.json tests/reputation/scoreService.test.ts && npx ts-node --project tsconfig.json tests/rewards/distributor.test.ts && npx ts-node --project tsconfig.json tests/blockchain/preflight_analyzer.test.ts && npx ts-node --project tsconfig.json tests/blockchain/rpc_client.test.ts && npx ts-node --project tsconfig.json tests/blockchain/transaction_builder.test.ts && npx ts-node --project tsconfig.json tests/state_archival.test.ts && npx ts-node --project tsconfig.json tests/dead_letter_queue.test.ts && npx ts-node --project tsconfig.json tests/config-drift.test.ts && TS_NODE_TRANSPILE_ONLY=true npx ts-node --project tsconfig.json tests/core/attestation/engine_extra.test.ts",
+    "test:consensus-sim": "npx ts-node --project tsconfig.json tests/consensus_sim.test.ts",
     "test:reputation": "npx ts-node --project tsconfig.json tests/reputation/scoreService.test.ts",
     "clean": "rm -rf dist"
   },

src/config-drift/auditor.ts

@@ -0,0 +1,135 @@
+import { getConfig } from '../config';
+import { loadBaselineSnapshot, ExampleConfigBaselineSource } from './baseline';
+import { computeDriftReport, pickCriticalPrefix } from './diff';
+import { DriftStorage } from './storage';
+import { CriticalDriftPolicy, ConfigDriftAlert } from './types';
+import { HttpPagerDutyClient, buildAlertIfCritical, PagerDutyOptions, PagerDutyClient } from './pagerduty';
+
+export interface ConfigDriftAuditorOptions {
+  intervalMs?: number;
+  baselineSources?: Array<{ loadBaseline(): Promise<unknown>; name: string }>;
+  storage?: DriftStorage;
+  pagerDutyClient?: PagerDutyClient;
+  criticalPolicy: CriticalDriftPolicy;
+  driftCategoryFilter?: 'all';
+}
+
+export class ConfigDriftAuditor {
+  private timer: NodeJS.Timeout | null = null;
+  private baseline: Awaited<ReturnType<typeof loadBaselineSnapshot>> | null = null;
+  private running = false;
+
+  constructor(private readonly options: ConfigDriftAuditorOptions) {
+    this.options.storage = this.options.storage ?? new DriftStorage();
+  }
+
+  async init(): Promise<void> {
+    const baselineSources = this.options.baselineSources ?? [new ExampleConfigBaselineSource()];
+    this.baseline = await loadBaselineSnapshot(baselineSources as any);
+  }
+
+  start(): void {
+    const intervalMs = this.options.intervalMs ?? 5 * 60 * 1000;
+    // immediate run
+    void this.runOnce();
+    this.timer = setInterval(() => void this.runOnce(), intervalMs);
+    this.timer.unref?.();
+  }
+
+  stop(): void {
+    if (this.timer) clearInterval(this.timer);
+    this.timer = null;
+  }
+
+  history(limit = 100) {
+    return this.options.storage!.history(limit);
+  }
+
+  latest() {
+    return this.options.storage!.latest();
+  }
+
+  private async runOnce(): Promise<void> {
+    if (this.running) return;
+    this.running = true;
+
+    try {
+      if (!this.baseline) return;
+
+      const startedAt = Date.now();
+      const snapshotId = `cfgdrift:${startedAt}`;
+
+      const runtimeConfig = getConfig();
+
+      const report = computeDriftReport({
+        snapshotId,
+        runtimeConfig,
+        baselineFlattened: this.baseline.flattened,
+        baselineHash: this.baseline.baselineHash,
+      });
+
+      this.options.storage!.add({
+        snapshotId,
+        capturedAt: Date.now(),
+        driftReport: report,
+      });
+
+      const policyMatchedPrefix = pickCriticalPrefix(
+        this.options.criticalPolicy.criticalKeyPrefixes,
+        report.findings,
+      );
+
+      const alert = buildAlertIfCritical({
+        report,
+        policy: this.options.criticalPolicy,
+        policyMatchedPrefix,
+      });
+
+      if (alert && this.options.pagerDutyClient) {
+        await this.options.pagerDutyClient.triggerAlert(alert as ConfigDriftAlert);
+      }
+    } catch {
+      // Auditor errors should not crash the server.
+      // (Logger module not used here to avoid adding dependencies; can be wired later.)
+    } finally {
+      this.running = false;
+    }
+  }
+}
+
+export function createConfigDriftAuditorFromEnv(args: {
+  storage?: DriftStorage;
+}): ConfigDriftAuditor {
+  const enabledPagerDuty = process.env.VERINODE_DRIFT_PAGERDUTY_ENABLED === 'true' || process.env.VERINODE_DRIFT_PAGERDUTY_ENABLED === '1';
+  const routingKey = process.env.VERINODE_DRIFT_PAGERDUTY_ROUTING_KEY ?? '';
+
+  const criticalKeyPrefixes = (process.env.VERINODE_DRIFT_CRITICAL_PREFIXES ?? 'db,mtls,tls,app,remote')
+    .split(',')
+    .map((s) => s.trim())
+    .filter(Boolean);
+
+  const intervalMs = Number(process.env.VERINODE_DRIFT_SNAPSHOT_INTERVAL_MS ?? String(5 * 60 * 1000));
+
+  let pagerDutyClient: PagerDutyClient | undefined = undefined;
+  const criticalPolicy: CriticalDriftPolicy = {
+    enabled: process.env.VERINODE_DRIFT_ALERTS_ENABLED === 'true' || process.env.VERINODE_DRIFT_ALERTS_ENABLED === '1',
+    criticalKeyPrefixes,
+  };
+
+  if (enabledPagerDuty && routingKey) {
+    const pdOpts: PagerDutyOptions = {
+      enabled: enabledPagerDuty,
+      routingKey,
+      criticalPolicy,
+    };
+    pagerDutyClient = new HttpPagerDutyClient(pdOpts);
+  }
+
+  return new ConfigDriftAuditor({
+    intervalMs,
+    storage: args.storage,
+    pagerDutyClient,
+    criticalPolicy,
+  });
+}
+

src/config-drift/baseline.ts

@@ -0,0 +1,40 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { flattenConfig, computeHashFromFlattened } from './flatten';
+
+export interface BaselineConfigSource {
+  name: string;
+  /**
+   * Load baseline config object (committed source).
+   */
+  loadBaseline(): Promise<unknown>;
+}
+
+export class ExampleConfigBaselineSource implements BaselineConfigSource {
+  name = 'repo:config.json.example';
+  constructor(
+    private readonly examplePath: string = path.resolve(__dirname, '../../../config.json.example'),
+  ) {}
+
+  async loadBaseline(): Promise<unknown> {
+    const content = fs.readFileSync(this.examplePath, 'utf8');
+    return JSON.parse(content);
+  }
+}
+
+export interface BaselineSnapshot {
+  sourceName: string;
+  baselineConfig: unknown;
+  flattened: Record<string, string>;
+  baselineHash: string;
+}
+
+export async function loadBaselineSnapshot(sources: BaselineConfigSource[]): Promise<BaselineSnapshot> {
+  const source = sources[0];
+  if (!source) throw new Error('No baseline sources configured');
+  const baselineConfig = await source.loadBaseline();
+  const flattened = flattenConfig(baselineConfig);
+  const baselineHash = computeHashFromFlattened(flattened);
+  return { sourceName: source.name, baselineConfig, flattened, baselineHash };
+}
+

src/config-drift/diff.ts

@@ -0,0 +1,112 @@
+import { flattenConfig, computeHashFromFlattened, keyMatchesPrefix } from './flatten';
+import { DriftFinding, DriftReport } from './types';
+
+export interface DriftDiffInput {
+  runtimeConfig: unknown;
+  runtimeFlattened?: Record<string, string>;
+  baselineFlattened: Record<string, string>;
+  baselineHash: string;
+  snapshotId: string;
+}
+
+export function diffFlattenedConfigs(params: {
+  runtimeFlattened: Record<string, string>;
+  baselineFlattened: Record<string, string>;
+}): { findings: DriftFinding[] } {
+  const { runtimeFlattened, baselineFlattened } = params;
+
+  const baselineKeys = new Set(Object.keys(baselineFlattened));
+  const runtimeKeys = new Set(Object.keys(runtimeFlattened));
+
+  const findings: DriftFinding[] = [];
+
+  // Added keys
+  for (const k of runtimeKeys) {
+    if (!baselineKeys.has(k)) {
+      findings.push({
+        category: 'key_added',
+        key: k,
+        runtimeValue: runtimeFlattened[k],
+      });
+    }
+  }
+
+  // Removed keys
+  for (const k of baselineKeys) {
+    if (!runtimeKeys.has(k)) {
+      findings.push({
+        category: 'key_removed',
+        key: k,
+        baselineValue: baselineFlattened[k],
+      });
+    }
+  }
+
+  // Value changes
+  for (const k of runtimeKeys) {
+    if (!baselineKeys.has(k)) continue;
+    const b = baselineFlattened[k];
+    const r = runtimeFlattened[k];
+    if (b !== r) {
+      findings.push({
+        category: 'value_change',
+        key: k,
+        baselineValue: b,
+        runtimeValue: r,
+      });
+    }
+  }
+
+  return { findings };
+}
+
+export function computeDriftReport(input: DriftDiffInput): DriftReport {
+  const startedAt = Date.now();
+  const runtimeFlattened = input.runtimeFlattened ?? flattenConfig(input.runtimeConfig);
+  const runtimeHash = computeHashFromFlattened(runtimeFlattened);
+
+  const { findings } = diffFlattenedConfigs({
+    runtimeFlattened,
+    baselineFlattened: input.baselineFlattened,
+  });
+
+  const endedAt = Date.now();
+
+  const summary = {
+    total: findings.length,
+    valueChanges: findings.filter((f) => f.category === 'value_change').length,
+    keyAdded: findings.filter((f) => f.category === 'key_added').length,
+    keyRemoved: findings.filter((f) => f.category === 'key_removed').length,
+  };
+
+  return {
+    snapshotId: input.snapshotId,
+    startedAt,
+    endedAt,
+    runtimeHash,
+    baselineHash: input.baselineHash,
+    findings: findings.sort((a, b) => {
+      // deterministic ordering
+      const ord: Record<string, number> = {
+        value_change: 1,
+        key_added: 2,
+        key_removed: 3,
+      };
+      const oa = ord[a.category] ?? 99;
+      const ob = ord[b.category] ?? 99;
+      if (oa !== ob) return oa - ob;
+      return a.key.localeCompare(b.key);
+    }),
+    summary,
+  };
+}
+
+export function pickCriticalPrefix(policyPrefixes: string[], findings: DriftFinding[]): string | undefined {
+  for (const f of findings) {
+    for (const prefix of policyPrefixes) {
+      if (keyMatchesPrefix(f.key, prefix)) return prefix;
+    }
+  }
+  return undefined;
+}
+