From 59065cd4ca7beb479a16c1ea85314f0ae2ad714c Mon Sep 17 00:00:00 2001 From: David Szabo-Stuban Date: Wed, 3 Jun 2026 13:10:56 +0200 Subject: [PATCH 1/5] docs(backups): define restore contract and drill --- .../ctrl/docs/guides/backups-recovery.mdx | 426 +----------------- 1 file changed, 1 insertion(+), 425 deletions(-) diff --git a/packages/ctrl/docs/guides/backups-recovery.mdx b/packages/ctrl/docs/guides/backups-recovery.mdx index 5700149aa..602d3bc36 100644 --- a/packages/ctrl/docs/guides/backups-recovery.mdx +++ b/packages/ctrl/docs/guides/backups-recovery.mdx @@ -1,425 +1 @@ ---- -title: "Backups & Recovery" -description: "Backup system overview and disaster recovery" -icon: "shield" ---- - -Alfred uses restic with S3-compatible storage for automated encrypted backups. Backups run daily and include the entire vault, configuration, and worker state. - -## Backup System Overview - - - - Vault, config, worker state, OpenClaw devices - - - AES-256 with restic password - - - 7 daily, 4 weekly, 6 monthly - - - -## What Gets Backed Up - -The backup includes: -- **Vault**: All records (`/mnt/encrypted/vault/`) -- **Configuration**: `.env`, `openclaw.json` -- **State**: Worker state files, mutation logs, audit logs -- **OpenClaw**: Device pairing data, tokens - -**Excluded**: -- Temporal database (workflow history — not critical) -- Docker volumes (ephemeral) -- Logs older than 7 days - -Total backup size: typically 50-500 MB depending on vault size. - -## Automated Backups - -Backups run automatically every day at 3:00 AM UTC via cron job. - -**Duration**: 2-5 minutes depending on vault size and S3 latency. - -**Impact**: Containers are stopped during backup for consistency, then restarted automatically. Downtime: ~30 seconds. - - -The backup script stops all containers, runs `restic backup`, and restarts containers. This ensures a consistent snapshot. - - -## Manual Backup - -Trigger an immediate backup: - -```bash -curl -X POST https://alfred-acme.tailnet.ts.net:3100/api/backup/create \ - -H "X-API-Key: YOUR_API_KEY" -``` - -**Response (202 Accepted):** - -```json -{ - "ok": true, - "message": "Backup started", - "job_id": "backup_20260226_153000" -} -``` - - -The backup runs asynchronously. The API returns immediately (202) without waiting for completion. Monitor logs to verify success. - - -Check backup status in logs: - -```bash -curl -N https://alfred-acme.tailnet.ts.net:3100/api/logs/stream?service=backup&tail=50 \ - -H "X-API-Key: YOUR_API_KEY" -``` - -**Log output:** - -``` -data: {"timestamp":"2026-02-26T15:30:00Z","service":"backup","level":"info","message":"Stopping containers..."} -data: {"timestamp":"2026-02-26T15:30:05Z","service":"backup","level":"info","message":"Running restic backup..."} -data: {"timestamp":"2026-02-26T15:32:30Z","service":"backup","level":"info","message":"Backup complete: 245 MB in 2m 30s"} -data: {"timestamp":"2026-02-26T15:32:35Z","service":"backup","level":"info","message":"Restarting containers..."} -data: {"timestamp":"2026-02-26T15:32:40Z","service":"backup","level":"info","message":"Backup finished successfully"} -``` - -## Listing Snapshots - -Get all backup snapshots: - -```bash -curl -X GET https://alfred-acme.tailnet.ts.net:3100/api/backup/snapshots \ - -H "X-API-Key: YOUR_API_KEY" -``` - -**Response:** - -```json -{ - "ok": true, - "snapshots": [ - { - "id": "a3f5b2c1", - "time": "2026-02-26T03:00:00Z", - "hostname": "alfred-acme", - "paths": ["/mnt/encrypted"], - "tags": ["automated"], - "size_mb": 245 - }, - { - "id": "b7d9e8f2", - "time": "2026-02-25T03:00:00Z", - "hostname": "alfred-acme", - "paths": ["/mnt/encrypted"], - "tags": ["automated"], - "size_mb": 243 - }, - { - "id": "c2a1f4e3", - "time": "2026-02-26T15:30:00Z", - "hostname": "alfred-acme", - "paths": ["/mnt/encrypted"], - "tags": ["manual"], - "size_mb": 246 - } - ], - "total_snapshots": 18, - "total_size_gb": 4.3 -} -``` - -### Snapshot Tags - -- `automated` — daily cron job -- `manual` — triggered via API - -## Retention Policy - -Restic prunes old snapshots automatically: - -| Keep | Period | -|------|--------| -| **7** | Daily (last 7 days) | -| **4** | Weekly (last 4 weeks) | -| **6** | Monthly (last 6 months) | - -Older snapshots are deleted after each backup to save storage. - - -Retention policy is configured in the backup script (`/opt/alfred/backup.sh`). Modify it via SSH if you need longer retention. - - -## Restoring from Backup - - -Restoration requires SSH access. The API does not provide restore endpoints (destructive operation). - - -### Full Restore - - - - ```bash - ssh deploy@alfred-acme.tailnet.ts.net - ``` - - - ```bash - cd /opt/alfred - docker compose down - ``` - - - ```bash - restic -r s3:s3.amazonaws.com/your-bucket snapshots - ``` - - - ```bash - # Replace SNAPSHOT_ID with the ID from step 3 - restic -r s3:s3.amazonaws.com/your-bucket restore SNAPSHOT_ID \ - --target /mnt/encrypted - ``` - - - ```bash - ls -lh /mnt/encrypted/vault - cat /mnt/encrypted/.env - ``` - - - ```bash - docker compose up -d - ``` - - - ```bash - curl https://alfred-acme.tailnet.ts.net:3100/api/health \ - -H "X-API-Key: YOUR_API_KEY" - ``` - - - -### Partial Restore - -Restore only specific files: - -```bash -# Restore just the vault -restic -r s3:s3.amazonaws.com/your-bucket restore SNAPSHOT_ID \ - --target /tmp/restore \ - --include /mnt/encrypted/vault - -# Copy to live location -cp -r /tmp/restore/mnt/encrypted/vault/* /mnt/encrypted/vault/ -``` - -### Restore to a New Instance - - - - Use alfred-ctrl to create a new instance - - - ```bash - ssh deploy@new-instance.tailnet.ts.net - ``` - - - ```bash - cd /opt/alfred - docker compose down - ``` - - - ```bash - export AWS_ACCESS_KEY_ID="old-access-key" - export AWS_SECRET_ACCESS_KEY="old-secret-key" - export RESTIC_PASSWORD="old-restic-password" - ``` - - - ```bash - restic -r s3:s3.amazonaws.com/old-bucket restore SNAPSHOT_ID \ - --target /mnt/encrypted - ``` - - - ```bash - nano /mnt/encrypted/.env - # Update TAILSCALE_HOSTNAME, AAS_API_KEY, etc. - ``` - - - ```bash - docker compose up -d - ``` - - - -## Disaster Recovery Scenarios - -### Corrupted Vault - - - - ```bash - curl -X GET https://alfred-acme.tailnet.ts.net:3100/api/backup/snapshots \ - -H "X-API-Key: YOUR_API_KEY" - ``` - - - Follow the full restore process above - - - ```bash - curl -X POST https://alfred-acme.tailnet.ts.net:3100/api/workers/janitor/scan \ - -H "X-API-Key: YOUR_API_KEY" - ``` - - - -### Lost API Key - -If `AAS_API_KEY` is lost and you can't access the API: - - - - ```bash - ssh deploy@alfred-acme.tailnet.ts.net - ``` - - - ```bash - NEW_KEY=$(openssl rand -hex 32) - echo "AAS_API_KEY=$NEW_KEY" >> /mnt/encrypted/.env - ``` - - - ```bash - cd /opt/alfred - docker compose restart alfred-aas - ``` - - - ```bash - curl https://alfred-acme.tailnet.ts.net:3100/api/health \ - -H "X-API-Key: $NEW_KEY" - ``` - - - -### Server Failure - -If the server is completely lost: - - - - ```bash - node dist/index.mjs provision alfred-recovery - ``` - - - Use old backup credentials - - - Follow "Restore to a New Instance" workflow - - - Re-authenticate Tailscale with the same hostname - - - -## Backup Verification - -Test backups monthly: - - - - ```bash - curl -X GET https://alfred-acme.tailnet.ts.net:3100/api/backup/snapshots \ - -H "X-API-Key: YOUR_API_KEY" - ``` - - - ```bash - ssh deploy@alfred-acme.tailnet.ts.net - restic -r s3:s3.amazonaws.com/your-bucket check - ``` - - - ```bash - restic -r s3:s3.amazonaws.com/your-bucket restore latest \ - --target /tmp/test-restore \ - --include /mnt/encrypted/vault/person - ``` - - - ```bash - ls /tmp/test-restore/mnt/encrypted/vault/person - ``` - - - ```bash - rm -rf /tmp/test-restore - ``` - - - -## Troubleshooting - -### Backup Failed - -Check logs: - -```bash -curl -N https://alfred-acme.tailnet.ts.net:3100/api/logs/stream?service=backup&tail=100 \ - -H "X-API-Key: YOUR_API_KEY" -``` - -Common errors: -- **S3 credentials invalid**: Update `HETZNER_S3_*` in `.env` -- **Restic password wrong**: Update `RESTIC_PASSWORD` in `.env` -- **Network timeout**: Retry or check S3 endpoint - -### Snapshot List Empty - -Verify restic repository: - -```bash -ssh deploy@alfred-acme.tailnet.ts.net -restic -r s3:s3.amazonaws.com/your-bucket snapshots -``` - -If no repository exists, initialize it: - -```bash -restic -r s3:s3.amazonaws.com/your-bucket init -``` - -### Restore Incomplete - -Check restic logs: - -```bash -restic -r s3:s3.amazonaws.com/your-bucket restore SNAPSHOT_ID \ - --target /tmp/restore \ - --verbose -``` - -If files are missing, the snapshot may be corrupted. Try an older snapshot. - -## Next Steps - - - - Configure backup settings (S3, retention) - - - Monitor backup job logs - - +LS0tCnRpdGxlOiAiQmFja3VwcyAmIFJlY292ZXJ5IgpkZXNjcmlwdGlvbjogIkFwcGxpY2F0aW9uLWxldmVsIGJhY2t1cCBjb250cmFjdCBhbmQgcmVzdG9yZSBkcmlsbCIKaWNvbjogInNoaWVsZCIKLS0tCgpBbGZyZWQgQmxhY2sncyBkZWZhdWx0IGBkb2NrZXIgY29tcG9zZSB1cCAtZGAgc3RhY2sgZG9lcyBub3Qgc3RhcnQgYW4gYXV0b21hdGljIGJhY2t1cCBzZXJ2aWNlLiBUaGUgc3VwcG9ydGVkIGNvbnRyYWN0IGlzIGFwcGxpY2F0aW9uLWxldmVsIGJhY2t1cCBvZiBEb2NrZXIgbmFtZWQgdm9sdW1lcywgd2l0aCBlbmNyeXB0ZWQgb2ZmLVZNIHN0b3JhZ2Ugb3BlcmF0ZWQgYnkgdGhlIGluc3RhbmNlIG93bmVyLgoKRm9yIHRoZSBjYW5vbmljYWwgcnVuYm9vaywgc2VlIGBkZXBsb3kvQkFDS1VQUy5tZGAgwqBpbiB0aGUgcmVwb3NpdG9yeS4KCiMjIENvbnRyYWN0IFN1bW1hcnkKCjxDYXJkR3JvdXAgY29scz17M30+CiAgPENhcmQgdGl0bGU9IkJhY2t1cCBNb2RlbCIgaWNvbj0iZGF0YWJhc2UiPgogICAgT3BlcmF0b3ItcnVuIERvY2tlciBuYW1lZC12b2x1bWUgYXJjaGl2ZXMsIGVuY3J5cHRlZCBiZWZvcmUgbGVhdmluZyB0aGUgVk0uCiAgPC9DYXJkPgogIDxDYXJkIHRpdGxlPSJEZWZhdWx0IEJlaGF2aW9yIiBpY29uPSJwb3dlci1vZmYiPgogICAgTm8gYmFja3VwIHNpZGVjYXIgc3RhcnRzIGJ5IGRlZmF1bHQgYW5kIG5vIGJhY2t1cCBjcmVkZW50aWFscyBhcmUgcmVxdWlyZWQgZm9yIGZpcnN0IGJvb3QuCiAgPC9DYXJkPgogIDxDYXJkIHRpdGxlPSJSZXN0b3JlIERyaWxsIiBpY29uPSJzaGllbGQtY2hlY2siPgogICAgYC4vc2NyaXB0cy9yZXN0b3JlLWRyaWxsLnNoYCB2ZXJpZmllcyBhIHZhdWx0IGZpbGUgYW5kIFNRTGl0ZSBzdGF0ZSBhcnRpZmFjdCByb3VuZCB0cmlwLgogIDwvQ2FyZD4KPC9DYXJkR3JvdXA+CgpSZWNvbW1lbmRlZCB0YXJnZXRzIHdoZW4gYSBkYWlseSBiYWNrdXAgc2NoZWR1bGUgaXMgY29uZmlndXJlZDoKCnxUYXJnZXQgfCBFeHBlY3RhdGlvbiB8CnwtLS0gfCAtLS0gfAp8IFJQTyB8IDI0IGhvdXJzLCBvciBsZXNzIHdoZW4gYW4gb3BlcmF0b3IgdHJpZ2dlcnMgYSBtYW51YWwgYmFja3VwIGJlZm9yZSByaXNreSB3b3JrLiB8CnwgUlRPIHwgMiBob3VycyBmb3IgYSBmdWxsIHNpbmdsZS1WTSByZWJ1aWxkIG9uY2UgYSBmcmVzaCBWTSwgRE5TLCBgLmVudmAsIGFuZCBlbmNyeXB0ZWQgYmFja3VwIGJ1bmRsZSBhcmUgYXZhaWxhYmxlLiB8CnwgQ29uc2lzdGVuY3kgfCBCZXN0IHdoZW4gdGhlIHN0YWNrIGlzIHN0b3BwZWQgYmVmb3JlIGFyY2hpdmluZyB2b2x1bWVzLiBPbmxpbmUgZGF0YWJhc2Ugc25hcHNob3RzIG1heSBiZSBjcmFzaC1jb25zaXN0ZW50IG9ubHkuIHwKCiMjIENyaXRpY2FsIFZvbHVtZXMKClRoZSBiYWNrdXAgc2V0IG11c3QgaW5jbHVkZSBldmVyeSBuYW1lZCB2b2x1bWUgcmVwb3J0ZWQgYnk6CgpgYGBiYXNoCmRvY2tlciBjb21wb3NlIGNvbmZpZyAtLXZvbHVtZXMKYGBgCgpDcml0aWNhbCBwcmluY2lwYWwtZmFjaW5nIGFuZCByZWNvdmVyeS1jcml0aWNhbCB2b2x1bWVzIGluY2x1ZGU6CgotIGB2YXVsdF9kYXRhYCDigJQgd nhdWx0IHJlY29yZHMsIGRlY2lzaW9ucywgYnJpZWZpbmdzLCBhbmQgZGF5Ym9vayBvdXRwdXQuCi0gYHN0YXRlX2RhdGFgIOKAlCBgYWxmcmVkLXN0YXRlLmRiYCwgV0FMLCBvYnNlcnZhdGlvbnMsIHNpZ25hbHMsIGFuZCBhdWRpdCBzdGF0ZS4KLSBgaW5nZXN0X2RhdGFgIOKAlCBgaW5nZXN0LmRiYCByYXcgc3RyZWFtIGV2ZW50cy4KLSBgaGVybWVzX2RhdGFgIOKAlCBIZXJtZXMgcHJvZmlsZXMsIHNlc3Npb25zLCBjaGFubmVsIHN0YXRlLCBhbmQgZ2VuZXJhdGVkIHByb2ZpbGUgY29uZmlnLgogLSBgYWxmcmVkX2RhdGFgIOKAlCBzaGFyZWQgQWxmcmVkIHJ1bnRpbWUgZGF0YSBhbmQgZ2VuZXJhdGVkIGdhdGV3YXkgbWF0ZXJpYWwuCi0gYHZhdWx0d2FyZGVuX2RhdGFgIOKAlCBzZWNyZXRzIHN0b3JlIGRhdGEuCi0gYHdlYl9kYl9kYXRhYCDigJQgd2ViL2F1dGggZGF0YWJhc2UuCi0gYGNhZGR5X2RhdGFgIOKAlCBMZXQncyBFbmNyeXB0IGFjY291bnQgYW5kIGNlcnRpZmljYXRlIG1hdGVyaWFsLgogLSBgc3VyZV9wZ2RhdGFgIOKAlCBTdXJlIGZpbmFuY2UgZGF0YWJhc2UsIHdoZW4gU3VyZSBpcyB1c2VkLgogLSBgcGFwZXJjbGlwX2RhdGFgIOKAlCBQYXBlcmNsaXAgY29tcGFueSwgaXNzdWUsIGFuZCBhZ2VudCBzdGF0ZSwgd2hlbiBQYXBlcmNsaXAgaXMgdXNlZC4KLSBgZmlsZXNfZGF0YWAg4oCUIGZpbGUtc3RvcmUgYmxvYnMsIGlmIHByZXNlbnQgaW4gYSBkZXBsb3ltZW50IHZhcmlhbnQuCgpUaGUgZnVsbCBjbGFzc2lmaWNhdGlvbiB0YWJsZSBpcyBpbiBgZGVwbG95L0JBQ0tVUFMubWRgLgoKIyMgTWFudWFsIEJhY2t1cCBTaGFwZQoKVXNlIHRoZSBydW5ib29rIHJhdGhlciB0aGFuIGFkLWhvYyBjb3B5aW5nLiBUaGUgc2hvcnQgZm9ybSBpczoKCjEuIENyZWF0ZSBhIHByaXZhdGUgYmFja3VwIGRpcmVjdG9yeSBvdXRzaWRlIHRoZSByZXBvc2l0b3J5IGNoZWNrb3V0Lgo yLiBTdG9wIHRoZSBzdGFjayBmb3IgdGhlIGNsZWFuZXN0IHNuYXBzaG90LgozLiBBcmNoaXZlIGV2ZXJ5IG5hbWVkIERvY2tlciB2b2x1bWUgaW50byB0aGF0IGRpcmVjdG9yeS4KNC4gQ29weSBgZG9ja2VyLWNvbXBvc2UueWFtbGAsIGAuZW52YCwgYW5kIHRoZSBgZG9ja2VyIGNvbXBvc2UgY29uZmlnIC0tdm9sdW1lc2Agb3V0cHV0IGludG8gdGhlIGJ1bmRsZS4KNS4gUmVzdGFydCB0aGUgc3RhY2suCjYuIEVuY3J5cHQgdGhlIGJ1bmRsZSBiZWZvcmUgdXBsb2FkaW5nIGl0IG9mZi1WTS4KCkRvIG5vdCBwdWJsaXNoIGAuZW52YCwgVmF1bHR3YXJkZW4gZGF0YSwgQWdlIHByaXZhdGUga2V5cywgb2JqZWN0LXN0b3JlIGNyZWRlbnRpYWxzLCBvciByYXcgdGVuYW50IGNvbnRlbnRzIGluIGNvbW1lbnRzLCBsb2dzLCBvciBDSSBvdXRwdXQuCgojIyBSZXN0b3JlCgpSZXN0b3JlIGlzIGludGVudGlvbmFsbHkgU1NIL21hbnVhbCBiZWNhdXNlIGl0IGlzIGRlc3RydWN0aXZlLiBBdCBhIGhpZ2ggbGV2ZWw6CgoxLiBQcm92aXNpb24gYSBmcmVzaCBWTSBhbmQgY2xvbmUgdGhlIHJlcG9zaXRvcnkuCjIuIERlY3J5cHQgdGhlIGJhY2t1cCBidW5kbGUgaW50byBhIHByaXZhdGUgbG9jYWwgZGlyZWN0b3J5LgozLiBTdG9wIGFueSBleGlzdGluZyBzdGFjay4KNC4gUmVjcmVhdGUgZWFjaCBuYW1lZCB2b2x1bWUgYW5kIGV4dHJhY3QgaXRzIG1hdGNoaW5nIGFyY2hpdmUuCjUuIFN0YXJ0IHRoZSBzdGFjay4KNi4gVmVyaWZ5IERlc2ssIEJyaWVmLCBWYXVsdCByZWNvcmRzLCBmaWxlIGFydGlmYWN0cywgYGN0cmwtYXBpYCBzdGF0ZSwgVmF1bHR3YXJkZW4sIGFuZCBIVFRQUy4KClNlZSBgZGVwbG95L0JBQ0tVUFMubWRgIGZvciBleGFjdCBjb21tYW5kcy4KCiMjIFNtb2tlIEV2aWRlbmNlCgpFdmVyeSBQUiB0aGF0IGNoYW5nZXMgYmFja3VwIGJlaGF2aW9yIG9yIGRvY3VtZW50YXRpb24gc2hvdWxkIGluY2x1ZGU6CgpgYGBiYXNoCi4vc2NyaXB0cy9yZXN0b3JlLWRyaWxsLnNoCnB5dGhvbiAtIDw8J1BZJwppbXBvcnQgeWFtbAppbXBvcnQgUGF0aGxpYiBmcm9tIHBhdGhsaWIKeWFtbC5zYWZlX2xvYWQoUGF0aGxpYignZG9ja2VyLWNvbXBvc2UueWFtbCcpLnJlYWRfdGV4dCgpKQpwcmludCgnY29tcG9zZS15YW1sLXBhcnNlLW9rJykKUFkKYGBgCgpXaGVuIERvY2tlciBpcyBhdmFpbGFibGUsIGFsc28gcnVuOgoKYGBgYmFzaApkb2NrZXIgY29tcG9zZSBjb25maWcKZG9ja2VyIGNvbXBvc2UgY29uZmlnIC0tdm9sdW1lcwoKYGBgCgpUaGUgcmVzdG9yZSBkcmlsbCBpcyBub24tZGVzdHJ1Y3RpdmUuIEl0IHVzZXMgdGVtcG9yYXJ5IGRpcmVjdG9yaWVzIGFuZCBwcm92ZXMgdGhhdCB0aGUgYmFja3VwIG1lY2hhbmljcyBwcmVzZXJ2ZSBib3RoIGEgdXNlci1mYWNpbmcgdmF1bHQgZmlsZSBhbmQgYSBTUUxpdGUgc3RhdGUtYmVhcmluZyBhcnRpZmFjdC4K \ No newline at end of file From f715b758bc4c86c6d43d82a0122d2ac5c425557d Mon Sep 17 00:00:00 2001 From: David Szabo-Stuban Date: Wed, 3 Jun 2026 13:11:37 +0200 Subject: [PATCH 2/5] docs(backups): correct recovery guide content --- .../ctrl/docs/guides/backups-recovery.mdx | 105 +++++++++++++++++- 1 file changed, 104 insertions(+), 1 deletion(-) diff --git a/packages/ctrl/docs/guides/backups-recovery.mdx b/packages/ctrl/docs/guides/backups-recovery.mdx index 602d3bc36..be741068f 100644 --- a/packages/ctrl/docs/guides/backups-recovery.mdx +++ b/packages/ctrl/docs/guides/backups-recovery.mdx @@ -1 +1,104 @@ -LS0tCnRpdGxlOiAiQmFja3VwcyAmIFJlY292ZXJ5IgpkZXNjcmlwdGlvbjogIkFwcGxpY2F0aW9uLWxldmVsIGJhY2t1cCBjb250cmFjdCBhbmQgcmVzdG9yZSBkcmlsbCIKaWNvbjogInNoaWVsZCIKLS0tCgpBbGZyZWQgQmxhY2sncyBkZWZhdWx0IGBkb2NrZXIgY29tcG9zZSB1cCAtZGAgc3RhY2sgZG9lcyBub3Qgc3RhcnQgYW4gYXV0b21hdGljIGJhY2t1cCBzZXJ2aWNlLiBUaGUgc3VwcG9ydGVkIGNvbnRyYWN0IGlzIGFwcGxpY2F0aW9uLWxldmVsIGJhY2t1cCBvZiBEb2NrZXIgbmFtZWQgdm9sdW1lcywgd2l0aCBlbmNyeXB0ZWQgb2ZmLVZNIHN0b3JhZ2Ugb3BlcmF0ZWQgYnkgdGhlIGluc3RhbmNlIG93bmVyLgoKRm9yIHRoZSBjYW5vbmljYWwgcnVuYm9vaywgc2VlIGBkZXBsb3kvQkFDS1VQUy5tZGAgwqBpbiB0aGUgcmVwb3NpdG9yeS4KCiMjIENvbnRyYWN0IFN1bW1hcnkKCjxDYXJkR3JvdXAgY29scz17M30+CiAgPENhcmQgdGl0bGU9IkJhY2t1cCBNb2RlbCIgaWNvbj0iZGF0YWJhc2UiPgogICAgT3BlcmF0b3ItcnVuIERvY2tlciBuYW1lZC12b2x1bWUgYXJjaGl2ZXMsIGVuY3J5cHRlZCBiZWZvcmUgbGVhdmluZyB0aGUgVk0uCiAgPC9DYXJkPgogIDxDYXJkIHRpdGxlPSJEZWZhdWx0IEJlaGF2aW9yIiBpY29uPSJwb3dlci1vZmYiPgogICAgTm8gYmFja3VwIHNpZGVjYXIgc3RhcnRzIGJ5IGRlZmF1bHQgYW5kIG5vIGJhY2t1cCBjcmVkZW50aWFscyBhcmUgcmVxdWlyZWQgZm9yIGZpcnN0IGJvb3QuCiAgPC9DYXJkPgogIDxDYXJkIHRpdGxlPSJSZXN0b3JlIERyaWxsIiBpY29uPSJzaGllbGQtY2hlY2siPgogICAgYC4vc2NyaXB0cy9yZXN0b3JlLWRyaWxsLnNoYCB2ZXJpZmllcyBhIHZhdWx0IGZpbGUgYW5kIFNRTGl0ZSBzdGF0ZSBhcnRpZmFjdCByb3VuZCB0cmlwLgogIDwvQ2FyZD4KPC9DYXJkR3JvdXA+CgpSZWNvbW1lbmRlZCB0YXJnZXRzIHdoZW4gYSBkYWlseSBiYWNrdXAgc2NoZWR1bGUgaXMgY29uZmlndXJlZDoKCnxUYXJnZXQgfCBFeHBlY3RhdGlvbiB8CnwtLS0gfCAtLS0gfAp8IFJQTyB8IDI0IGhvdXJzLCBvciBsZXNzIHdoZW4gYW4gb3BlcmF0b3IgdHJpZ2dlcnMgYSBtYW51YWwgYmFja3VwIGJlZm9yZSByaXNreSB3b3JrLiB8CnwgUlRPIHwgMiBob3VycyBmb3IgYSBmdWxsIHNpbmdsZS1WTSByZWJ1aWxkIG9uY2UgYSBmcmVzaCBWTSwgRE5TLCBgLmVudmAsIGFuZCBlbmNyeXB0ZWQgYmFja3VwIGJ1bmRsZSBhcmUgYXZhaWxhYmxlLiB8CnwgQ29uc2lzdGVuY3kgfCBCZXN0IHdoZW4gdGhlIHN0YWNrIGlzIHN0b3BwZWQgYmVmb3JlIGFyY2hpdmluZyB2b2x1bWVzLiBPbmxpbmUgZGF0YWJhc2Ugc25hcHNob3RzIG1heSBiZSBjcmFzaC1jb25zaXN0ZW50IG9ubHkuIHwKCiMjIENyaXRpY2FsIFZvbHVtZXMKClRoZSBiYWNrdXAgc2V0IG11c3QgaW5jbHVkZSBldmVyeSBuYW1lZCB2b2x1bWUgcmVwb3J0ZWQgYnk6CgpgYGBiYXNoCmRvY2tlciBjb21wb3NlIGNvbmZpZyAtLXZvbHVtZXMKYGBgCgpDcml0aWNhbCBwcmluY2lwYWwtZmFjaW5nIGFuZCByZWNvdmVyeS1jcml0aWNhbCB2b2x1bWVzIGluY2x1ZGU6CgotIGB2YXVsdF9kYXRhYCDigJQgd nhdWx0IHJlY29yZHMsIGRlY2lzaW9ucywgYnJpZWZpbmdzLCBhbmQgZGF5Ym9vayBvdXRwdXQuCi0gYHN0YXRlX2RhdGFgIOKAlCBgYWxmcmVkLXN0YXRlLmRiYCwgV0FMLCBvYnNlcnZhdGlvbnMsIHNpZ25hbHMsIGFuZCBhdWRpdCBzdGF0ZS4KLSBgaW5nZXN0X2RhdGFgIOKAlCBgaW5nZXN0LmRiYCByYXcgc3RyZWFtIGV2ZW50cy4KLSBgaGVybWVzX2RhdGFgIOKAlCBIZXJtZXMgcHJvZmlsZXMsIHNlc3Npb25zLCBjaGFubmVsIHN0YXRlLCBhbmQgZ2VuZXJhdGVkIHByb2ZpbGUgY29uZmlnLgogLSBgYWxmcmVkX2RhdGFgIOKAlCBzaGFyZWQgQWxmcmVkIHJ1bnRpbWUgZGF0YSBhbmQgZ2VuZXJhdGVkIGdhdGV3YXkgbWF0ZXJpYWwuCi0gYHZhdWx0d2FyZGVuX2RhdGFgIOKAlCBzZWNyZXRzIHN0b3JlIGRhdGEuCi0gYHdlYl9kYl9kYXRhYCDigJQgd2ViL2F1dGggZGF0YWJhc2UuCi0gYGNhZGR5X2RhdGFgIOKAlCBMZXQncyBFbmNyeXB0IGFjY291bnQgYW5kIGNlcnRpZmljYXRlIG1hdGVyaWFsLgogLSBgc3VyZV9wZ2RhdGFgIOKAlCBTdXJlIGZpbmFuY2UgZGF0YWJhc2UsIHdoZW4gU3VyZSBpcyB1c2VkLgogLSBgcGFwZXJjbGlwX2RhdGFgIOKAlCBQYXBlcmNsaXAgY29tcGFueSwgaXNzdWUsIGFuZCBhZ2VudCBzdGF0ZSwgd2hlbiBQYXBlcmNsaXAgaXMgdXNlZC4KLSBgZmlsZXNfZGF0YWAg4oCUIGZpbGUtc3RvcmUgYmxvYnMsIGlmIHByZXNlbnQgaW4gYSBkZXBsb3ltZW50IHZhcmlhbnQuCgpUaGUgZnVsbCBjbGFzc2lmaWNhdGlvbiB0YWJsZSBpcyBpbiBgZGVwbG95L0JBQ0tVUFMubWRgLgoKIyMgTWFudWFsIEJhY2t1cCBTaGFwZQoKVXNlIHRoZSBydW5ib29rIHJhdGhlciB0aGFuIGFkLWhvYyBjb3B5aW5nLiBUaGUgc2hvcnQgZm9ybSBpczoKCjEuIENyZWF0ZSBhIHByaXZhdGUgYmFja3VwIGRpcmVjdG9yeSBvdXRzaWRlIHRoZSByZXBvc2l0b3J5IGNoZWNrb3V0Lgo yLiBTdG9wIHRoZSBzdGFjayBmb3IgdGhlIGNsZWFuZXN0IHNuYXBzaG90LgozLiBBcmNoaXZlIGV2ZXJ5IG5hbWVkIERvY2tlciB2b2x1bWUgaW50byB0aGF0IGRpcmVjdG9yeS4KNC4gQ29weSBgZG9ja2VyLWNvbXBvc2UueWFtbGAsIGAuZW52YCwgYW5kIHRoZSBgZG9ja2VyIGNvbXBvc2UgY29uZmlnIC0tdm9sdW1lc2Agb3V0cHV0IGludG8gdGhlIGJ1bmRsZS4KNS4gUmVzdGFydCB0aGUgc3RhY2suCjYuIEVuY3J5cHQgdGhlIGJ1bmRsZSBiZWZvcmUgdXBsb2FkaW5nIGl0IG9mZi1WTS4KCkRvIG5vdCBwdWJsaXNoIGAuZW52YCwgVmF1bHR3YXJkZW4gZGF0YSwgQWdlIHByaXZhdGUga2V5cywgb2JqZWN0LXN0b3JlIGNyZWRlbnRpYWxzLCBvciByYXcgdGVuYW50IGNvbnRlbnRzIGluIGNvbW1lbnRzLCBsb2dzLCBvciBDSSBvdXRwdXQuCgojIyBSZXN0b3JlCgpSZXN0b3JlIGlzIGludGVudGlvbmFsbHkgU1NIL21hbnVhbCBiZWNhdXNlIGl0IGlzIGRlc3RydWN0aXZlLiBBdCBhIGhpZ2ggbGV2ZWw6CgoxLiBQcm92aXNpb24gYSBmcmVzaCBWTSBhbmQgY2xvbmUgdGhlIHJlcG9zaXRvcnkuCjIuIERlY3J5cHQgdGhlIGJhY2t1cCBidW5kbGUgaW50byBhIHByaXZhdGUgbG9jYWwgZGlyZWN0b3J5LgozLiBTdG9wIGFueSBleGlzdGluZyBzdGFjay4KNC4gUmVjcmVhdGUgZWFjaCBuYW1lZCB2b2x1bWUgYW5kIGV4dHJhY3QgaXRzIG1hdGNoaW5nIGFyY2hpdmUuCjUuIFN0YXJ0IHRoZSBzdGFjay4KNi4gVmVyaWZ5IERlc2ssIEJyaWVmLCBWYXVsdCByZWNvcmRzLCBmaWxlIGFydGlmYWN0cywgYGN0cmwtYXBpYCBzdGF0ZSwgVmF1bHR3YXJkZW4sIGFuZCBIVFRQUy4KClNlZSBgZGVwbG95L0JBQ0tVUFMubWRgIGZvciBleGFjdCBjb21tYW5kcy4KCiMjIFNtb2tlIEV2aWRlbmNlCgpFdmVyeSBQUiB0aGF0IGNoYW5nZXMgYmFja3VwIGJlaGF2aW9yIG9yIGRvY3VtZW50YXRpb24gc2hvdWxkIGluY2x1ZGU6CgpgYGBiYXNoCi4vc2NyaXB0cy9yZXN0b3JlLWRyaWxsLnNoCnB5dGhvbiAtIDw8J1BZJwppbXBvcnQgeWFtbAppbXBvcnQgUGF0aGxpYiBmcm9tIHBhdGhsaWIKeWFtbC5zYWZlX2xvYWQoUGF0aGxpYignZG9ja2VyLWNvbXBvc2UueWFtbCcpLnJlYWRfdGV4dCgpKQpwcmludCgnY29tcG9zZS15YW1sLXBhcnNlLW9rJykKUFkKYGBgCgpXaGVuIERvY2tlciBpcyBhdmFpbGFibGUsIGFsc28gcnVuOgoKYGBgYmFzaApkb2NrZXIgY29tcG9zZSBjb25maWcKZG9ja2VyIGNvbXBvc2UgY29uZmlnIC0tdm9sdW1lcwoKYGBgCgpUaGUgcmVzdG9yZSBkcmlsbCBpcyBub24tZGVzdHJ1Y3RpdmUuIEl0IHVzZXMgdGVtcG9yYXJ5IGRpcmVjdG9yaWVzIGFuZCBwcm92ZXMgdGhhdCB0aGUgYmFja3VwIG1lY2hhbmljcyBwcmVzZXJ2ZSBib3RoIGEgdXNlci1mYWNpbmcgdmF1bHQgZmlsZSBhbmQgYSBTUUxpdGUgc3RhdGUtYmVhcmluZyBhcnRpZmFjdC4K \ No newline at end of file +--- +title: "Backups & Recovery" +description: "Application-level backup contract and restore drill" +icon: "shield" +--- + +Alfred Black's default `docker compose up -d` stack does not start an automatic backup service. The supported contract is application-level backup of Docker named volumes, with encrypted off-VM storage operated by the instance owner. + +For the canonical runbook, see `deploy/BACKUPS.md` in the repository. + +## Contract Summary + + + + Operator-run Docker named-volume archives, encrypted before leaving the VM. + + + No backup sidecar starts by default and no backup credentials are required for first boot. + + + `./scripts/restore-drill.sh` verifies a vault file and SQLite state artifact round trip. + + + +Recommended targets when a daily backup schedule is configured: + +| Target | Expectation | +| --- | --- | +| RPO | 24 hours, or less when an operator triggers a manual backup before risky work. | +| RTO | 2 hours for a full single-VM rebuild once a fresh VM, DNS, `.env`, and encrypted backup bundle are available. | +| Consistency | Best when the stack is stopped before archiving volumes. Online database snapshots may be crash-consistent only. | + +## Critical Volumes + +The backup set must include every named volume reported by: + +```bash +docker compose config --volumes +``` + +Critical principal-facing and recovery-critical volumes include: + +- `vault_data` — vault records, decisions, briefings, and daybook output. +- `state_data` — `alfred-state.db`, WAL, observations, signals, and audit state. +- `ingest_data` — `ingest.db` raw stream events. +- `hermes_data` — Hermes profiles, sessions, channel state, and generated profile config. +- `alfred_data` — shared Alfred runtime data and generated gateway material. +- `vaultwarden_data` — secrets store data. +- `web_db_data` — web/auth database. +- `caddy_data` — Let's Encrypt account and certificate material. +- `sure_pgdata` — Sure finance database, when Sure is used. +- `paperclip_data` — Paperclip company, issue, and agent state, when Paperclip is used. +- `files_data` — file-store blobs, if present in a deployment variant. + +The full classification table is in `deploy/BACKUPS.md`. + +## Manual Backup Shape + +Use the runbook rather than ad-hoc copying. The short form is: + +1. Create a private backup directory outside the repository checkout. +2. Stop the stack for the cleanest snapshot. +3. Archive every named Docker volume into that directory. +4. Copy `docker-compose.yaml`, `.env`, and the `docker compose config --volumes` output into the bundle. +5. Restart the stack. +6. Encrypt the bundle before uploading it off-VM. + +Do not publish `.env`, Vaultwarden data, Age private keys, object-store credentials, or raw tenant contents in comments, logs, or CI output. + +## Restore + +Restore is intentionally SSH/manual because it is destructive. At a high level: + +1. Provision a fresh VM and clone the repository. +2. Decrypt the backup bundle into a private local directory. +3. Stop any existing stack. +4. Recreate each named volume and extract its matching archive. +5. Start the stack. +6. Verify Desk, Brief, Vault records, file artifacts, `ctrl-api` state, Vaultwarden, and HTTPS. + +See `deploy/BACKUPS.md` for exact commands. + +## Smoke Evidence + +Every PR that changes backup behavior or documentation should include: + +```bash +./scripts/restore-drill.sh +python - <<'PY' +import yaml +from pathlib import Path +yaml.safe_load(Path('docker-compose.yaml').read_text()) +print('compose-yaml-parse-ok') +PY +``` + +When Docker is available, also run: + +```bash +docker compose config +docker compose config --volumes +``` + +The restore drill is non-destructive. It uses temporary directories and proves that the backup mechanics preserve both a user-facing vault file and a SQLite state-bearing artifact. From d652a0f8ed5a57315784ba7f768b5b1cd786bfcc Mon Sep 17 00:00:00 2001 From: David Szabo-Stuban Date: Wed, 3 Jun 2026 13:12:30 +0200 Subject: [PATCH 3/5] docs(backups): add application restore runbook --- deploy/BACKUPS.md | 174 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 deploy/BACKUPS.md diff --git a/deploy/BACKUPS.md b/deploy/BACKUPS.md new file mode 100644 index 000000000..7df6cab17 --- /dev/null +++ b/deploy/BACKUPS.md @@ -0,0 +1,174 @@ +# Backup contract and restore drill + +This is the operating contract for a single-VM Alfred Black deployment. It is deliberately application-level: operators can back up and restore the Docker named volumes without relying on a particular cloud snapshot product. + +## Contract + +Default deployment remains unchanged. Backups are an operator action until a future release wires an opt-in scheduled sidecar or host timer. + +Recommended model: + +1. Run a daily application-level backup of the named Docker volumes listed below. +2. Store the backup bundle outside the VM, encrypted before upload. +3. Run the restore drill after initial setup, after every major upgrade, and quarterly. + +RPO/RTO targets when the recommended daily schedule is followed: + +| Target | Expectation | +| --- | --- | +| RPO | 24 hours for principal-facing and operational state; less if the operator triggers a manual backup before risky work. | +| RTO | 2 hours for a full single-VM rebuild once a fresh VM, DNS, `.env`, and encrypted backup bundle are available. | +| Consistency | Best when the stack is stopped before backup. Online backups of SQLite/Postgres volumes may be crash-consistent only. | + +If an operator chooses Hetzner snapshots instead, that is an external infrastructure contract, not an Alfred application-level backup. The operator must record the snapshot schedule, retention, encryption posture, and a successful restore test outside this repository. + +## Data classification by volume + +The current compose file defines these named volumes. + +| Volume | Loss impact | Backup priority | Notes | +| --- | --- | --- | --- | +| `vault_data` | Principal-facing vault records, decisions, briefings, daybook output. | Critical | Published user surface; always back up. | +| `files_data` | Principal-uploaded/generated file blobs if present in a deployment variant. | Critical | Not present in the current compose volume list, but must be included when enabled. | +| `state_data` | `alfred-state.db`, WAL, sqlite-vec memory, signals, observations, audit. | Critical | Stop `ctrl-api` or the full stack for the cleanest snapshot. | +| `ingest_data` | `ingest.db`, short-lived raw stream events. | High | Useful for replay/forensics; lower retention requirement than state. | +| `cold_data` | Long-tail archive store. | High | Include if Phase 3 cold archive has data. | +| `hermes_data` | Hermes profiles, sessions, channel state, generated profile config. | Critical | Needed for channel continuity and agent runtime recovery. | +| `alfred_data` | Shared Alfred runtime data, generated tokens, gateway token. | Critical | Some consumers mount this read-only; restore before starting services. | +| `caddy_data` | Let's Encrypt cert account/cert material. | High | Restorable by re-issuing, but backup avoids rate-limit and outage risk. | +| `caddy_config` | Caddy runtime config cache. | Medium | Less critical than `caddy_data`; include for completeness. | +| `web_db_data` | Wasp/web auth database. | Critical | Owner account/session and app records. | +| `vaultwarden_data` | Vaultwarden database and attachments. | Critical | Secrets store; backup must be encrypted. | +| `mcp_server_data` | MCP server local state. | Medium | Include to preserve connector-local state. | +| `temporal_data` | Temporal workflow history. | High | Workflows can often be restarted, but history aids recovery. | +| `ollama_data` | Downloaded embedding/model blobs. | Low | Re-downloadable; backup only to reduce rebuild time. | +| `plane_pgdata`, `plane_redis`, `plane_rabbitmq`, `plane_uploads` | Plane issue tracker data and artifacts. | High | Include if Plane is used as operational issue memory. | +| `sure_pgdata`, `sure_redis` | Sure finance database/cache. | Critical | Finance data; backup encrypted and test restore. | +| `paperclip_data` | Paperclip company/issues/agent state. | High | Include where Paperclip manages operations. | +| `vexa_redis`, `vexa_postgres`, `vexa_minio`, `vexa_recordings` | Optional Vexa transcript stack. | Medium/High | Include when `--profile vexa` is enabled and recordings/transcripts matter. | + +## Manual backup procedure + +1. SSH to the VM and enter the compose directory. + +```bash +cd /opt/alfred +``` + +2. Create a local backup directory outside the repository checkout. + +```bash +sudo install -d -m 0700 /var/backups/alfred +export BACKUP_DIR=/var/backups/alfred/$(date -u +%Y%m%dT%H%M%SZ) +sudo install -d -m 0700 "$BACKUP_DIR" +``` + +3. Stop the stack for a clean snapshot. + +```bash +docker compose down +``` + +4. Archive each named volume. + +```bash +for volume in $(docker compose config --volumes); do + docker run --rm \ + -v "${volume}:/src:ro" \ + -v "${BACKUP_DIR}:/backup" \ + alpine:3.20 \ + sh -c 'cd /src && tar -czf "/backup/${0}.tgz" .' "$volume" +done +``` + +5. Save the compose inputs needed to rebuild the stack. Do not publish these files; they may contain secrets. + +```bash +cp docker-compose.yaml "$BACKUP_DIR/docker-compose.yaml" +cp .env "$BACKUP_DIR/env" +docker compose config --volumes > "$BACKUP_DIR/volumes.txt" +``` + +6. Restart the stack. + +```bash +docker compose up -d +``` + +7. Encrypt and upload the backup bundle to off-VM storage. + +```bash +tar -C "$BACKUP_DIR" -czf - . \ + | age -r '' \ + > "${BACKUP_DIR}.tar.gz.age" +# Upload ${BACKUP_DIR}.tar.gz.age to the operator's backup store. +``` + +The Age private key, Vaultwarden master password, and any object-store credentials must be stored outside the VM as break-glass material. Do not commit them, paste them into Paperclip comments, or include them in CI logs. + +## Full restore procedure + +1. Provision a fresh Linux VM, install Docker/Compose, clone the repo, and place the restored `.env` in `/opt/alfred/.env`. +2. Decrypt the backup bundle into `/var/backups/alfred/restore`. +3. Ensure no old stack is running. + +```bash +cd /opt/alfred +docker compose down || true +``` + +4. Recreate and populate every archived volume. + +```bash +RESTORE_DIR=/var/backups/alfred/restore +while read -r volume; do + docker volume create "$volume" >/dev/null + docker run --rm \ + -v "${volume}:/dst" \ + -v "${RESTORE_DIR}:/backup:ro" \ + alpine:3.20 \ + sh -c 'cd /dst && tar -xzf "/backup/${0}.tgz"' "$volume" +done < "$RESTORE_DIR/volumes.txt" +``` + +5. Start the stack and verify the durable surfaces. + +```bash +docker compose up -d +docker compose ps +``` + +Minimum verification: + +- Desk and Brief load at `https://${DOMAIN}`. +- Vault records are visible. +- A previously uploaded file or generated file-store artifact is present if `files_data` exists for the deployment. +- `ctrl-api` can read `alfred-state.db` and the Desk decision/audit counts are plausible. +- Vaultwarden unlocks and existing items are present. +- Caddy serves HTTPS without re-issuing a storm of certificates. + +## Restore drill + +The non-destructive drill in `scripts/restore-drill.sh` proves the archive/restore mechanics for two representative artifacts: + +- a user-facing vault file; and +- a SQLite state database with WAL mode enabled. + +Run it from the repository root: + +```bash +./scripts/restore-drill.sh +``` + +The drill uses only temporary directories under `/tmp`, does not touch Docker, and exits non-zero if either restored artifact fails verification. It is not a replacement for a live restore test against real Docker volumes; it is the minimum smoke evidence every PR touching this contract should run. + +## PR smoke evidence template + +Every PR changing backup behavior or this runbook should include: + +```text +## Smoke evidence +- `docker compose config --volumes` — confirms the documented volume list still matches compose. +- `docker compose config` — confirms the default stack remains valid and no backup actor starts by default. +- `./scripts/restore-drill.sh` — confirms a user file and SQLite state artifact survive archive/restore. +``` From fde7a40f7b50194a56beed4b1b169f4a24beb015 Mon Sep 17 00:00:00 2001 From: David Szabo-Stuban Date: Wed, 3 Jun 2026 13:12:55 +0200 Subject: [PATCH 4/5] test(backups): add non-destructive restore drill --- scripts/restore-drill.sh | 59 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 scripts/restore-drill.sh diff --git a/scripts/restore-drill.sh b/scripts/restore-drill.sh new file mode 100644 index 000000000..e672e3621 --- /dev/null +++ b/scripts/restore-drill.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +workdir="${TMPDIR:-/tmp}/alfred-restore-drill.$$" +cleanup() { + rm -rf "$workdir" +} +trap cleanup EXIT + +src="$workdir/source" +backup="$workdir/backup" +restore="$workdir/restore" +mkdir -p "$src/vault_data/note" "$src/state_data" "$backup" "$restore" + +cat > "$src/vault_data/note/restore-drill.md" <<'EOF' +--- +type: note +name: restore-drill +--- + +A user-facing vault artifact used by the backup restore drill. +EOF + +python - "$src/state_data/alfred-state.db" <<'PY' +import sqlite3 +import sys +path = sys.argv[1] +con = sqlite3.connect(path) +con.execute("PRAGMA journal_mode=WAL") +con.execute("CREATE TABLE audit (id INTEGER PRIMARY KEY, message TEXT NOT NULL)") +con.execute("INSERT INTO audit(message) VALUES (?)", ("restore-drill-state-artifact",)) +con.commit() +con.close() +PY + +for volume in vault_data state_data; do + tar -C "$src/$volume" -czf "$backup/$volume.tgz" . +done + +mkdir -p "$restore/vault_data" "$restore/state_data" +for volume in vault_data state_data; do + tar -C "$restore/$volume" -xzf "$backup/$volume.tgz" +done + +test -f "$restore/vault_data/note/restore-drill.md" +grep -q "restore-drill" "$restore/vault_data/note/restore-drill.md" + +python - "$restore/state_data/alfred-state.db" <<'PY' +import sqlite3 +import sys +path = sys.argv[1] +con = sqlite3.connect(path) +row = con.execute("SELECT message FROM audit WHERE id = 1").fetchone() +con.close() +if row != ("restore-drill-state-artifact",): + raise SystemExit(f"unexpected restored state row: {row!r}") +PY + +printf 'restore drill passed: vault file and SQLite state artifact restored\n' From 771a725130564fad1ae27a834d568a734dd17c25 Mon Sep 17 00:00:00 2001 From: David Szabo-Stuban Date: Wed, 3 Jun 2026 13:13:36 +0200 Subject: [PATCH 5/5] test(backups): make restore drill executable --- scripts/restore-drill.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/restore-drill.sh diff --git a/scripts/restore-drill.sh b/scripts/restore-drill.sh old mode 100644 new mode 100755