terraform-snapshots #111
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: terraform-snapshots | |
| on: | |
| schedule: | |
| - cron: "0 3 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| project_id: | |
| description: "Scaleway Project ID" | |
| required: true | |
| default: "4ca4f592-005a-45dc-a016-91bd11dd5348" | |
| volume_name: | |
| description: "Block Volume name to snapshot" | |
| required: true | |
| default: "devsh-k3s-prod-data-node1" | |
| manual_ttl_hours: | |
| description: "TTL (hours) for manual snapshots (default 24)" | |
| required: false | |
| default: "24" | |
| manual_snapshot_name: | |
| description: "Optional manual snapshot key (auto-generated when empty)" | |
| required: false | |
| default: "" | |
| tfstate_bucket: | |
| description: "Object Storage bucket for Terraform state" | |
| required: true | |
| default: "terra-snapshots-state" | |
| tfstate_key: | |
| description: "Object Storage key for Terraform state" | |
| required: true | |
| default: "terraform/snapshots/terraform.tfstate" | |
| tfstate_region: | |
| description: "Object Storage region" | |
| required: true | |
| default: "fr-par" | |
| tfstate_endpoint: | |
| description: "Object Storage endpoint" | |
| required: true | |
| default: "https://s3.fr-par.scw.cloud" | |
| concurrency: | |
| group: terraform-snapshots-${{ github.ref_name }} | |
| cancel-in-progress: false | |
| permissions: | |
| contents: read | |
| jobs: | |
| snapshots: | |
| if: github.ref_name == 'env/prod' | |
| runs-on: ubuntu-latest | |
| environment: prod | |
| env: | |
| TF_IN_AUTOMATION: "true" | |
| TF_INPUT: "false" | |
| SCW_ACCESS_KEY: ${{ secrets.SNAPSHOTS_SCW_ACCESS_KEY }} | |
| SCW_SECRET_KEY: ${{ secrets.SNAPSHOTS_SCW_SECRET_KEY }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.SNAPSHOTS_SCW_ACCESS_KEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.SNAPSHOTS_SCW_SECRET_KEY }} | |
| SNAPSHOTS_TFSTATE_BUCKET: ${{ github.event.inputs.tfstate_bucket || 'terra-snapshots-state' }} | |
| SNAPSHOTS_TFSTATE_ENDPOINT: ${{ github.event.inputs.tfstate_endpoint || 'https://s3.fr-par.scw.cloud' }} | |
| SNAPSHOTS_TFSTATE_KEY: ${{ github.event.inputs.tfstate_key || 'terraform/snapshots/terraform.tfstate' }} | |
| SNAPSHOTS_TFSTATE_REGION: ${{ github.event.inputs.tfstate_region || 'fr-par' }} | |
| SNAPSHOTS_PROJECT_ID: ${{ github.event.inputs.project_id || '4ca4f592-005a-45dc-a016-91bd11dd5348' }} | |
| SNAPSHOTS_VOLUME_NAME: ${{ github.event.inputs.volume_name || 'devsh-k3s-prod-data-node1' }} | |
| SNAPSHOTS_MANUAL_TTL_HOURS: ${{ github.event.inputs.manual_ttl_hours || '24' }} | |
| SNAPSHOTS_MANUAL_SNAPSHOT_NAME: ${{ github.event.inputs.manual_snapshot_name || '' }} | |
| SNAPSHOTS_DISCORD_WEBHOOK_URL: ${{ secrets.SNAPSHOTS_DISCORD_WEBHOOK_URL }} | |
| TF_VAR_project_id: ${{ github.event.inputs.project_id || '4ca4f592-005a-45dc-a016-91bd11dd5348' }} | |
| TF_VAR_volume_name: ${{ github.event.inputs.volume_name || 'devsh-k3s-prod-data-node1' }} | |
| TF_VAR_env_name: prod | |
| defaults: | |
| run: | |
| shell: bash | |
| working-directory: terraform/snapshots | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: false | |
| fetch-depth: 1 | |
| - uses: hashicorp/setup-terraform@v3 | |
| with: | |
| terraform_version: 1.6.6 | |
| terraform_wrapper: false | |
| - name: Terraform init (remote state) | |
| run: | | |
| terraform init -input=false \ | |
| -backend-config="bucket=${SNAPSHOTS_TFSTATE_BUCKET}" \ | |
| -backend-config="key=${SNAPSHOTS_TFSTATE_KEY}" \ | |
| -backend-config="region=${SNAPSHOTS_TFSTATE_REGION}" \ | |
| -backend-config="endpoint=${SNAPSHOTS_TFSTATE_ENDPOINT}" \ | |
| -backend-config="skip_requesting_account_id=true" \ | |
| -backend-config="skip_credentials_validation=true" \ | |
| -backend-config="skip_metadata_api_check=true" \ | |
| -backend-config="skip_region_validation=true" \ | |
| -backend-config="force_path_style=true" | |
| - name: Load Terraform state inputs | |
| id: state | |
| run: | | |
| set -euo pipefail | |
| manual_snapshots="{}" | |
| if raw="$(terraform output -json manual_snapshots 2>/dev/null)"; then | |
| if parsed="$(echo "${raw}" | jq -c . 2>/dev/null)"; then | |
| if [[ "${parsed}" != "null" && -n "${parsed}" ]]; then | |
| manual_snapshots="${parsed}" | |
| fi | |
| fi | |
| fi | |
| auto_snapshot_trigger="" | |
| if out="$(terraform output -raw auto_snapshot_trigger 2>/dev/null)"; then | |
| auto_snapshot_trigger="${out}" | |
| fi | |
| if [[ -z "${auto_snapshot_trigger}" ]]; then | |
| auto_snapshot_trigger="$(terraform state pull 2>/dev/null | jq -r ' | |
| .resources[] | |
| | select(.type == "time_static" and .name == "snapshot_trigger") | |
| | .instances[0].attributes.triggers.bucket | |
| // empty | |
| ' 2>/dev/null || true)" | |
| fi | |
| if [[ "${auto_snapshot_trigger}" == "null" ]]; then | |
| auto_snapshot_trigger="" | |
| fi | |
| if [[ -n "${auto_snapshot_trigger}" ]] && ! [[ "${auto_snapshot_trigger}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then | |
| auto_snapshot_trigger="" | |
| fi | |
| echo "manual_snapshots=${manual_snapshots}" >> "${GITHUB_OUTPUT}" | |
| echo "auto_snapshot_trigger=${auto_snapshot_trigger}" >> "${GITHUB_OUTPUT}" | |
| - name: Prepare Terraform variables | |
| id: vars | |
| run: | | |
| set -euo pipefail | |
| manual_snapshots='${{ steps.state.outputs.manual_snapshots }}' | |
| auto_snapshot_trigger_state='${{ steps.state.outputs.auto_snapshot_trigger }}' | |
| auto_snapshot_trigger_refresh="${auto_snapshot_trigger_state}" | |
| if [[ -z "${auto_snapshot_trigger_refresh}" || "${auto_snapshot_trigger_refresh}" == "null" ]] || ! [[ "${auto_snapshot_trigger_refresh}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then | |
| auto_snapshot_trigger_refresh="$(date -u +%Y-%m-%d)" | |
| fi | |
| if terraform state pull >/dev/null 2>&1; then | |
| jq -n \ | |
| --arg auto_snapshot_trigger "${auto_snapshot_trigger_refresh}" \ | |
| --arg requested_manual_snapshot_name "" \ | |
| --argjson manual_snapshots "${manual_snapshots}" \ | |
| '{auto_snapshot_trigger: $auto_snapshot_trigger, requested_manual_snapshot_name: $requested_manual_snapshot_name, manual_snapshots: $manual_snapshots}' \ | |
| > refresh.auto.tfvars.json | |
| terraform apply -refresh-only -auto-approve -no-color -parallelism=1 -var-file=refresh.auto.tfvars.json | |
| keep_keys="$(terraform output -json manual_snapshot_ids 2>/dev/null | jq -c 'keys' 2>/dev/null || echo '[]')" | |
| manual_snapshots="$(echo "${manual_snapshots}" | jq -c --argjson keep "${keep_keys}" 'with_entries(select(.key as $k | $keep | index($k)))')" | |
| else | |
| echo "No Terraform state yet; skipping refresh-only manual snapshot pruning." | |
| fi | |
| auto_snapshot_trigger="${auto_snapshot_trigger_state}" | |
| if [[ "${{ github.event_name }}" == "schedule" ]]; then | |
| auto_snapshot_trigger="$(date -u +%Y-%m-%d)" | |
| fi | |
| if [[ -z "${auto_snapshot_trigger}" || "${auto_snapshot_trigger}" == "null" ]]; then | |
| auto_snapshot_trigger="$(date -u +%Y-%m-%d)" | |
| fi | |
| if ! [[ "${auto_snapshot_trigger}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]; then | |
| auto_snapshot_trigger="$(date -u +%Y-%m-%d)" | |
| fi | |
| requested_manual_snapshot_name="" | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then | |
| ttl_hours="${SNAPSHOTS_MANUAL_TTL_HOURS}" | |
| if ! [[ "${ttl_hours}" =~ ^[0-9]+$ ]] || (( ttl_hours <= 0 )); then | |
| echo "Invalid manual_ttl_hours='${ttl_hours}' (expected positive integer)" >&2 | |
| exit 1 | |
| fi | |
| created_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" | |
| ts="$(date -u +%Y%m%dT%H%M%SZ)" | |
| requested_manual_snapshot_name="${SNAPSHOTS_MANUAL_SNAPSHOT_NAME}" | |
| if [[ -z "${requested_manual_snapshot_name}" ]]; then | |
| requested_manual_snapshot_name="${ts}-run${GITHUB_RUN_ID}" | |
| fi | |
| manual_snapshots="$(echo "${manual_snapshots}" | jq -c \ | |
| --arg name "${requested_manual_snapshot_name}" \ | |
| --arg created_at "${created_at}" \ | |
| --argjson ttl_hours "${ttl_hours}" \ | |
| '. + {($name): {created_at: $created_at, ttl_hours: $ttl_hours}}')" | |
| fi | |
| jq -n \ | |
| --arg auto_snapshot_trigger "${auto_snapshot_trigger}" \ | |
| --arg requested_manual_snapshot_name "${requested_manual_snapshot_name}" \ | |
| --argjson manual_snapshots "${manual_snapshots}" \ | |
| '{auto_snapshot_trigger: $auto_snapshot_trigger, requested_manual_snapshot_name: $requested_manual_snapshot_name, manual_snapshots: $manual_snapshots}' \ | |
| > ci.auto.tfvars.json | |
| echo "auto_snapshot_trigger=${auto_snapshot_trigger}" >> "${GITHUB_OUTPUT}" | |
| echo "requested_manual_snapshot_name=${requested_manual_snapshot_name}" >> "${GITHUB_OUTPUT}" | |
| - name: Terraform apply | |
| id: apply | |
| run: | | |
| set -euo pipefail | |
| terraform apply -auto-approve -no-color -parallelism=1 -var-file=ci.auto.tfvars.json 2>&1 | tee terraform-apply.log | |
| - name: Capture snapshot info | |
| id: snapshot | |
| run: | | |
| set -euo pipefail | |
| volume_id="$(terraform output -raw volume_id)" | |
| if [[ "${{ github.event_name }}" == "schedule" ]]; then | |
| snapshot_kind="auto" | |
| snapshot_id="$(terraform output -raw latest_snapshot_id)" | |
| snapshot_name="$(terraform output -raw latest_snapshot_name)" | |
| retention="24h (rotating, max 1 kept)" | |
| else | |
| snapshot_kind="manual" | |
| snapshot_id="$(terraform output -raw requested_manual_snapshot_id)" | |
| snapshot_name="$(terraform output -raw requested_manual_snapshot_name)" | |
| retention="${SNAPSHOTS_MANUAL_TTL_HOURS}h (Terraform-managed)" | |
| fi | |
| if [[ -z "${snapshot_id}" ]]; then | |
| echo "Snapshot ID is empty (kind=${snapshot_kind})" >&2 | |
| exit 1 | |
| fi | |
| echo "snapshot_kind=${snapshot_kind}" >> "${GITHUB_OUTPUT}" | |
| echo "snapshot_id=${snapshot_id}" >> "${GITHUB_OUTPUT}" | |
| echo "snapshot_name=${snapshot_name}" >> "${GITHUB_OUTPUT}" | |
| echo "volume_id=${volume_id}" >> "${GITHUB_OUTPUT}" | |
| echo "retention=${retention}" >> "${GITHUB_OUTPUT}" | |
| - name: Show snapshot details | |
| env: | |
| SNAPSHOT_ID: ${{ steps.snapshot.outputs.snapshot_id }} | |
| SNAPSHOT_NAME: ${{ steps.snapshot.outputs.snapshot_name }} | |
| SNAPSHOT_KIND: ${{ steps.snapshot.outputs.snapshot_kind }} | |
| RETENTION: ${{ steps.snapshot.outputs.retention }} | |
| run: | | |
| echo "Snapshot kind: ${SNAPSHOT_KIND}" | |
| echo "Snapshot name: ${SNAPSHOT_NAME}" | |
| echo "Snapshot ID: ${SNAPSHOT_ID}" | |
| echo "Retention: ${RETENTION}" | |
| - name: Notify Discord (snapshot status) | |
| if: ${{ always() }} | |
| env: | |
| JOB_STATUS: ${{ job.status }} | |
| APPLY_OUTCOME: ${{ steps.apply.outcome }} | |
| SNAPSHOT_ID: ${{ steps.snapshot.outputs.snapshot_id }} | |
| SNAPSHOT_NAME: ${{ steps.snapshot.outputs.snapshot_name }} | |
| SNAPSHOT_KIND: ${{ steps.snapshot.outputs.snapshot_kind }} | |
| RETENTION: ${{ steps.snapshot.outputs.retention }} | |
| VOLUME_ID: ${{ steps.snapshot.outputs.volume_id }} | |
| AUTO_SNAPSHOT_TRIGGER: ${{ steps.vars.outputs.auto_snapshot_trigger }} | |
| REQUESTED_MANUAL_SNAPSHOT_KEY: ${{ steps.vars.outputs.requested_manual_snapshot_name }} | |
| run: | | |
| set -euo pipefail | |
| if [[ -z "${SNAPSHOTS_DISCORD_WEBHOOK_URL}" ]]; then | |
| echo "SNAPSHOTS_DISCORD_WEBHOOK_URL is empty; skipping Discord notification." | |
| exit 0 | |
| fi | |
| run_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| trigger="${{ github.event_name }}" | |
| schedule="${{ github.event.schedule }}" | |
| branch="${{ github.ref_name }}" | |
| actor="${{ github.actor }}" | |
| ts="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" | |
| trigger_label="${trigger}" | |
| if [[ "${trigger}" == "schedule" && -n "${schedule}" ]]; then | |
| trigger_label="cron (${schedule})" | |
| elif [[ "${trigger}" == "workflow_dispatch" ]]; then | |
| trigger_label="dispatch" | |
| fi | |
| status="${JOB_STATUS}" | |
| status_label="${status}" | |
| color=9807270 | |
| title="Scaleway snapshot ${status_label}" | |
| snapshot_kind="${SNAPSHOT_KIND}" | |
| snapshot_id="${SNAPSHOT_ID}" | |
| snapshot_name="${SNAPSHOT_NAME}" | |
| retention="${RETENTION}" | |
| if [[ "${trigger}" == "workflow_dispatch" ]]; then | |
| snapshot_kind="${snapshot_kind:-manual}" | |
| if [[ -z "${snapshot_name}" && -n "${REQUESTED_MANUAL_SNAPSHOT_KEY}" ]]; then | |
| snapshot_name="devsh-k3s-prod-data-manual-${REQUESTED_MANUAL_SNAPSHOT_KEY}" | |
| fi | |
| retention="${retention:-${SNAPSHOTS_MANUAL_TTL_HOURS}h (Terraform-managed)}" | |
| else | |
| snapshot_kind="${snapshot_kind:-auto}" | |
| retention="${retention:-24h (rotating, max 1 kept)}" | |
| fi | |
| if [[ "${status}" == "success" ]]; then | |
| color=3066993 | |
| title="Scaleway snapshot created (${snapshot_kind})" | |
| if [[ "${snapshot_kind}" == "auto" ]]; then | |
| color=3447003 | |
| fi | |
| else | |
| color=15158332 | |
| title="Scaleway snapshot failed (${snapshot_kind})" | |
| fi | |
| log_tail="" | |
| if [[ "${status}" != "success" && -f terraform-apply.log ]]; then | |
| log_tail="$(tail -n 60 terraform-apply.log | sed -e 's/\r$//' | tail -n 30)" | |
| log_tail="${log_tail:0:900}" | |
| fi | |
| payload="$(jq -n \ | |
| --arg title "${title}" \ | |
| --arg status "${status}" \ | |
| --arg kind "${snapshot_kind}" \ | |
| --arg trigger "${trigger_label}" \ | |
| --arg branch "${branch}" \ | |
| --arg actor "${actor}" \ | |
| --arg snapshot_name "${snapshot_name}" \ | |
| --arg snapshot_id "${snapshot_id}" \ | |
| --arg volume_name "${SNAPSHOTS_VOLUME_NAME}" \ | |
| --arg volume_id "${VOLUME_ID}" \ | |
| --arg project_id "${SNAPSHOTS_PROJECT_ID}" \ | |
| --arg retention "${retention}" \ | |
| --arg run_url "${run_url}" \ | |
| --arg timestamp "${ts}" \ | |
| --arg log_tail "${log_tail}" \ | |
| --arg auto_trigger "${AUTO_SNAPSHOT_TRIGGER}" \ | |
| --argjson color "${color}" \ | |
| '{ | |
| username: "TerraInfra", | |
| embeds: [ | |
| { | |
| title: $title, | |
| color: $color, | |
| fields: [ | |
| {name: "Status", value: $status, inline: true}, | |
| {name: "Type", value: $kind, inline: true}, | |
| {name: "Trigger", value: $trigger, inline: true}, | |
| {name: "Branch", value: $branch, inline: true}, | |
| {name: "Snapshot", value: (if ($snapshot_name | length) > 0 then ("`" + $snapshot_name + "`") else "n/a" end), inline: false}, | |
| {name: "Snapshot ID", value: (if ($snapshot_id | length) > 0 then ("`" + $snapshot_id + "`") else "n/a" end), inline: false}, | |
| {name: "Volume", value: ("`" + $volume_name + "`\n`" + $volume_id + "`"), inline: false}, | |
| {name: "Project", value: ("`" + $project_id + "`"), inline: false}, | |
| {name: "Retention", value: $retention, inline: true}, | |
| {name: "GitHub run", value: ("[Open run](" + $run_url + ")"), inline: true}, | |
| {name: "Actor", value: $actor, inline: true} | |
| ], | |
| timestamp: $timestamp | |
| } | |
| ] | |
| }')" | |
| if [[ "${status}" != "success" ]]; then | |
| payload="$(echo "${payload}" | jq -c --arg log_tail "${log_tail}" --arg auto_trigger "${AUTO_SNAPSHOT_TRIGGER}" ' | |
| .embeds[0].fields += [ | |
| {name: "Auto trigger", value: (if ($auto_trigger | length) > 0 then ("`" + $auto_trigger + "`") else "n/a" end), inline: true} | |
| ] | | |
| (if ($log_tail | length) > 0 then | |
| .embeds[0].fields += [{name: "Terraform (tail)", value: ("```" + $log_tail + "```"), inline: false}] | |
| else | |
| . | |
| end) | |
| ')" | |
| fi | |
| curl -fsS -H 'Content-Type: application/json' -d "${payload}" "${SNAPSHOTS_DISCORD_WEBHOOK_URL}" || echo "Discord webhook notification failed" >&2 |