diff --git a/.github/actions/apply-infrastructure-changes/action.yaml b/.github/actions/apply-infrastructure-changes/action.yaml new file mode 100644 index 0000000000..ae4d51492a --- /dev/null +++ b/.github/actions/apply-infrastructure-changes/action.yaml @@ -0,0 +1,98 @@ +name: Apply infrastructure changes +description: Run IaC plan and apply steps + +inputs: + plugin-cache-dir: + description: Directory to store Terraform plugin cache + required: false + default: /home/runner/.terraform.d/plugin-cache + summary-title: + description: Title used for the plan output summary + required: false + default: Terraform Plan Output + tfbackend-path: + description: Path to write Terraform backend config + required: true + tfbackend-content: + description: Content for Terraform backend config + required: true + tfvars-path: + description: Path to write Terraform variables + required: true + tfvars-content: + description: Content for Terraform variables + required: true + working-directory: + description: Terraform working directory + required: true + +runs: + using: composite + steps: + - name: Install Terraform + uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 + with: + terraform_version: 1.14 + + - name: Create plugin cache directory + env: + TF_PLUGIN_CACHE_DIR: ${{ inputs.plugin-cache-dir }} + run: mkdir -p "${TF_PLUGIN_CACHE_DIR}" + shell: bash + + - name: Cache Terraform plugins + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + key: ${{ runner.os }}-infrastructure-${{ hashFiles('infrastructure/**/.terraform.lock.hcl') }} + path: ${{ inputs.plugin-cache-dir }} + + - name: Prepare Terraform backend + env: + TF_BACKEND_CONTENT: ${{ inputs.tfbackend-content }} + TF_BACKEND_PATH: ${{ inputs.tfbackend-path }} + run: | + umask 377 + printf '%s\n' "$TF_BACKEND_CONTENT" > "$TF_BACKEND_PATH" + shell: bash + + - name: Prepare Terraform variables + env: + TF_VARS_CONTENT: ${{ inputs.tfvars-content }} + TF_VARS_PATH: ${{ inputs.tfvars-path }} + run: | + umask 377 + printf '%s\n' "$TF_VARS_CONTENT" > "$TF_VARS_PATH" + shell: bash + + - name: Initialize Terraform + env: + TF_PLUGIN_CACHE_DIR: ${{ inputs.plugin-cache-dir }} + run: terraform init -backend-config=terraform.tfbackend + shell: bash + working-directory: ${{ inputs.working-directory }} + + - name: Validate Terraform configuration + run: terraform validate + shell: bash + working-directory: ${{ inputs.working-directory }} + + - name: Plan Terraform changes + run: terraform plan -out=tfplan + shell: bash + working-directory: ${{ inputs.working-directory }} + + - name: Show plan summary + env: + SUMMARY_TITLE: ${{ inputs.summary-title }} + run: | + echo "## $SUMMARY_TITLE" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + terraform show -no-color tfplan >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + shell: bash + working-directory: ${{ inputs.working-directory }} + + - name: Apply Terraform changes + run: terraform apply -auto-approve tfplan + shell: bash + working-directory: ${{ inputs.working-directory }} diff --git a/.github/actions/install-backend-dependencies/action.yaml b/.github/actions/install-backend-dependencies/action.yaml new file mode 100644 index 0000000000..6f1f43d71b --- /dev/null +++ b/.github/actions/install-backend-dependencies/action.yaml @@ -0,0 +1,28 @@ +name: Install backend dependencies + +description: >- + Installs Python and Poetry, and installs backend dependencies. + +runs: + using: composite + steps: + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.13' + + - name: Install Poetry + uses: ./.github/actions/install-poetry + + - name: Cache backend dependencies + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + key: poetry-venv-${{ runner.os }}-${{ hashFiles('backend/poetry.lock') }} + path: backend/.venv + restore-keys: | + poetry-venv-${{ runner.os }}- + + - name: Install backend dependencies + run: poetry install --no-interaction --no-root + shell: bash + working-directory: backend diff --git a/.github/actions/install-frontend-dependencies/action.yaml b/.github/actions/install-frontend-dependencies/action.yaml new file mode 100644 index 0000000000..87af727bcb --- /dev/null +++ b/.github/actions/install-frontend-dependencies/action.yaml @@ -0,0 +1,24 @@ +name: Install frontend dependencies + +description: >- + Installs pnpm and Node, and installs frontend dependencies. + +runs: + using: composite + steps: + - name: Install pnpm + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 + with: + version: 11.5.1 + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + cache: pnpm + cache-dependency-path: frontend/pnpm-lock.yaml + node-version: 24 + + - name: Install frontend dependencies + run: pnpm install --frozen-lockfile + shell: bash + working-directory: frontend diff --git a/.github/actions/install-poetry/action.yaml b/.github/actions/install-poetry/action.yaml new file mode 100644 index 0000000000..b4b5fdd2c8 --- /dev/null +++ b/.github/actions/install-poetry/action.yaml @@ -0,0 +1,32 @@ +name: Install Poetry + +description: >- + Installs Poetry from backend/requirements.build.txt with pip cache. Run after actions/setup-python. + +runs: + using: composite + steps: + - name: Cache pip for Poetry install + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + key: pip-poetry-${{ runner.os }}-${{ hashFiles('backend/requirements.build.txt') }} + path: ${{ runner.temp }}/poetry-ci-home/.cache/pip + restore-keys: | + pip-poetry-${{ runner.os }}- + + - name: Prepare pip cache directory + env: + POETRY_CI_HOME: ${{ runner.temp }}/poetry-ci-home + run: | + set -euo pipefail + mkdir -p "$POETRY_CI_HOME/.cache/pip" + chown -R "$(id -u):$(id -g)" "$POETRY_CI_HOME" + chmod -R u+rwX "$POETRY_CI_HOME" + shell: bash + + - name: Install Poetry + env: + HOME: ${{ runner.temp }}/poetry-ci-home + PIP_ROOT_USER_ACTION: ignore + run: python -m pip install --requirement backend/requirements.build.txt + shell: bash diff --git a/.github/actions/run-trivy-scan/action.yaml b/.github/actions/run-trivy-scan/action.yaml new file mode 100644 index 0000000000..17748469ce --- /dev/null +++ b/.github/actions/run-trivy-scan/action.yaml @@ -0,0 +1,30 @@ +name: Run Trivy scan +description: Configure Trivy cache and run Trivy scan + +inputs: + command: + description: Command to run the Trivy scan + required: true + +runs: + using: composite + steps: + - name: Set Trivy cache key + env: + DOCKERFILE_HASH: ${{ hashFiles('docker/trivy/Dockerfile') }} + RUNNER_OS: ${{ runner.os }} + run: echo "TRIVY_CACHE_KEY=trivy-${RUNNER_OS}-${DOCKERFILE_HASH}" >> "$GITHUB_ENV" + shell: bash + + - name: Cache Trivy + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + key: ${{ env.TRIVY_CACHE_KEY }} + path: .trivy-cache + restore-keys: trivy-${{ runner.os }}- + + - name: Run Trivy scan + env: + TRIVY_RUN_COMMAND: ${{ inputs.command }} + run: bash -c "$TRIVY_RUN_COMMAND" + shell: bash diff --git a/.github/actions/setup-backend-environment/action.yaml b/.github/actions/setup-backend-environment/action.yaml new file mode 100644 index 0000000000..9606713a05 --- /dev/null +++ b/.github/actions/setup-backend-environment/action.yaml @@ -0,0 +1,104 @@ +name: Set up Backend environment + +description: >- + Fetches nest.dump from S3 using the same Poetry environment as local Make targets, waits for + the Postgres service container, runs migrations, restores the database dump, starts the backend, + and waits for it to be ready. + +inputs: + backend_port: + description: Backend port to bind + required: true + db_name: + description: Database name + required: true + db_password: + description: Database password + required: true + db_username: + description: Database username + required: true + env_file: + description: Backend environment file + required: true + +runs: + using: composite + steps: + - name: Install backend dependencies + uses: ./.github/actions/install-backend-dependencies + + - name: Fetch nest.dump from S3 + run: poetry run python -m scripts.fetch_nest_dump + shell: bash + working-directory: backend + + - name: Wait for database to be ready + env: + DB_NAME: ${{ inputs.db_name }} + DB_USERNAME: ${{ inputs.db_username }} + run: | + timeout 5m bash -c ' + until docker exec ${{ job.services.db.id }} pg_isready -U "$DB_USERNAME" -d "$DB_NAME"; do + echo "Waiting for database..." + sleep 5 + done + ' + shell: bash + + - name: Migrate Database + env: + ENV_FILE: ${{ inputs.env_file }} + run: | + set -euo pipefail && set -a && source $ENV_FILE && set +a + export DJANGO_DB_HOST=localhost + export DJANGO_REDIS_AUTH_ENABLED=False + export DJANGO_REDIS_HOST=localhost + poetry run python manage.py migrate + shell: bash + working-directory: backend + + - name: Load Postgres data + env: + DB_NAME: ${{ inputs.db_name }} + DB_USERNAME: ${{ inputs.db_username }} + PGPASSWORD: ${{ inputs.db_password }} # The env name required by PostgreSQL clients. + run: | + set -euo pipefail + if ! docker exec -i -e PGPASSWORD="$PGPASSWORD" ${{ job.services.db.id }} \ + pg_restore -U "$DB_USERNAME" -d "$DB_NAME" < backend/data/nest.dump; then + echo "Data loading failed" + exit 1 + fi + echo "Data loading completed." + shell: bash + + - name: Start Backend in the background + env: + BACKEND_PORT: ${{ inputs.backend_port }} + ENV_FILE: ${{ inputs.env_file }} + run: | + set -euo pipefail && set -a && source $ENV_FILE && set +a + export DJANGO_DB_HOST=localhost + export DJANGO_REDIS_AUTH_ENABLED=False + export DJANGO_REDIS_HOST=localhost + nohup poetry run gunicorn wsgi:application \ + --bind 0.0.0.0:$BACKEND_PORT \ + --workers 2 \ + > /dev/null 2>&1 & + disown + shell: bash + working-directory: backend + + - name: Waiting for the backend to be ready + env: + BACKEND_PORT: ${{ inputs.backend_port }} + run: | + timeout 5m bash -c ' + until wget --spider http://localhost:$BACKEND_PORT/a; do + echo "Waiting for backend..." + sleep 5 + done + ' + echo "Backend is up!" + shell: bash diff --git a/.github/labeler.yml b/.github/pr-labeler.yml similarity index 100% rename from .github/labeler.yml rename to .github/pr-labeler.yml diff --git a/.github/workflows/ci-cd-production.yaml b/.github/workflows/ci-cd-production.yaml new file mode 100644 index 0000000000..63b6172f20 --- /dev/null +++ b/.github/workflows/ci-cd-production.yaml @@ -0,0 +1,59 @@ +name: CI/CD Production + +on: + release: + types: + - published + +concurrency: + cancel-in-progress: false + group: ci-cd-production + +permissions: {} + +env: + FORCE_COLOR: 1 + +jobs: + run-ci-cd: + name: Run production CI/CD + permissions: + actions: write + contents: write + id-token: write + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + AWS_ROLE_EXTERNAL_ID: ${{ secrets.AWS_ROLE_EXTERNAL_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + BOOTSTRAP_AWS_ACCESS_KEY_ID: ${{ secrets.BOOTSTRAP_AWS_ACCESS_KEY_ID }} + BOOTSTRAP_AWS_SECRET_ACCESS_KEY: ${{ secrets.BOOTSTRAP_AWS_SECRET_ACCESS_KEY }} + BOOTSTRAP_TF_STATE_BUCKET_NAME: ${{ secrets.BOOTSTRAP_TF_STATE_BUCKET_NAME }} + NEXT_PUBLIC_GTM_ID: ${{ secrets.NEXT_PUBLIC_GTM_ID }} + NEXT_PUBLIC_IS_PROJECT_HEALTH_ENABLED: ${{ secrets.NEXT_PUBLIC_IS_PROJECT_HEALTH_ENABLED }} + NEXT_PUBLIC_POSTHOG_HOST: ${{ secrets.NEXT_PUBLIC_POSTHOG_HOST }} + NEXT_PUBLIC_POSTHOG_KEY: ${{ secrets.NEXT_PUBLIC_POSTHOG_KEY }} + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + TF_STATE_BUCKET_NAME: ${{ secrets.TF_STATE_BUCKET_NAME }} + VITE_API_URL: ${{ secrets.VITE_API_URL }} + VITE_CSRF_URL: ${{ secrets.VITE_CSRF_URL }} + VITE_ENVIRONMENT: ${{ secrets.VITE_ENVIRONMENT }} + VITE_GRAPHQL_URL: ${{ secrets.VITE_GRAPHQL_URL }} + VITE_IDX_URL: ${{ secrets.VITE_IDX_URL }} + VITE_SENTRY_DSN: ${{ secrets.VITE_SENTRY_DSN }} + uses: ./.github/workflows/run-ci-cd.yaml + with: + backend_gid: '1002' + backend_uid: '1002' + backend_use_fargate_spot: false + base_url: https://nest.owasp.org + django_configuration: Production + django_settings_module: settings.production + enable_additional_parameters: true + enable_cron_tasks: true + enable_nat_gateway: true + enable_rds_proxy: true + environment: production + frontend_use_fargate_spot: false + release_tag: ${{ github.event.release.tag_name }} + tasks_use_fargate_spot: false diff --git a/.github/workflows/ci-cd-staging.yaml b/.github/workflows/ci-cd-staging.yaml new file mode 100644 index 0000000000..a2dc2f09fa --- /dev/null +++ b/.github/workflows/ci-cd-staging.yaml @@ -0,0 +1,57 @@ +name: CI/CD Staging + +on: + schedule: + - cron: 0 2 * * * + workflow_dispatch: + +concurrency: + cancel-in-progress: false + group: ci-cd-staging + +permissions: {} + +env: + FORCE_COLOR: 1 + +jobs: + run-ci-cd: + name: Run staging CI/CD + permissions: + actions: write + contents: read + id-token: write + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + AWS_ROLE_EXTERNAL_ID: ${{ secrets.AWS_ROLE_EXTERNAL_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + BOOTSTRAP_AWS_ACCESS_KEY_ID: ${{ secrets.BOOTSTRAP_AWS_ACCESS_KEY_ID }} + BOOTSTRAP_AWS_SECRET_ACCESS_KEY: ${{ secrets.BOOTSTRAP_AWS_SECRET_ACCESS_KEY }} + BOOTSTRAP_TF_STATE_BUCKET_NAME: ${{ secrets.BOOTSTRAP_TF_STATE_BUCKET_NAME }} + NEXT_PUBLIC_GTM_ID: ${{ secrets.NEXT_PUBLIC_GTM_ID }} + NEXT_PUBLIC_IS_PROJECT_HEALTH_ENABLED: ${{ secrets.NEXT_PUBLIC_IS_PROJECT_HEALTH_ENABLED }} + NEXT_PUBLIC_POSTHOG_HOST: ${{ secrets.NEXT_PUBLIC_POSTHOG_HOST }} + NEXT_PUBLIC_POSTHOG_KEY: ${{ secrets.NEXT_PUBLIC_POSTHOG_KEY }} + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + TF_STATE_BUCKET_NAME: ${{ secrets.TF_STATE_BUCKET_NAME }} + VITE_API_URL: ${{ secrets.VITE_API_URL }} + VITE_CSRF_URL: ${{ secrets.VITE_CSRF_URL }} + VITE_ENVIRONMENT: ${{ secrets.VITE_ENVIRONMENT }} + VITE_GRAPHQL_URL: ${{ secrets.VITE_GRAPHQL_URL }} + VITE_IDX_URL: ${{ secrets.VITE_IDX_URL }} + VITE_SENTRY_DSN: ${{ secrets.VITE_SENTRY_DSN }} + uses: ./.github/workflows/run-ci-cd.yaml + with: + backend_gid: '1001' + backend_uid: '1001' + backend_use_fargate_spot: true + base_url: https://nest.owasp.dev + django_configuration: Staging + django_settings_module: settings.staging + enable_cron_tasks: false + enable_nat_gateway: false + enable_rds_proxy: false + environment: staging + frontend_use_fargate_spot: true + tasks_use_fargate_spot: true diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000000..609fa027d6 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,38 @@ +name: CI + +on: + merge_group: + pull_request: + branches: + - feature/** + - main + push: + branches: + - feature/** + - main + +concurrency: + cancel-in-progress: true + group: ci-${{ github.workflow }}-${{ github.ref }} + +permissions: {} + +env: + FORCE_COLOR: 1 + +jobs: + run-code-checks: + name: Code checks + permissions: + contents: read + uses: ./.github/workflows/run-code-checks.yaml + + run-code-tests: + name: Code tests + needs: + - run-code-checks + permissions: + actions: write + contents: read + id-token: write + uses: ./.github/workflows/run-code-tests.yaml diff --git a/.github/workflows/run-code-ql.yaml b/.github/workflows/code-ql.yaml similarity index 59% rename from .github/workflows/run-code-ql.yaml rename to .github/workflows/code-ql.yaml index b7b8892333..535597dda6 100644 --- a/.github/workflows/run-code-ql.yaml +++ b/.github/workflows/code-ql.yaml @@ -1,4 +1,4 @@ -name: Run CodeQL +name: CodeQL on: merge_group: @@ -10,12 +10,18 @@ on: branches: - feature/** - main - workflow_dispatch: + +concurrency: + cancel-in-progress: true + group: code-ql-${{ github.workflow }}-${{ github.ref }} permissions: {} +env: + FORCE_COLOR: 1 + jobs: - code-ql: + run-code-ql: name: CodeQL permissions: contents: read @@ -37,27 +43,16 @@ jobs: with: languages: ${{ matrix.language }} - - name: Install pnpm - uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 - with: - run_install: false - version: 11.5.1 - - - name: Set up Node - if: matrix.language == 'javascript-typescript' - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - cache-dependency-path: frontend/pnpm-lock.yaml - cache: pnpm - node-version: 24 + - name: Install backend dependencies + if: matrix.language == 'python' + uses: ./.github/actions/install-backend-dependencies - - name: Install dependencies for frontend + - name: Install frontend dependencies if: matrix.language == 'javascript-typescript' - run: pnpm install --frozen-lockfile - working-directory: frontend + uses: ./.github/actions/install-frontend-dependencies - name: Perform CodeQL analysis uses: github/codeql-action/analyze@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5 with: category: /language:${{ matrix.language }} - timeout-minutes: 5 + timeout-minutes: 15 diff --git a/.github/workflows/run-dependency-review.yaml b/.github/workflows/dependency-review.yaml similarity index 54% rename from .github/workflows/run-dependency-review.yaml rename to .github/workflows/dependency-review.yaml index 112fc81177..1d8c89267a 100644 --- a/.github/workflows/run-dependency-review.yaml +++ b/.github/workflows/dependency-review.yaml @@ -1,24 +1,34 @@ -name: Run dependency review +name: Dependency review on: pull_request: branches: - - dev - feature/** - main + paths: + - '**/package.json' + - '**/pnpm-lock.yaml' + - '**/poetry.lock' + - '**/pyproject.toml' + - backend/requirements.build.txt + +concurrency: + cancel-in-progress: true + group: dependency-review-${{ github.workflow }}-${{ github.ref }} permissions: {} jobs: dependency-review: - name: Review dependencies + name: Dependency review permissions: contents: read - pull-requests: read runs-on: ubuntu-latest steps: - name: Check out repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Dependency review uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0 diff --git a/.github/workflows/label-issues.yaml b/.github/workflows/label-issues.yaml deleted file mode 100644 index 170036c2d7..0000000000 --- a/.github/workflows/label-issues.yaml +++ /dev/null @@ -1,45 +0,0 @@ -name: Auto Label Issues - -on: - issues: - types: - - edited - - opened - -permissions: {} - -jobs: - label: - permissions: - issues: write - runs-on: ubuntu-latest - steps: - - name: Apply Labels to Issues - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - script: | - const issue = context.payload.issue; - if (!issue || (!issue.title && !issue.body)) return; - - const title = issue.title?.toLowerCase() || ""; - const body = issue.body?.toLowerCase() || ""; - - const keywordMapping = { - "bug": ["error", "failure", "not working", "crash", "unexpected"], - "enhancement": ["add", "feature request", "improve", "suggestion"], - "question": ["clarification", "help", "how to", "guidance"] - }; - - const labels = Object.entries(keywordMapping) - .filter(([_, words]) => words.some(word => title.includes(word) || body.includes(word))) - .map(([label, _]) => label); - - if (labels.length > 0) { - github.rest.issues.addLabels({ - issue_number: context.issue.number, - labels: labels, - owner: context.repo.owner, - repo: context.repo.repo - }); - } - timeout-minutes: 5 diff --git a/.github/workflows/label-pull-requests.yaml b/.github/workflows/pr-labeler.yaml similarity index 86% rename from .github/workflows/label-pull-requests.yaml rename to .github/workflows/pr-labeler.yaml index f638054d7f..225bc6e989 100644 --- a/.github/workflows/label-pull-requests.yaml +++ b/.github/workflows/pr-labeler.yaml @@ -14,6 +14,6 @@ jobs: steps: - uses: actions/labeler@f27b608878404679385c85cfa523b85ccb86e213 # v6.1.0 with: - configuration-path: .github/labeler.yml + configuration-path: .github/pr-labeler.yml sync-labels: true timeout-minutes: 5 diff --git a/.github/workflows/run-backend-tests.yaml b/.github/workflows/run-backend-tests.yaml new file mode 100644 index 0000000000..e4ce1167eb --- /dev/null +++ b/.github/workflows/run-backend-tests.yaml @@ -0,0 +1,37 @@ +name: Backend tests + +on: + workflow_call: + +env: + FORCE_COLOR: 1 + +permissions: {} + +jobs: + run-unit-tests: + name: Backend unit tests + permissions: + actions: write + contents: read + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install backend dependencies + uses: ./.github/actions/install-backend-dependencies + + - name: Run backend tests + run: set -a && source .env.unit-tests && set +a && poetry run pytest tests/unit + working-directory: backend + + - name: Upload coverage artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + if-no-files-found: error + name: coverage-xml + path: backend/coverage.xml + timeout-minutes: 5 diff --git a/.github/workflows/run-ci-cd.yaml b/.github/workflows/run-ci-cd.yaml index 54425ba516..16bdd5bc61 100644 --- a/.github/workflows/run-ci-cd.yaml +++ b/.github/workflows/run-ci-cd.yaml @@ -1,23 +1,105 @@ -name: Run CI/CD +name: CI/CD on: - merge_group: - pull_request: - branches: - - dev - - feature/** - - main - push: - branches: - - dev - - feature/** - - main - release: - types: - - published - schedule: - - cron: 0 2 * * * # Run staging deploy daily at 2am UTC. - workflow_dispatch: + workflow_call: + inputs: + backend_gid: + description: Backend GID for Docker build + required: true + type: string + backend_uid: + description: Backend UID for Docker build + required: true + type: string + backend_use_fargate_spot: + description: Use Fargate spot for backend + required: true + type: boolean + base_url: + description: The base URL for Lighthouse and ZAP scans + required: true + type: string + django_configuration: + description: Django configuration string + required: true + type: string + django_settings_module: + description: Django settings module + required: true + type: string + enable_additional_parameters: + description: Enable additional SSM parameters + required: false + default: false + type: boolean + enable_cron_tasks: + description: Enable cron tasks + required: true + type: boolean + enable_nat_gateway: + description: Enable NAT gateway + required: true + type: boolean + enable_rds_proxy: + description: Enable RDS proxy + required: true + type: boolean + environment: + description: The environment name (e.g., staging, production) + required: true + type: string + frontend_use_fargate_spot: + description: Use Fargate spot for frontend + required: true + type: boolean + release_tag: + description: Release tag for uploading SBOM (Production only) + required: false + type: string + default: '' + tasks_use_fargate_spot: + description: Use Fargate spot for tasks + required: true + type: boolean + secrets: + AWS_ACCESS_KEY_ID: + required: true + AWS_ACCOUNT_ID: + required: true + AWS_ROLE_EXTERNAL_ID: + required: true + AWS_SECRET_ACCESS_KEY: + required: true + BOOTSTRAP_AWS_ACCESS_KEY_ID: + required: true + BOOTSTRAP_AWS_SECRET_ACCESS_KEY: + required: true + BOOTSTRAP_TF_STATE_BUCKET_NAME: + required: true + NEXT_PUBLIC_GTM_ID: + required: true + NEXT_PUBLIC_IS_PROJECT_HEALTH_ENABLED: + required: true + NEXT_PUBLIC_POSTHOG_HOST: + required: true + NEXT_PUBLIC_POSTHOG_KEY: + required: true + SENTRY_AUTH_TOKEN: + required: true + TF_STATE_BUCKET_NAME: + required: true + VITE_API_URL: + required: true + VITE_CSRF_URL: + required: true + VITE_ENVIRONMENT: + required: true + VITE_GRAPHQL_URL: + required: true + VITE_IDX_URL: + required: true + VITE_SENTRY_DSN: + required: true permissions: {} @@ -25,1669 +107,150 @@ env: FORCE_COLOR: 1 jobs: - pre-commit: - name: Run pre-commit checks + run-code-checks: + name: Code checks permissions: contents: read - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false + uses: ./.github/workflows/run-code-checks.yaml - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: '3.13' - - - name: Set up pre-commit cache - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: ~/.cache/pre-commit - key: pre-commit-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }} - restore-keys: | - pre-commit-${{ runner.os }}- - - - name: Install Terraform - uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 - with: - terraform_version: 1.14 - - - name: Setup TFLint - uses: terraform-linters/setup-tflint@b480b8fcdaa6f2c577f8e4fa799e89e756bb7c93 # v6.2.2 - with: - cache: true - github_token: ${{ secrets.GITHUB_TOKEN }} - tflint_version: v0.60.0 - - - name: Run pre-commit - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 - - - name: Check for uncommitted changes - run: | - git diff --exit-code || (echo 'Unstaged changes detected. \ - Run `make check` and use `git add` to address it.' && exit 1) - - check-frontend: - name: Run frontend checks - permissions: - contents: read - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Install pnpm - uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 - with: - run_install: false - version: 11.5.1 - - - name: Set up Node - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: 24 - cache: pnpm - cache-dependency-path: frontend/pnpm-lock.yaml - - - name: Install frontend dependencies - run: pnpm install --frozen-lockfile - working-directory: frontend - - - name: Run pnpm format - working-directory: frontend - run: pnpm run format - - - name: Run pnpm lint check - working-directory: frontend - run: pnpm run lint:check - - - name: Check for uncommitted changes - run: | - git diff --exit-code || (echo 'Unstaged changes detected. \ - Run `make check` and use `git add` to address it.' && exit 1) - - spellcheck: - name: Run spell check - permissions: - contents: read - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Set up Docker buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - - - name: Build cspell image - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 - with: - cache-from: type=gha,scope=cspell - cache-to: type=gha,scope=cspell,mode=max - context: cspell - file: docker/cspell/Dockerfile - load: true - tags: cspell:ci - - - name: Run cspell - run: | - make check-spelling - - run-security-scan-code: - name: Run code security scan - permissions: - contents: read - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Run Semgrep security scan - run: make security-scan-code-semgrep - - - name: Cache Trivy DB - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: .trivy-cache - key: trivy-${{ runner.os }}-${{ hashFiles('docker/trivy/Dockerfile') }} - restore-keys: trivy-${{ runner.os }}- - - - name: Run Trivy security scan - run: make security-scan-code-trivy SCANNERS=misconfig,secret,vuln - timeout-minutes: 15 - - audit-dependencies: - name: Audit dependencies - permissions: - contents: read - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Install pnpm - uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 - with: - run_install: false - version: 11.5.1 - - - name: Set up Node - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: 24 - - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: '3.13' - - - name: Install build dependencies - run: python -m pip install -r backend/requirements.build.txt - - - name: Audit dependencies - run: make audit-dependencies - timeout-minutes: 5 - - run-backend-tests: - name: Run backend tests + run-code-tests: + name: Code tests needs: - - audit-dependencies - - check-frontend - - pre-commit - - run-security-scan-code - - spellcheck + - run-code-checks permissions: + actions: write contents: read id-token: write - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Set up Docker buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - - - name: Build backend test image - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 - with: - cache-from: | - type=gha - type=registry,ref=owasp/nest:test-backend-cache - cache-to: | - type=gha,compression=zstd - context: backend - file: docker/backend/Dockerfile.unit-tests - load: true - platforms: linux/amd64 - tags: owasp/nest:test-backend-latest - - - name: Run backend tests - run: | - CONTAINER_ID=$(docker create \ - -e DJANGO_SETTINGS_MODULE=settings.test \ - --env-file backend/.env.unit-tests \ - owasp/nest:test-backend-latest pytest) - docker start -a $CONTAINER_ID - EXIT_CODE=$? - docker cp $CONTAINER_ID:/home/owasp/coverage.xml backend/coverage.xml 2>/dev/null || true - docker rm $CONTAINER_ID >/dev/null 2>&1 - exit $EXIT_CODE - - - name: Upload coverage report to Codecov - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 - with: - fail_ci_if_error: false - files: backend/coverage.xml - flags: backend - use_oidc: true - timeout-minutes: 5 - - run-frontend-unit-tests: - name: Run frontend unit tests - needs: - - audit-dependencies - - check-frontend - - pre-commit - - run-security-scan-code - - spellcheck - permissions: - contents: read - id-token: write - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Set up Docker buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - - - name: Build frontend unit-testing image - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 - with: - cache-from: | - type=gha - type=registry,ref=owasp/nest:test-frontend-unit-cache - cache-to: | - type=gha,compression=zstd - context: frontend - file: docker/frontend/Dockerfile.unit-tests - load: true - platforms: linux/amd64 - tags: owasp/nest:test-frontend-unit-latest - - - name: Run frontend unit tests - run: | - CONTAINER_ID=$(docker create \ - --env-file frontend/.env.example \ - owasp/nest:test-frontend-unit-latest pnpm run test:unit) - docker start -a $CONTAINER_ID - EXIT_CODE=$? - docker cp $CONTAINER_ID:/app/coverage frontend/coverage 2>/dev/null || true - docker rm $CONTAINER_ID >/dev/null 2>&1 - exit $EXIT_CODE - - - name: Upload coverage report to Codecov - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 - with: - fail_ci_if_error: false - files: frontend/coverage/lcov.info - flags: frontend - use_oidc: true - timeout-minutes: 10 - - run-frontend-e2e-tests: - name: Run frontend e2e tests - needs: - - audit-dependencies - - check-frontend - - pre-commit - - run-security-scan-code - - spellcheck - permissions: - contents: read - runs-on: ubuntu-latest - services: - db: - image: pgvector/pgvector:pg16@sha256:00ba258a66dac104fd5171074a0084462a64a1369d8513f3d0a634e2f24d15bc - env: - POSTGRES_DB: nest_db_e2e - POSTGRES_PASSWORD: nest_user_e2e_password - POSTGRES_USER: nest_user_e2e - options: >- - --health-cmd="pg_isready -U nest_user_e2e -d nest_db_e2e -h localhost -p 5432" - --health-interval=5s - --health-timeout=5s - --health-retries=5 - ports: - - 5432:5432 - cache: - image: redis:8.8.0-alpine3.23@sha256:09160599abd229764c0fb44cb6be640294e1d360a54b19985ab4843dcf2d90f1 - options: >- - --health-cmd="redis-cli ping" - --health-interval=5s - --health-retries=5 - --health-timeout=5s - ports: - - 6379:6379 - steps: - - name: Check out repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Set up Docker buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - - - name: Setup Backend environment - uses: ./.github/workflows/setup-backend-environment - with: - db_username: nest_user_e2e - db_name: nest_db_e2e - - - name: Start Backend in the background - run: | - docker run -d --rm --name e2e-nest-backend \ - --env-file backend/.env.e2e-tests \ - --network host \ - -e DJANGO_DB_HOST=localhost \ - -e DJANGO_REDIS_AUTH_ENABLED=False \ - -e DJANGO_REDIS_HOST=localhost \ - -p 9000:9000 \ - owasp/nest:test-backend-latest \ - sh -c ' - python manage.py migrate && - gunicorn wsgi:application --bind 0.0.0.0:9000 - ' + uses: ./.github/workflows/run-code-tests.yaml - - name: Waiting for the backend to be ready - run: | - timeout 5m bash -c ' - until wget --spider http://localhost:9000/a; do - echo "Waiting for the backend..." - sleep 5 - done - ' - echo "Backend is up!" - - - name: Load Postgres data - env: - PGPASSWORD: nest_user_e2e_password - run: | - pg_restore -h localhost -U nest_user_e2e -d nest_db_e2e < backend/data/nest.dump - - - name: Build frontend image - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 - with: - build-args: | - ENV_FILE=.env.e2e - NEXT_PUBLIC_E2E_BACKEND_BASE_URL=http://localhost:9000 - cache-from: | - type=gha - type=registry,ref=owasp/nest:frontend-e2e-cache - cache-to: | - type=gha,compression=zstd - context: frontend - file: docker/frontend/Dockerfile - load: true - platforms: linux/amd64 - tags: owasp/nest:frontend-e2e-latest - - - name: Run frontend in the background - run: | - docker run \ - --env-file frontend/.env.e2e \ - --name e2e-nest-frontend \ - --network host \ - -d \ - -e NEXT_SERVER_CSRF_URL=http://localhost:9000/csrf/ \ - -e NEXT_SERVER_GRAPHQL_URL=http://localhost:9000/graphql/ \ - -p 3000:3000 \ - owasp/nest:frontend-e2e-latest - - - name: Wait for frontend to be ready - run: | - timeout 5m bash -c ' - until wget --spider http://localhost:3000/; do - echo "Waiting for the frontend..." - sleep 5 - done - ' - echo "Frontend is up!" - - - name: Build e2e testing image - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 - with: - cache-from: | - type=gha - type=registry,ref=owasp/nest:frontend-e2e-tests-cache - cache-to: | - type=gha,compression=zstd - context: . - file: docker/e2e/Dockerfile - load: true - platforms: linux/amd64 - tags: owasp/nest:test-frontend-e2e-latest - - - name: Run frontend end-to-end tests - run: | - docker run \ - --network host \ - -e FRONTEND_URL=http://localhost:3000 \ - owasp/nest:test-frontend-e2e-latest pnpm run test:e2e - timeout-minutes: 20 - - run-frontend-a11y-tests: - name: Run frontend accessibility tests - needs: - - audit-dependencies - - check-frontend - - pre-commit - - run-security-scan-code - - spellcheck - permissions: - contents: read - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Set up Docker buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - - - name: Build frontend a11y-testing image - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 - with: - cache-from: | - type=gha - type=registry,ref=owasp/nest:test-frontend-a11y-cache - cache-to: | - type=gha,compression=zstd - context: frontend - file: docker/frontend/Dockerfile.a11y-tests - load: true - platforms: linux/amd64 - tags: owasp/nest:test-frontend-a11y-latest - - - name: Run frontend a11y tests - run: | - docker run --env-file frontend/.env.example owasp/nest:test-frontend-a11y-latest pnpm run test:a11y - timeout-minutes: 5 - - run-graphql-fuzz-tests: - name: Run GraphQL fuzz tests - if: false + run-release-version-resolution: + name: Resolve release version needs: - - audit-dependencies - - check-frontend - - pre-commit - - run-security-scan-code - - spellcheck - uses: ./.github/workflows/run-fuzz-tests.yaml - with: - test-file: graphql_test.py - - run-rest-fuzz-tests: - name: Run REST fuzz tests - needs: - - audit-dependencies - - check-frontend - - pre-commit - - run-security-scan-code - - spellcheck - uses: ./.github/workflows/run-fuzz-tests.yaml - with: - test-file: rest_test.py - rest-url: http://localhost:9500/api/v0 - - run-infrastructure-tests: - name: Run Infrastructure tests - needs: - - audit-dependencies - - check-frontend - - pre-commit - - run-security-scan-code - - spellcheck - permissions: - contents: read - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Install Terraform - uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 - with: - terraform_version: 1.14 - - - name: Run Infrastructure tests - run: make test-infrastructure - - set-release-version: - name: Set release version - if: | - github.repository == 'OWASP/Nest' && - ( - github.event_name == 'schedule' || - github.event_name == 'workflow_dispatch' || - (github.event_name == 'release' && github.event.action == 'published') - ) - needs: - - run-backend-tests - - run-frontend-a11y-tests - - run-frontend-e2e-tests - - run-frontend-unit-tests - - run-infrastructure-tests - - run-rest-fuzz-tests - outputs: - release_version: ${{ steps.set.outputs.release_version }} + - run-code-tests permissions: {} - runs-on: ubuntu-latest - steps: - - name: Set release version - id: set - run: | - if [ -n "${{ github.event.release.tag_name }}" ]; then - echo "release_version=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT - else - echo "release_version=$(date '+%y.%-m.%-d')-${GITHUB_SHA::7}" >> $GITHUB_OUTPUT - fi - timeout-minutes: 5 - - build-staging-images: - name: Build Staging Images - env: - RELEASE_VERSION: ${{ needs.set-release-version.outputs.release_version }} - environment: staging - if: | - github.repository == 'OWASP/Nest' && - (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') - needs: - - set-release-version - permissions: - contents: read - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Set up QEMU - uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 - - - name: Set up Docker buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - - - name: Login to Docker Hub - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-region: ${{ vars.AWS_REGION }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - role-duration-seconds: 3600 - role-external-id: ${{ secrets.AWS_ROLE_EXTERNAL_ID }} - role-session-name: GitHubActions-BuildStagingImages - role-skip-session-tagging: true - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/nest-staging-terraform - - - name: Login to Amazon ECR - uses: aws-actions/amazon-ecr-login@fa648b43de3d4d023bcb3f89ed6940096949c419 # v2.1.5 - - - name: Build backend image - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 - with: - build-args: | - OWASP_GID=1001 - OWASP_UID=1001 - cache-from: | - type=gha - type=registry,ref=owasp/nest:backend-staging-cache - cache-to: | - type=registry,ref=owasp/nest:backend-staging-cache - context: backend - file: docker/backend/Dockerfile - load: true - platforms: linux/amd64 - push: true - tags: | - owasp/nest:backend-staging - ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/nest-staging-backend:${{ env.RELEASE_VERSION }} - - - name: Prepare frontend public environment - env: - NEXT_PUBLIC_API_URL: ${{ secrets.VITE_API_URL }} - NEXT_PUBLIC_CSRF_URL: ${{ secrets.VITE_CSRF_URL }} - NEXT_PUBLIC_ENVIRONMENT: ${{ secrets.VITE_ENVIRONMENT }} - NEXT_PUBLIC_GRAPHQL_URL: ${{ secrets.VITE_GRAPHQL_URL }} - NEXT_PUBLIC_GTM_ID: ${{ secrets.NEXT_PUBLIC_GTM_ID }} - NEXT_PUBLIC_IDX_URL: ${{ secrets.VITE_IDX_URL }} - NEXT_PUBLIC_IS_PROJECT_HEALTH_ENABLED: ${{ secrets.NEXT_PUBLIC_IS_PROJECT_HEALTH_ENABLED }} - NEXT_PUBLIC_POSTHOG_HOST: ${{ secrets.NEXT_PUBLIC_POSTHOG_HOST }} - NEXT_PUBLIC_POSTHOG_KEY: ${{ secrets.NEXT_PUBLIC_POSTHOG_KEY }} - NEXT_PUBLIC_RELEASE_VERSION: ${{ env.RELEASE_VERSION }} - NEXT_PUBLIC_SENTRY_DSN: ${{ secrets.VITE_SENTRY_DSN }} - run: | - umask 377 - cat > frontend/.env <> $GITHUB_OUTPUT - - - name: Build frontend image - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 - with: - cache-from: | - type=gha - type=registry,ref=owasp/nest:frontend-staging-cache - cache-to: | - type=registry,ref=owasp/nest:frontend-staging-cache - context: frontend - file: docker/frontend/Dockerfile - load: true - platforms: linux/amd64 - push: true - secrets: | - RELEASE_VERSION=${{ env.RELEASE_VERSION }} - SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} - tags: | - owasp/nest:frontend-staging - ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/nest-staging-frontend:${{ env.RELEASE_VERSION }} - - - name: Get frontend image size - id: frontend-size - run: | - IMAGE_NAME="owasp/nest:frontend-staging" - RAW_SIZE=$(docker image inspect "$IMAGE_NAME" --format='{{.Size}}') - DISPLAY_SIZE=$(numfmt --to=iec --suffix=B "$RAW_SIZE") - echo "human_readable=$DISPLAY_SIZE" >> $GITHUB_OUTPUT - - - name: Create Docker image size report - run: | - { - echo "## Docker Image Size Report" - echo "" - echo "**Backend:** ${{ steps.backend-size.outputs.human_readable }}" - echo "**Frontend:** ${{ steps.frontend-size.outputs.human_readable }}" - } >> $GITHUB_STEP_SUMMARY - timeout-minutes: 10 - - scan-staging-images: - name: Scan Staging Images - needs: - - build-staging-images - - set-release-version - env: - RELEASE_VERSION: ${{ needs.set-release-version.outputs.release_version }} - permissions: - contents: write - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Cache Trivy DB - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: .trivy-cache - key: trivy-${{ runner.os }}-${{ hashFiles('docker/trivy/Dockerfile') }} - restore-keys: trivy-${{ runner.os }}- - - - name: Run Trivy security scan via Makefile - run: | - make security-scan-backend-image BACKEND_IMAGE_NAME=owasp/nest:backend-staging - make security-scan-frontend-image FRONTEND_IMAGE_NAME=owasp/nest:frontend-staging - - - name: Generate SBOM for backend image - run: | - make sbom-backend-image BACKEND_IMAGE_NAME=owasp/nest:backend-staging - - - name: Generate SBOM for frontend image - run: | - make sbom-frontend-image FRONTEND_IMAGE_NAME=owasp/nest:frontend-staging - - - name: Upload SBOMs - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: staging-sbom-${{ env.RELEASE_VERSION }} - path: | - backend-sbom-${{ env.RELEASE_VERSION }}.cdx.json - frontend-sbom-${{ env.RELEASE_VERSION }}.cdx.json - timeout-minutes: 5 - - bootstrap-nest-staging-infrastructure: - name: Bootstrap Nest Staging Infrastructure - env: - TF_IN_AUTOMATION: true - TF_INPUT: false - environment: staging - if: | - (github.repository == 'OWASP/Nest' && - (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')) - - needs: - - scan-staging-images - permissions: - contents: read - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false + uses: ./.github/workflows/run-release-version-resolution.yaml - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1 - with: - aws-access-key-id: ${{ secrets.BOOTSTRAP_AWS_ACCESS_KEY_ID }} - aws-region: ${{ vars.AWS_REGION }} - aws-secret-access-key: ${{ secrets.BOOTSTRAP_AWS_SECRET_ACCESS_KEY }} - - - name: Install Terraform - uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 - with: - terraform_version: 1.14 - - - name: Prepare terraform backend - env: - AWS_REGION: ${{ vars.AWS_REGION }} - TF_STATE_BUCKET_NAME: ${{ secrets.BOOTSTRAP_TF_STATE_BUCKET_NAME }} - run: | - umask 377 - cat > infrastructure/bootstrap/terraform.tfbackend <<-EOF - bucket="$TF_STATE_BUCKET_NAME" - region="$AWS_REGION" - EOF - - - name: Prepare terraform variables - env: - AWS_REGION: ${{ vars.AWS_REGION }} - PROJECT_NAME: nest - AWS_ROLE_EXTERNAL_ID: ${{ secrets.AWS_ROLE_EXTERNAL_ID }} - run: | - umask 377 - cat > infrastructure/bootstrap/terraform.tfvars <<-EOF - aws_region="$AWS_REGION" - project_name="$PROJECT_NAME" - aws_role_external_id="$AWS_ROLE_EXTERNAL_ID" - EOF - - - name: Terraform Init - working-directory: infrastructure/bootstrap - run: terraform init -backend-config=terraform.tfbackend - - - name: Terraform Validate - working-directory: infrastructure/bootstrap - run: terraform validate - - - name: Terraform Plan - working-directory: infrastructure/bootstrap - run: terraform plan -out=tfplan - - - name: Show plan in summary - working-directory: infrastructure/bootstrap - run: | - echo "## Bootstrap Terraform Plan Output" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - terraform show -no-color tfplan >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - - - name: Terraform Apply - working-directory: infrastructure/bootstrap - run: terraform apply -auto-approve tfplan - timeout-minutes: 10 - - deploy-staging-nest: - name: Plan and Deploy Nest Staging - env: - TF_IN_AUTOMATION: true - TF_INPUT: false - environment: staging - if: | - github.repository == 'OWASP/Nest' && - (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') + run-image-build: + name: Image build needs: - - bootstrap-nest-staging-infrastructure - - set-release-version + - run-release-version-resolution permissions: contents: read - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-region: ${{ vars.AWS_REGION }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - role-duration-seconds: 3600 - role-external-id: ${{ secrets.AWS_ROLE_EXTERNAL_ID }} - role-session-name: GitHubActions-StagingDeploy - role-skip-session-tagging: true - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/nest-staging-terraform - - - name: Install Terraform - uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 - with: - terraform_version: 1.14 - - - name: Prepare terraform backend - env: - AWS_REGION: ${{ vars.AWS_REGION }} - TF_STATE_BUCKET_NAME: ${{ secrets.TF_STATE_BUCKET_NAME }} - TF_STATE_KEY: staging/terraform.tfstate - run: | - umask 377 - cat > infrastructure/live/terraform.tfbackend <<-EOF - bucket="$TF_STATE_BUCKET_NAME" - key="${TF_STATE_KEY}" - region="$AWS_REGION" - EOF - - - name: Prepare terraform variables - env: - AVAILABILITY_ZONES: ${{ vars.AWS_AVAILABILITY_ZONES }} - AWS_REGION: ${{ vars.AWS_REGION }} - BACKEND_USE_FARGATE_SPOT: true - DOMAIN_NAME: ${{ vars.DOMAIN_NAME }} - DJANGO_CONFIGURATION: Staging - DJANGO_SETTINGS_MODULE: settings.staging - ENABLE_CRON_TASKS: false - ENABLE_NAT_GATEWAY: false - ENABLE_RDS_PROXY: false - ENVIRONMENT: staging - FRONTEND_USE_FARGATE_SPOT: true - PROJECT_NAME: nest - RELEASE_VERSION: ${{ needs.set-release-version.outputs.release_version }} - TASKS_USE_FARGATE_SPOT: true - run: | - umask 377 - cat > infrastructure/live/terraform.tfvars <<-EOF - availability_zones=$AVAILABILITY_ZONES - aws_region="$AWS_REGION" - backend_image_tag="$RELEASE_VERSION" - backend_use_fargate_spot=$BACKEND_USE_FARGATE_SPOT - domain_name="$DOMAIN_NAME" - django_configuration="$DJANGO_CONFIGURATION" - django_release_version="$RELEASE_VERSION" - django_settings_module="$DJANGO_SETTINGS_MODULE" - enable_cron_tasks=$ENABLE_CRON_TASKS - enable_nat_gateway=$ENABLE_NAT_GATEWAY - enable_rds_proxy=$ENABLE_RDS_PROXY - environment="$ENVIRONMENT" - frontend_use_fargate_spot=$FRONTEND_USE_FARGATE_SPOT - frontend_image_tag="$RELEASE_VERSION" - project_name="$PROJECT_NAME" - tasks_use_fargate_spot=$TASKS_USE_FARGATE_SPOT - EOF - - - name: Terraform Init - working-directory: infrastructure/live - run: terraform init -backend-config=terraform.tfbackend - - - name: Terraform Validate - working-directory: infrastructure/live - run: terraform validate - - - name: Terraform Plan - working-directory: infrastructure/live - run: terraform plan -out=tfplan - - - name: Show plan in summary - working-directory: infrastructure/live - run: | - echo "## Terraform Plan Output" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - terraform show -no-color tfplan >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - - - name: Terraform Apply - working-directory: infrastructure/live - run: terraform apply -auto-approve tfplan - - - name: Capture terraform outputs - id: tf-outputs - working-directory: infrastructure/live - run: | - echo "backend_cluster_name=$(terraform output -raw backend_cluster_name)" >> $GITHUB_OUTPUT - echo "backend_service_name=$(terraform output -raw backend_service_name)" >> $GITHUB_OUTPUT - echo "frontend_cluster_name=$(terraform output -raw frontend_cluster_name)" >> $GITHUB_OUTPUT - echo "frontend_service_name=$(terraform output -raw frontend_service_name)" >> $GITHUB_OUTPUT - echo "nat_gateway_enabled=$(terraform output -raw nat_gateway_enabled)" >> $GITHUB_OUTPUT - echo "tasks_cluster_name=$(terraform output -raw tasks_cluster_name)" >> $GITHUB_OUTPUT - echo "tasks_security_group_id=$(terraform output -raw tasks_security_group_id)" >> $GITHUB_OUTPUT - echo "tasks_subnet_ids=$(terraform output -json tasks_subnet_ids | jq -r 'join(",")')" >> $GITHUB_OUTPUT - - - name: Run ECS migrate task - id: migrate-task - env: - CLUSTER_NAME: ${{ steps.tf-outputs.outputs.tasks_cluster_name }} - NAT_GATEWAY_ENABLED: ${{ steps.tf-outputs.outputs.nat_gateway_enabled }} - TASKS_SECURITY_GROUP_ID: ${{ steps.tf-outputs.outputs.tasks_security_group_id }} - TASK_DEFINITION: nest-staging-migrate - TASKS_SUBNET_IDS: ${{ steps.tf-outputs.outputs.tasks_subnet_ids }} - run: | - ASSIGN_PUBLIC_IP=$([ "$NAT_GATEWAY_ENABLED" = "true" ] && echo "DISABLED" || echo "ENABLED") - TASK_ARN=$(aws ecs run-task \ - --cluster "$CLUSTER_NAME" \ - --task-definition "$TASK_DEFINITION" \ - --launch-type FARGATE \ - --network-configuration "awsvpcConfiguration={subnets=[$TASKS_SUBNET_IDS],securityGroups=[$TASKS_SECURITY_GROUP_ID],assignPublicIp=$ASSIGN_PUBLIC_IP}" \ - --query 'tasks[0].taskArn' \ - --output text) - echo "task_arn=$TASK_ARN" >> $GITHUB_OUTPUT - - - name: Wait for migrate task - env: - CLUSTER_NAME: ${{ steps.tf-outputs.outputs.tasks_cluster_name }} - run: | - aws ecs wait tasks-stopped \ - --cluster "$CLUSTER_NAME" \ - --tasks "${{ steps.migrate-task.outputs.task_arn }}" - - - name: Check migrate task exit code - env: - CLUSTER_NAME: ${{ steps.tf-outputs.outputs.tasks_cluster_name }} - run: | - EXIT_CODE=$(aws ecs describe-tasks \ - --cluster "$CLUSTER_NAME" \ - --tasks "${{ steps.migrate-task.outputs.task_arn }}" \ - --query 'tasks[0].containers[0].exitCode' \ - --output text) - - if [ "$EXIT_CODE" != "0" ]; then - echo "::error::Migrate task failed with exit code $EXIT_CODE" - exit 1 - fi - echo "Migrate task completed successfully" - - - name: Run ECS index-data task - env: - CLUSTER_NAME: ${{ steps.tf-outputs.outputs.tasks_cluster_name }} - NAT_GATEWAY_ENABLED: ${{ steps.tf-outputs.outputs.nat_gateway_enabled }} - TASK_DEFINITION: nest-staging-index-data - TASKS_SECURITY_GROUP_ID: ${{ steps.tf-outputs.outputs.tasks_security_group_id }} - TASKS_SUBNET_IDS: ${{ steps.tf-outputs.outputs.tasks_subnet_ids }} - run: | - ASSIGN_PUBLIC_IP=$([ "$NAT_GATEWAY_ENABLED" = "true" ] && echo "DISABLED" || echo "ENABLED") - RESPONSE=$(aws ecs run-task \ - --cluster "$CLUSTER_NAME" \ - --task-definition "$TASK_DEFINITION" \ - --launch-type FARGATE \ - --network-configuration "awsvpcConfiguration={subnets=[$TASKS_SUBNET_IDS],securityGroups=[$TASKS_SECURITY_GROUP_ID],assignPublicIp=$ASSIGN_PUBLIC_IP}") - - FAILURE_COUNT=$(echo "$RESPONSE" | jq '.failures | length') - TASK_COUNT=$(echo "$RESPONSE" | jq '.tasks | length') - - if [ "$FAILURE_COUNT" -gt 0 ]; then - echo "Error: ECS run-task for $TASK_DEFINITION on cluster $CLUSTER_NAME returned failures:" - echo "$RESPONSE" | jq '.failures' - exit 1 - fi - - if [ "$TASK_COUNT" -eq 0 ]; then - echo "Error: ECS run-task for $TASK_DEFINITION on cluster $CLUSTER_NAME returned no tasks" - echo "$RESPONSE" - exit 1 - fi - - echo "Index-data task started successfully (runs async, ~30 min)" - - - name: Wait for backend service stability - env: - CLUSTER_NAME: ${{ steps.tf-outputs.outputs.backend_cluster_name }} - SERVICE_NAME: ${{ steps.tf-outputs.outputs.backend_service_name }} - run: | - aws ecs wait services-stable \ - --cluster "$CLUSTER_NAME" \ - --services "$SERVICE_NAME" - - - name: Wait for frontend service stability - env: - CLUSTER_NAME: ${{ steps.tf-outputs.outputs.frontend_cluster_name }} - SERVICE_NAME: ${{ steps.tf-outputs.outputs.frontend_service_name }} - run: | - aws ecs wait services-stable \ - --cluster "$CLUSTER_NAME" \ - --services "$SERVICE_NAME" - timeout-minutes: 60 - - run-staging-lighthouse-ci: - name: Run staging Lighthouse CI - needs: - - deploy-staging-nest - permissions: - contents: read - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Install pnpm - uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 - with: - run_install: false - version: 11.5.1 - - - name: Set up Node - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - cache-dependency-path: frontend/pnpm-lock.yaml - cache: pnpm - node-version: 24 - - - name: Install frontend dependencies - run: pnpm install --frozen-lockfile - working-directory: frontend - - - name: Run lighthouse-ci - env: - LHCI_BASE_URL: https://nest.owasp.dev - run: | - pnpm run lighthouse-ci - working-directory: frontend - timeout-minutes: 5 - - run-staging-zap-baseline-scan: - name: Run staging ZAP baseline scan - needs: - - deploy-staging-nest - permissions: - contents: read - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Run baseline scan - uses: zaproxy/action-baseline@de8ad967d3548d44ef623df22cf95c3b0baf8b25 # v0.15.0 - with: - token: ${{ secrets.GITHUB_TOKEN }} - target: https://nest.owasp.dev - allow_issue_writing: false - fail_action: true - cmd_options: -a -c .zapconfig -r report_html.html - - - name: Upload report - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: zap-baseline-scan-report-${{ github.run_id }} - path: report_html.html - timeout-minutes: 5 - - build-production-images: - name: Build Production Images - env: - RELEASE_VERSION: ${{ needs.set-release-version.outputs.release_version }} - environment: production - if: | - github.repository == 'OWASP/Nest' && - github.event_name == 'release' && - github.event.action == 'published' - needs: - - set-release-version - permissions: - contents: read - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Set up QEMU - uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 - - - name: Set up Docker buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - - - name: Login to Docker Hub - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-region: ${{ vars.AWS_REGION }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - role-duration-seconds: 3600 - role-external-id: ${{ secrets.AWS_ROLE_EXTERNAL_ID }} - role-session-name: GitHubActions-BuildProductionImages - role-skip-session-tagging: true - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/nest-production-terraform - - - name: Login to Amazon ECR - uses: aws-actions/amazon-ecr-login@fa648b43de3d4d023bcb3f89ed6940096949c419 # v2.1.5 - - - name: Build backend image - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 - with: - build-args: | - OWASP_GID=1002 - OWASP_UID=1002 - cache-from: | - type=gha - type=registry,ref=owasp/nest:backend-staging-cache - context: backend - file: docker/backend/Dockerfile - load: true - platforms: linux/amd64 - push: true - tags: | - owasp/nest:backend-production - ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/nest-production-backend:${{ env.RELEASE_VERSION }} - - - name: Get backend image size - id: backend-size - run: | - IMAGE_NAME="owasp/nest:backend-production" - RAW_SIZE=$(docker image inspect "$IMAGE_NAME" --format='{{.Size}}') - DISPLAY_SIZE=$(numfmt --to=iec --suffix=B "$RAW_SIZE") - echo "human_readable=$DISPLAY_SIZE" >> $GITHUB_OUTPUT - - - name: Prepare frontend public environment - env: - NEXT_PUBLIC_API_URL: ${{ secrets.VITE_API_URL }} - NEXT_PUBLIC_CSRF_URL: ${{ secrets.VITE_CSRF_URL }} - NEXT_PUBLIC_ENVIRONMENT: ${{ secrets.VITE_ENVIRONMENT }} - NEXT_PUBLIC_GRAPHQL_URL: ${{ secrets.VITE_GRAPHQL_URL }} - NEXT_PUBLIC_GTM_ID: ${{ secrets.NEXT_PUBLIC_GTM_ID }} - NEXT_PUBLIC_IDX_URL: ${{ secrets.VITE_IDX_URL }} - NEXT_PUBLIC_IS_PROJECT_HEALTH_ENABLED: ${{ secrets.NEXT_PUBLIC_IS_PROJECT_HEALTH_ENABLED }} - NEXT_PUBLIC_POSTHOG_HOST: ${{ secrets.NEXT_PUBLIC_POSTHOG_HOST }} - NEXT_PUBLIC_POSTHOG_KEY: ${{ secrets.NEXT_PUBLIC_POSTHOG_KEY }} - NEXT_PUBLIC_RELEASE_VERSION: ${{ env.RELEASE_VERSION }} - NEXT_PUBLIC_SENTRY_DSN: ${{ secrets.VITE_SENTRY_DSN }} - run: | - umask 377 - cat > frontend/.env <> $GITHUB_OUTPUT - - - name: Create Docker image size report - run: | - { - echo "## Docker Image Size Report" - echo "" - echo "**Backend:** ${{ steps.backend-size.outputs.human_readable }}" - echo "**Frontend:** ${{ steps.frontend-size.outputs.human_readable }}" - } >> $GITHUB_STEP_SUMMARY - timeout-minutes: 5 - - scan-production-images: - name: Scan Production Images - if: | - github.repository == 'OWASP/Nest' && - github.event_name == 'release' && - github.event.action == 'published' + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + AWS_ROLE_EXTERNAL_ID: ${{ secrets.AWS_ROLE_EXTERNAL_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + NEXT_PUBLIC_GTM_ID: ${{ secrets.NEXT_PUBLIC_GTM_ID }} + NEXT_PUBLIC_IS_PROJECT_HEALTH_ENABLED: ${{ secrets.NEXT_PUBLIC_IS_PROJECT_HEALTH_ENABLED }} + NEXT_PUBLIC_POSTHOG_HOST: ${{ secrets.NEXT_PUBLIC_POSTHOG_HOST }} + NEXT_PUBLIC_POSTHOG_KEY: ${{ secrets.NEXT_PUBLIC_POSTHOG_KEY }} + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + VITE_API_URL: ${{ secrets.VITE_API_URL }} + VITE_CSRF_URL: ${{ secrets.VITE_CSRF_URL }} + VITE_ENVIRONMENT: ${{ secrets.VITE_ENVIRONMENT }} + VITE_GRAPHQL_URL: ${{ secrets.VITE_GRAPHQL_URL }} + VITE_IDX_URL: ${{ secrets.VITE_IDX_URL }} + VITE_SENTRY_DSN: ${{ secrets.VITE_SENTRY_DSN }} + uses: ./.github/workflows/run-image-build.yaml + with: + aws_role_name: nest-${{ inputs.environment }}-terraform + aws_role_session_name: github-actions-${{ inputs.environment }}-image-build + backend_ecr_cache_repo: nest-${{ inputs.environment }}-backend-cache + backend_ecr_repo: nest-${{ inputs.environment }}-backend + backend_uid: ${{ inputs.backend_uid }} + backend_gid: ${{ inputs.backend_gid }} + environment: ${{ inputs.environment }} + frontend_ecr_cache_repo: nest-${{ inputs.environment }}-frontend-cache + frontend_ecr_repo: nest-${{ inputs.environment }}-frontend + release_version: ${{ needs.run-release-version-resolution.outputs.release_version }} + + run-image-scan: + name: Image scan needs: - - build-production-images - - set-release-version - env: - RELEASE_VERSION: ${{ needs.set-release-version.outputs.release_version }} + - run-image-build + - run-release-version-resolution permissions: + actions: write contents: write - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Cache Trivy DB - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: .trivy-cache - key: trivy-${{ runner.os }}-${{ hashFiles('docker/trivy/Dockerfile') }} - restore-keys: trivy-${{ runner.os }}- - - - name: Run Trivy security scan via Makefile - run: | - make security-scan-backend-image BACKEND_IMAGE_NAME=owasp/nest:backend-production - make security-scan-frontend-image FRONTEND_IMAGE_NAME=owasp/nest:frontend-production - - - name: Generate SBOM for backend image - run: | - make sbom-backend-image BACKEND_IMAGE_NAME=owasp/nest:backend-production - - - name: Generate SBOM for frontend image - run: | - make sbom-frontend-image FRONTEND_IMAGE_NAME=owasp/nest:frontend-production - - - name: Upload SBOMs - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh release upload "${{ github.event.release.tag_name }}" \ - backend-sbom-${{ env.RELEASE_VERSION }}.cdx.json \ - frontend-sbom-${{ env.RELEASE_VERSION }}.cdx.json - timeout-minutes: 5 - - bootstrap-nest-production-infrastructure: - name: Bootstrap Nest Production Infrastructure - env: - TF_IN_AUTOMATION: true - TF_INPUT: false - environment: production - if: | - github.repository == 'OWASP/Nest' && - github.event_name == 'release' && - github.event.action == 'published' - + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + AWS_ROLE_EXTERNAL_ID: ${{ secrets.AWS_ROLE_EXTERNAL_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + uses: ./.github/workflows/run-image-scan.yaml + with: + aws_role_name: nest-${{ inputs.environment }}-terraform + aws_role_session_name: github-actions-${{ inputs.environment }}-image-scan + backend_ecr_repo: nest-${{ inputs.environment }}-backend + environment: ${{ inputs.environment }} + frontend_ecr_repo: nest-${{ inputs.environment }}-frontend + release_tag: ${{ inputs.release_tag }} + release_version: ${{ needs.run-release-version-resolution.outputs.release_version }} + + run-infrastructure-bootstrap: + name: Infrastructure bootstrap needs: - - scan-production-images + - run-image-scan permissions: contents: read - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1 - with: - aws-access-key-id: ${{ secrets.BOOTSTRAP_AWS_ACCESS_KEY_ID }} - aws-region: ${{ vars.AWS_REGION }} - aws-secret-access-key: ${{ secrets.BOOTSTRAP_AWS_SECRET_ACCESS_KEY }} - - - name: Install Terraform - uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 - with: - terraform_version: 1.14 - - - name: Prepare terraform backend - env: - AWS_REGION: ${{ vars.AWS_REGION }} - TF_STATE_BUCKET_NAME: ${{ secrets.BOOTSTRAP_TF_STATE_BUCKET_NAME }} - run: | - umask 377 - cat > infrastructure/bootstrap/terraform.tfbackend <<-EOF - bucket="$TF_STATE_BUCKET_NAME" - region="$AWS_REGION" - EOF - - - name: Prepare terraform variables - env: - AWS_REGION: ${{ vars.AWS_REGION }} - PROJECT_NAME: nest - AWS_ROLE_EXTERNAL_ID: ${{ secrets.AWS_ROLE_EXTERNAL_ID }} - run: | - umask 377 - cat > infrastructure/bootstrap/terraform.tfvars <<-EOF - aws_region="$AWS_REGION" - project_name="$PROJECT_NAME" - aws_role_external_id="$AWS_ROLE_EXTERNAL_ID" - EOF - - - name: Terraform Init - working-directory: infrastructure/bootstrap - run: terraform init -backend-config=terraform.tfbackend - - - name: Terraform Validate - working-directory: infrastructure/bootstrap - run: terraform validate - - - name: Terraform Plan - working-directory: infrastructure/bootstrap - run: terraform plan -out=tfplan - - - name: Show plan in summary - working-directory: infrastructure/bootstrap - run: | - echo "## Bootstrap Terraform Plan Output" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - terraform show -no-color tfplan >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - - - name: Terraform Apply - working-directory: infrastructure/bootstrap - run: terraform apply -auto-approve tfplan - timeout-minutes: 10 + secrets: + AWS_ROLE_EXTERNAL_ID: ${{ secrets.AWS_ROLE_EXTERNAL_ID }} + BOOTSTRAP_AWS_ACCESS_KEY_ID: ${{ secrets.BOOTSTRAP_AWS_ACCESS_KEY_ID }} + BOOTSTRAP_AWS_SECRET_ACCESS_KEY: ${{ secrets.BOOTSTRAP_AWS_SECRET_ACCESS_KEY }} + BOOTSTRAP_TF_STATE_BUCKET_NAME: ${{ secrets.BOOTSTRAP_TF_STATE_BUCKET_NAME }} + uses: ./.github/workflows/run-infrastructure-bootstrap.yaml + with: + environment: ${{ inputs.environment }} - deploy-production-nest: - name: Plan and Deploy Nest Production - env: - TF_IN_AUTOMATION: true - TF_INPUT: false - environment: production - if: | - github.repository == 'OWASP/Nest' && - github.event_name == 'release' && - github.event.action == 'published' + run-deploy: + name: Deploy needs: - - bootstrap-nest-production-infrastructure - - set-release-version + - run-infrastructure-bootstrap + - run-release-version-resolution permissions: contents: read - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-region: ${{ vars.AWS_REGION }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - role-duration-seconds: 3600 - role-external-id: ${{ secrets.AWS_ROLE_EXTERNAL_ID }} - role-session-name: GitHubActions-ProductionDeploy - role-skip-session-tagging: true - role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/nest-production-terraform - - - name: Install Terraform - uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 - with: - terraform_version: 1.14 - - - name: Prepare terraform backend - env: - AWS_REGION: ${{ vars.AWS_REGION }} - TF_STATE_BUCKET_NAME: ${{ secrets.TF_STATE_BUCKET_NAME }} - TF_STATE_KEY: production/terraform.tfstate - run: | - umask 377 - cat > infrastructure/live/terraform.tfbackend <<-EOF - bucket="$TF_STATE_BUCKET_NAME" - key="${TF_STATE_KEY}" - region="$AWS_REGION" - EOF - - - name: Prepare terraform variables - env: - AVAILABILITY_ZONES: ${{ vars.AWS_AVAILABILITY_ZONES }} - AWS_REGION: ${{ vars.AWS_REGION }} - BACKEND_USE_FARGATE_SPOT: false - DOMAIN_NAME: ${{ vars.DOMAIN_NAME }} - DJANGO_CONFIGURATION: Production - DJANGO_SETTINGS_MODULE: settings.production - ENABLE_ADDITIONAL_PARAMETERS: true - ENABLE_CRON_TASKS: true - ENABLE_NAT_GATEWAY: true - ENABLE_RDS_PROXY: true - ENVIRONMENT: production - FRONTEND_USE_FARGATE_SPOT: false - PROJECT_NAME: nest - RELEASE_VERSION: ${{ needs.set-release-version.outputs.release_version }} - TASKS_USE_FARGATE_SPOT: false - run: | - umask 377 - cat > infrastructure/live/terraform.tfvars <<-EOF - availability_zones=$AVAILABILITY_ZONES - aws_region="$AWS_REGION" - backend_image_tag="$RELEASE_VERSION" - backend_use_fargate_spot=$BACKEND_USE_FARGATE_SPOT - domain_name="$DOMAIN_NAME" - django_configuration="$DJANGO_CONFIGURATION" - django_release_version="$RELEASE_VERSION" - django_settings_module="$DJANGO_SETTINGS_MODULE" - enable_additional_parameters=$ENABLE_ADDITIONAL_PARAMETERS - enable_cron_tasks=$ENABLE_CRON_TASKS - enable_nat_gateway=$ENABLE_NAT_GATEWAY - enable_rds_proxy=$ENABLE_RDS_PROXY - environment="$ENVIRONMENT" - frontend_use_fargate_spot=$FRONTEND_USE_FARGATE_SPOT - frontend_image_tag="$RELEASE_VERSION" - project_name="$PROJECT_NAME" - tasks_use_fargate_spot=$TASKS_USE_FARGATE_SPOT - EOF - - - name: Terraform Init - working-directory: infrastructure/live - run: terraform init -backend-config=terraform.tfbackend - - - name: Terraform Validate - working-directory: infrastructure/live - run: terraform validate - - - name: Terraform Plan - working-directory: infrastructure/live - run: terraform plan -out=tfplan - - - name: Show plan in summary - working-directory: infrastructure/live - run: | - echo "## Terraform Plan Output" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - terraform show -no-color tfplan >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - - - name: Terraform Apply - working-directory: infrastructure/live - run: terraform apply -auto-approve tfplan - - - name: Capture terraform outputs - id: tf-outputs - working-directory: infrastructure/live - run: | - echo "backend_cluster_name=$(terraform output -raw backend_cluster_name)" >> $GITHUB_OUTPUT - echo "backend_service_name=$(terraform output -raw backend_service_name)" >> $GITHUB_OUTPUT - echo "frontend_cluster_name=$(terraform output -raw frontend_cluster_name)" >> $GITHUB_OUTPUT - echo "frontend_service_name=$(terraform output -raw frontend_service_name)" >> $GITHUB_OUTPUT - echo "nat_gateway_enabled=$(terraform output -raw nat_gateway_enabled)" >> $GITHUB_OUTPUT - echo "tasks_cluster_name=$(terraform output -raw tasks_cluster_name)" >> $GITHUB_OUTPUT - echo "tasks_security_group_id=$(terraform output -raw tasks_security_group_id)" >> $GITHUB_OUTPUT - echo "tasks_subnet_ids=$(terraform output -json tasks_subnet_ids | jq -r 'join(",")')" >> $GITHUB_OUTPUT - - - name: Run ECS migrate task - id: migrate-task - env: - CLUSTER_NAME: ${{ steps.tf-outputs.outputs.tasks_cluster_name }} - NAT_GATEWAY_ENABLED: ${{ steps.tf-outputs.outputs.nat_gateway_enabled }} - TASKS_SECURITY_GROUP_ID: ${{ steps.tf-outputs.outputs.tasks_security_group_id }} - TASK_DEFINITION: nest-production-migrate - TASKS_SUBNET_IDS: ${{ steps.tf-outputs.outputs.tasks_subnet_ids }} - run: | - ASSIGN_PUBLIC_IP=$([ "$NAT_GATEWAY_ENABLED" = "true" ] && echo "DISABLED" || echo "ENABLED") - TASK_ARN=$(aws ecs run-task \ - --cluster "$CLUSTER_NAME" \ - --task-definition "$TASK_DEFINITION" \ - --launch-type FARGATE \ - --network-configuration "awsvpcConfiguration={subnets=[$TASKS_SUBNET_IDS],securityGroups=[$TASKS_SECURITY_GROUP_ID],assignPublicIp=$ASSIGN_PUBLIC_IP}" \ - --query 'tasks[0].taskArn' \ - --output text) - echo "task_arn=$TASK_ARN" >> $GITHUB_OUTPUT - - - name: Wait for migrate task - env: - CLUSTER_NAME: ${{ steps.tf-outputs.outputs.tasks_cluster_name }} - run: | - aws ecs wait tasks-stopped \ - --cluster "$CLUSTER_NAME" \ - --tasks "${{ steps.migrate-task.outputs.task_arn }}" - - - name: Check migrate task exit code - env: - CLUSTER_NAME: ${{ steps.tf-outputs.outputs.tasks_cluster_name }} - run: | - EXIT_CODE=$(aws ecs describe-tasks \ - --cluster "$CLUSTER_NAME" \ - --tasks "${{ steps.migrate-task.outputs.task_arn }}" \ - --query 'tasks[0].containers[0].exitCode' \ - --output text) - - if [ "$EXIT_CODE" != "0" ]; then - echo "::error::Migrate task failed with exit code $EXIT_CODE" - exit 1 - fi - echo "Migrate task completed successfully" - - - name: Run ECS index-data task - env: - CLUSTER_NAME: ${{ steps.tf-outputs.outputs.tasks_cluster_name }} - NAT_GATEWAY_ENABLED: ${{ steps.tf-outputs.outputs.nat_gateway_enabled }} - TASK_DEFINITION: nest-production-index-data - TASKS_SECURITY_GROUP_ID: ${{ steps.tf-outputs.outputs.tasks_security_group_id }} - TASKS_SUBNET_IDS: ${{ steps.tf-outputs.outputs.tasks_subnet_ids }} - run: | - ASSIGN_PUBLIC_IP=$([ "$NAT_GATEWAY_ENABLED" = "true" ] && echo "DISABLED" || echo "ENABLED") - RESPONSE=$(aws ecs run-task \ - --cluster "$CLUSTER_NAME" \ - --task-definition "$TASK_DEFINITION" \ - --launch-type FARGATE \ - --network-configuration "awsvpcConfiguration={subnets=[$TASKS_SUBNET_IDS],securityGroups=[$TASKS_SECURITY_GROUP_ID],assignPublicIp=$ASSIGN_PUBLIC_IP}") - - FAILURE_COUNT=$(echo "$RESPONSE" | jq '.failures | length') - TASK_COUNT=$(echo "$RESPONSE" | jq '.tasks | length') - - if [ "$FAILURE_COUNT" -gt 0 ]; then - echo "Error: ECS run-task for $TASK_DEFINITION on cluster $CLUSTER_NAME returned failures:" - echo "$RESPONSE" | jq '.failures' - exit 1 - fi - - if [ "$TASK_COUNT" -eq 0 ]; then - echo "Error: ECS run-task for $TASK_DEFINITION on cluster $CLUSTER_NAME returned no tasks" - echo "$RESPONSE" - exit 1 - fi - - echo "Index-data task started successfully (runs async, ~30 min)" - - - name: Wait for backend service stability - env: - CLUSTER_NAME: ${{ steps.tf-outputs.outputs.backend_cluster_name }} - SERVICE_NAME: ${{ steps.tf-outputs.outputs.backend_service_name }} - run: | - aws ecs wait services-stable \ - --cluster "$CLUSTER_NAME" \ - --services "$SERVICE_NAME" - - - name: Wait for frontend service stability - env: - CLUSTER_NAME: ${{ steps.tf-outputs.outputs.frontend_cluster_name }} - SERVICE_NAME: ${{ steps.tf-outputs.outputs.frontend_service_name }} - run: | - aws ecs wait services-stable \ - --cluster "$CLUSTER_NAME" \ - --services "$SERVICE_NAME" - - run-production-lighthouse-ci: - name: Run production Lighthouse CI - if: | - github.repository == 'OWASP/Nest' && - github.event_name == 'release' && - github.event.action == 'published' + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} + AWS_ROLE_EXTERNAL_ID: ${{ secrets.AWS_ROLE_EXTERNAL_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + TF_STATE_BUCKET_NAME: ${{ secrets.TF_STATE_BUCKET_NAME }} + uses: ./.github/workflows/run-deploy.yaml + with: + aws_role_name: nest-${{ inputs.environment }}-terraform + aws_role_session_name: github-actions-${{ inputs.environment }}-deploy + backend_use_fargate_spot: ${{ inputs.backend_use_fargate_spot }} + django_configuration: ${{ inputs.django_configuration }} + django_settings_module: ${{ inputs.django_settings_module }} + enable_additional_parameters: ${{ inputs.enable_additional_parameters }} + enable_cron_tasks: ${{ inputs.enable_cron_tasks }} + enable_nat_gateway: ${{ inputs.enable_nat_gateway }} + enable_rds_proxy: ${{ inputs.enable_rds_proxy }} + environment: ${{ inputs.environment }} + frontend_use_fargate_spot: ${{ inputs.frontend_use_fargate_spot }} + index_task_definition: nest-${{ inputs.environment }}-index-data + migrate_task_definition: nest-${{ inputs.environment }}-migrate + release_version: ${{ needs.run-release-version-resolution.outputs.release_version }} + tasks_use_fargate_spot: ${{ inputs.tasks_use_fargate_spot }} + tf_state_key: ${{ inputs.environment }}/terraform.tfstate + + run-lighthouse-ci: + name: Lighthouse CI needs: - - deploy-production-nest + - run-deploy permissions: contents: read - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Install pnpm - uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 - with: - run_install: false - version: 11.5.1 - - - name: Set up Node - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - cache-dependency-path: frontend/pnpm-lock.yaml - cache: pnpm - node-version: 24 - - - name: Install frontend dependencies - run: pnpm install --frozen-lockfile - working-directory: frontend - - - name: Run lighthouse-ci - env: - LHCI_BASE_URL: https://nest.owasp.org - run: | - pnpm run lighthouse-ci - working-directory: frontend - timeout-minutes: 5 + uses: ./.github/workflows/run-lighthouse-ci.yaml + with: + base_url: ${{ inputs.base_url }} - run-production-zap-baseline-scan: - name: Run production ZAP baseline scan - if: | - github.repository == 'OWASP/Nest' && - github.event_name == 'release' && - github.event.action == 'published' + run-zap-baseline-scan: + name: ZAP baseline scan needs: - - deploy-production-nest + - run-deploy permissions: contents: read - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Run baseline scan - uses: zaproxy/action-baseline@de8ad967d3548d44ef623df22cf95c3b0baf8b25 # v0.15.0 - with: - token: ${{ secrets.GITHUB_TOKEN }} - target: https://nest.owasp.org - allow_issue_writing: false - fail_action: true - cmd_options: -a -c .zapconfig -r report_html.html - - - name: Upload report - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: zap-baseline-scan-report-${{ github.run_id }} - path: report_html.html - timeout-minutes: 5 + uses: ./.github/workflows/run-zap-baseline-scan.yaml + with: + base_url: ${{ inputs.base_url }} diff --git a/.github/workflows/run-code-checks.yaml b/.github/workflows/run-code-checks.yaml new file mode 100644 index 0000000000..2f08b701d1 --- /dev/null +++ b/.github/workflows/run-code-checks.yaml @@ -0,0 +1,168 @@ +name: Code checks + +on: + workflow_call: + +env: + FORCE_COLOR: 1 + +permissions: {} + +jobs: + run-dependency-audit: + name: Dependency audit + permissions: + contents: read + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install pnpm + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 + with: + run_install: false + version: 11.5.1 + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 24 + + - name: Install build dependencies + run: python -m pip install -r backend/requirements.build.txt + + - name: Audit dependencies + run: make audit-dependencies + timeout-minutes: 5 + + run-frontend-checks: + name: Frontend + permissions: + contents: read + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install frontend dependencies + uses: ./.github/actions/install-frontend-dependencies + + - name: Run pnpm format + run: pnpm run format + working-directory: frontend + + - name: Run pnpm lint check + run: pnpm run lint:check + working-directory: frontend + + - name: Check for uncommitted changes + run: | + git diff --exit-code || (echo 'Unstaged changes detected. \ + Run `make check` and use `git add` to address it.' && exit 1) + + run-pre-commit-checks: + name: Pre-commit + permissions: + contents: read + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.13' + + - name: Install Poetry + uses: ./.github/actions/install-poetry + + - name: Set up pre-commit cache + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + key: pre-commit-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }} + path: ~/.cache/pre-commit + restore-keys: | + pre-commit-${{ runner.os }}- + + - name: Install Terraform + uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 + with: + terraform_version: 1.14 + + - name: Setup TFLint + uses: terraform-linters/setup-tflint@b480b8fcdaa6f2c577f8e4fa799e89e756bb7c93 # v6.2.2 + with: + cache: true + github_token: ${{ secrets.GITHUB_TOKEN }} + tflint_config_path: infrastructure/.tflint.hcl + tflint_version: v0.60.0 + + - name: Run pre-commit + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 + + - name: Check for uncommitted changes + run: | + git diff --exit-code || (echo 'Unstaged changes detected. \ + Run `make check` and use `git add` to address it.' && exit 1) + + run-security-checks: + name: Security + permissions: + contents: read + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Run Semgrep security scan + run: make security-scan-code-semgrep + + - name: Run trivy scan + uses: ./.github/actions/run-trivy-scan + with: + command: make security-scan-code-trivy SCANNERS=misconfig,secret,vuln + timeout-minutes: 15 + + run-spelling-checks: + name: Spelling checks + permissions: + contents: read + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install pnpm + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 + with: + run_install: false + version: 10.33.3 + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + cache-dependency-path: cspell/pnpm-lock.yaml + cache: pnpm + node-version: 24 + + - name: Install cspell dependencies + run: pnpm install --frozen-lockfile --prefer-offline + working-directory: cspell + + - name: Run cspell + run: pnpm exec cspell --config cspell.json --no-progress -r .. + working-directory: cspell diff --git a/.github/workflows/run-code-tests.yaml b/.github/workflows/run-code-tests.yaml new file mode 100644 index 0000000000..00477992a2 --- /dev/null +++ b/.github/workflows/run-code-tests.yaml @@ -0,0 +1,84 @@ +name: Code tests + +on: + workflow_call: + +permissions: {} + +env: + FORCE_COLOR: 1 + +jobs: + run-backend-tests: + name: Backend tests + permissions: + actions: write + contents: read + uses: ./.github/workflows/run-backend-tests.yaml + + run-backend-coverage-upload: + name: Backend coverage upload + needs: + - run-backend-tests + permissions: + actions: read + contents: read + id-token: write + uses: ./.github/workflows/run-coverage-upload.yaml + with: + artifact_name: coverage-xml + artifact_path: backend + coverage_flags: backend + coverage_path: backend/coverage.xml + + run-frontend-tests: + name: Frontend tests + permissions: + actions: write + contents: read + uses: ./.github/workflows/run-frontend-tests.yaml + + run-frontend-coverage-upload: + name: Frontend coverage upload + needs: + - run-frontend-tests + permissions: + actions: read + contents: read + id-token: write + uses: ./.github/workflows/run-coverage-upload.yaml + with: + artifact_name: coverage-lcov + artifact_path: frontend/coverage + coverage_flags: frontend + coverage_path: frontend/coverage/lcov.info + + run-e2e-tests: + name: E2E tests + permissions: + contents: read + uses: ./.github/workflows/run-e2e-tests.yaml + + run-fuzz-graphql-tests: + name: GraphQL fuzz tests + if: false + permissions: + contents: read + uses: ./.github/workflows/run-fuzz-tests.yaml + with: + test-file: graphql_test.py + + run-fuzz-rest-tests: + name: REST fuzz tests + permissions: + contents: read + uses: ./.github/workflows/run-fuzz-tests.yaml + with: + rest-url: http://localhost:9500/api/v0 + test-file: rest_test.py + + run-infrastructure-tests: + name: Infrastructure tests + permissions: + contents: read + uses: ./.github/workflows/run-infrastructure-tests.yaml diff --git a/.github/workflows/run-coverage-upload.yaml b/.github/workflows/run-coverage-upload.yaml new file mode 100644 index 0000000000..d2dbfe244e --- /dev/null +++ b/.github/workflows/run-coverage-upload.yaml @@ -0,0 +1,57 @@ +name: Coverage upload + +on: + workflow_call: + inputs: + artifact_name: + description: Coverage artifact name + required: true + type: string + artifact_path: + description: Coverage artifact download path + required: true + type: string + coverage_flags: + description: Codecov flags + required: false + type: string + default: '' + coverage_path: + description: Coverage report file path + required: true + type: string + + +permissions: {} + +jobs: + run-coverage-upload: + name: Coverage upload + permissions: + actions: read + contents: read + id-token: write + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Download coverage report + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ${{ inputs.artifact_name }} + path: ${{ inputs.artifact_path }} + + - name: Import Codecov GPG verification key + run: curl -sf https://uploader.codecov.io/verification.gpg | gpg --import + + - name: Upload coverage report to Codecov + uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + with: + fail_ci_if_error: true + files: ${{ inputs.coverage_path }} + flags: ${{ inputs.coverage_flags }} + use_oidc: true + timeout-minutes: 5 diff --git a/.github/workflows/run-deploy.yaml b/.github/workflows/run-deploy.yaml new file mode 100644 index 0000000000..4bfb88ac87 --- /dev/null +++ b/.github/workflows/run-deploy.yaml @@ -0,0 +1,251 @@ +name: Deploy + +on: + workflow_call: + inputs: + aws_role_name: + description: AWS role name to assume + required: true + type: string + aws_role_session_name: + description: AWS role session name + required: true + type: string + backend_use_fargate_spot: + description: Use Fargate spot for backend + required: true + type: boolean + django_configuration: + description: Django configuration string + required: true + type: string + django_settings_module: + description: Django settings module + required: true + type: string + enable_additional_parameters: + description: Enable additional SSM parameters + required: false + default: false + type: boolean + enable_cron_tasks: + description: Enable cron tasks + required: true + type: boolean + enable_nat_gateway: + description: Enable NAT gateway + required: true + type: boolean + enable_rds_proxy: + description: Enable RDS proxy + required: true + type: boolean + environment: + description: GitHub environment name + required: true + type: string + frontend_use_fargate_spot: + description: Use Fargate spot for frontend + required: true + type: boolean + index_task_definition: + description: ECS task definition for index task + required: true + type: string + migrate_task_definition: + description: ECS task definition for migrate task + required: true + type: string + release_version: + description: Release version to deploy + required: true + type: string + tasks_use_fargate_spot: + description: Use Fargate spot for tasks + required: true + type: boolean + tf_state_key: + description: Terraform state key + required: true + type: string + secrets: + AWS_ACCESS_KEY_ID: + required: true + AWS_ACCOUNT_ID: + required: true + AWS_ROLE_EXTERNAL_ID: + required: true + AWS_SECRET_ACCESS_KEY: + required: true + TF_STATE_BUCKET_NAME: + required: true + +env: + FORCE_COLOR: 1 + +permissions: {} + +jobs: + run-deploy: + name: Deploy + environment: ${{ inputs.environment }} + env: + TF_IN_AUTOMATION: true + TF_INPUT: false + permissions: + contents: read + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-region: ${{ vars.AWS_REGION }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + role-duration-seconds: 3600 + role-external-id: ${{ secrets.AWS_ROLE_EXTERNAL_ID }} + role-session-name: ${{ inputs.aws_role_session_name }} + role-skip-session-tagging: true + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ inputs.aws_role_name }} + + - name: Apply infrastructure changes + uses: ./.github/actions/apply-infrastructure-changes + with: + summary-title: Terraform Plan Output + tfbackend-path: infrastructure/live/terraform.tfbackend + tfbackend-content: | + bucket="${{ secrets.TF_STATE_BUCKET_NAME }}" + key="${{ inputs.tf_state_key }}" + region="${{ vars.AWS_REGION }}" + tfvars-path: infrastructure/live/terraform.tfvars + tfvars-content: | + availability_zones=${{ vars.AWS_AVAILABILITY_ZONES }} + aws_region="${{ vars.AWS_REGION }}" + backend_image_tag="${{ inputs.release_version }}" + backend_use_fargate_spot=${{ inputs.backend_use_fargate_spot }} + domain_name="${{ vars.DOMAIN_NAME }}" + django_configuration="${{ inputs.django_configuration }}" + django_release_version="${{ inputs.release_version }}" + django_settings_module="${{ inputs.django_settings_module }}" + enable_additional_parameters=${{ inputs.enable_additional_parameters }} + enable_cron_tasks=${{ inputs.enable_cron_tasks }} + enable_nat_gateway=${{ inputs.enable_nat_gateway }} + enable_rds_proxy=${{ inputs.enable_rds_proxy }} + environment="${{ inputs.environment }}" + frontend_use_fargate_spot=${{ inputs.frontend_use_fargate_spot }} + frontend_image_tag="${{ inputs.release_version }}" + project_name="nest" + tasks_use_fargate_spot=${{ inputs.tasks_use_fargate_spot }} + working-directory: infrastructure/live + + - name: Capture terraform outputs + id: tf-outputs + run: | + echo "backend_cluster_name=$(terraform output -raw backend_cluster_name)" >> $GITHUB_OUTPUT + echo "backend_service_name=$(terraform output -raw backend_service_name)" >> $GITHUB_OUTPUT + echo "frontend_cluster_name=$(terraform output -raw frontend_cluster_name)" >> $GITHUB_OUTPUT + echo "frontend_service_name=$(terraform output -raw frontend_service_name)" >> $GITHUB_OUTPUT + echo "nat_gateway_enabled=$(terraform output -raw nat_gateway_enabled)" >> $GITHUB_OUTPUT + echo "tasks_cluster_name=$(terraform output -raw tasks_cluster_name)" >> $GITHUB_OUTPUT + echo "tasks_security_group_id=$(terraform output -raw tasks_security_group_id)" >> $GITHUB_OUTPUT + echo "tasks_subnet_ids=$(terraform output -json tasks_subnet_ids | jq -r 'join(",")')" >> $GITHUB_OUTPUT + working-directory: infrastructure/live + + - name: Run ECS migrate task + id: migrate-task + env: + CLUSTER_NAME: ${{ steps.tf-outputs.outputs.tasks_cluster_name }} + NAT_GATEWAY_ENABLED: ${{ steps.tf-outputs.outputs.nat_gateway_enabled }} + TASKS_SECURITY_GROUP_ID: ${{ steps.tf-outputs.outputs.tasks_security_group_id }} + TASK_DEFINITION: ${{ inputs.migrate_task_definition }} + TASKS_SUBNET_IDS: ${{ steps.tf-outputs.outputs.tasks_subnet_ids }} + run: | + ASSIGN_PUBLIC_IP=$([ "$NAT_GATEWAY_ENABLED" = "true" ] && echo "DISABLED" || echo "ENABLED") + TASK_ARN=$(aws ecs run-task \ + --cluster "$CLUSTER_NAME" \ + --task-definition "$TASK_DEFINITION" \ + --launch-type FARGATE \ + --network-configuration "awsvpcConfiguration={subnets=[$TASKS_SUBNET_IDS],securityGroups=[$TASKS_SECURITY_GROUP_ID],assignPublicIp=$ASSIGN_PUBLIC_IP}" \ + --query 'tasks[0].taskArn' \ + --output text) + echo "task_arn=$TASK_ARN" >> $GITHUB_OUTPUT + + - name: Wait for migrate task + env: + CLUSTER_NAME: ${{ steps.tf-outputs.outputs.tasks_cluster_name }} + run: | + aws ecs wait tasks-stopped \ + --cluster "$CLUSTER_NAME" \ + --tasks "${{ steps.migrate-task.outputs.task_arn }}" + + - name: Check migrate task exit code + env: + CLUSTER_NAME: ${{ steps.tf-outputs.outputs.tasks_cluster_name }} + run: | + EXIT_CODE=$(aws ecs describe-tasks \ + --cluster "$CLUSTER_NAME" \ + --tasks "${{ steps.migrate-task.outputs.task_arn }}" \ + --query 'tasks[0].containers[0].exitCode' \ + --output text) + + if [ "$EXIT_CODE" != "0" ]; then + echo "::error::Migrate task failed with exit code $EXIT_CODE" + exit 1 + fi + echo "Migrate task completed successfully" + + - name: Run ECS index-data task + env: + CLUSTER_NAME: ${{ steps.tf-outputs.outputs.tasks_cluster_name }} + NAT_GATEWAY_ENABLED: ${{ steps.tf-outputs.outputs.nat_gateway_enabled }} + TASK_DEFINITION: ${{ inputs.index_task_definition }} + TASKS_SECURITY_GROUP_ID: ${{ steps.tf-outputs.outputs.tasks_security_group_id }} + TASKS_SUBNET_IDS: ${{ steps.tf-outputs.outputs.tasks_subnet_ids }} + run: | + ASSIGN_PUBLIC_IP=$([ "$NAT_GATEWAY_ENABLED" = "true" ] && echo "DISABLED" || echo "ENABLED") + RESPONSE=$(aws ecs run-task \ + --cluster "$CLUSTER_NAME" \ + --task-definition "$TASK_DEFINITION" \ + --launch-type FARGATE \ + --network-configuration "awsvpcConfiguration={subnets=[$TASKS_SUBNET_IDS],securityGroups=[$TASKS_SECURITY_GROUP_ID],assignPublicIp=$ASSIGN_PUBLIC_IP}") + + FAILURE_COUNT=$(echo "$RESPONSE" | jq '.failures | length') + TASK_COUNT=$(echo "$RESPONSE" | jq '.tasks | length') + + if [ "$FAILURE_COUNT" -gt 0 ]; then + echo "Error: ECS run-task for $TASK_DEFINITION on cluster $CLUSTER_NAME returned failures:" + echo "$RESPONSE" | jq '.failures' + exit 1 + fi + + if [ "$TASK_COUNT" -eq 0 ]; then + echo "Error: ECS run-task for $TASK_DEFINITION on cluster $CLUSTER_NAME returned no tasks" + echo "$RESPONSE" + exit 1 + fi + + echo "Index-data task started successfully (runs async, ~30 min)" + + - name: Wait for backend service stability + env: + CLUSTER_NAME: ${{ steps.tf-outputs.outputs.backend_cluster_name }} + SERVICE_NAME: ${{ steps.tf-outputs.outputs.backend_service_name }} + run: | + aws ecs wait services-stable \ + --cluster "$CLUSTER_NAME" \ + --services "$SERVICE_NAME" + + - name: Wait for frontend service stability + env: + CLUSTER_NAME: ${{ steps.tf-outputs.outputs.frontend_cluster_name }} + SERVICE_NAME: ${{ steps.tf-outputs.outputs.frontend_service_name }} + run: | + aws ecs wait services-stable \ + --cluster "$CLUSTER_NAME" \ + --services "$SERVICE_NAME" + timeout-minutes: 60 diff --git a/.github/workflows/run-e2e-tests.yaml b/.github/workflows/run-e2e-tests.yaml new file mode 100644 index 0000000000..18663cb07a --- /dev/null +++ b/.github/workflows/run-e2e-tests.yaml @@ -0,0 +1,138 @@ +name: E2E tests + +on: + workflow_call: + +env: + FORCE_COLOR: 1 + +permissions: {} + +jobs: + run-e2e-tests: + name: E2E tests + env: + NEXT_PUBLIC_E2E_BACKEND_BASE_URL: http://localhost:9000 + NEXT_SERVER_CSRF_URL: http://localhost:9000/csrf/ + NEXT_SERVER_GRAPHQL_URL: http://localhost:9000/graphql/ + permissions: + contents: read + runs-on: ubuntu-latest + services: + cache: + image: redis:8.8.0-alpine3.23@sha256:09160599abd229764c0fb44cb6be640294e1d360a54b19985ab4843dcf2d90f1 + options: >- + --health-cmd="redis-cli ping" + --health-interval=5s + --health-retries=5 + --health-timeout=5s + ports: + - 6379:6379 + db: + env: + POSTGRES_DB: nest_db_e2e + POSTGRES_PASSWORD: nest_user_e2e_password + POSTGRES_USER: nest_user_e2e + image: pgvector/pgvector:pg16@sha256:00ba258a66dac104fd5171074a0084462a64a1369d8513f3d0a634e2f24d15bc + options: >- + --health-cmd="pg_isready -U nest_user_e2e -d nest_db_e2e -h localhost -p 5432" + --health-interval=5s + --health-retries=5 + --health-timeout=5s + ports: + - 5432:5432 + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Set up backend environment + uses: ./.github/actions/setup-backend-environment + with: + backend_port: 9000 + db_name: nest_db_e2e + db_password: nest_user_e2e_password + db_username: nest_user_e2e + env_file: .env.e2e-tests + + - name: Install frontend dependencies + uses: ./.github/actions/install-frontend-dependencies + + - name: Install e2e dependencies + run: pnpm install --frozen-lockfile + working-directory: e2e + + - name: Cache Playwright Chromium + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + key: e2e-playwright-chromium-${{ runner.os }}-${{ hashFiles('e2e/package.json', 'e2e/pnpm-lock.yaml') }} + path: ~/.cache/ms-playwright + restore-keys: | + e2e-playwright-chromium-${{ runner.os }}- + + - name: Install Playwright Chromium + run: | + npx playwright install-deps chromium + npx playwright install chromium + working-directory: e2e + + - name: Cache Next.js build + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + key: next-cache-e2e-${{ runner.os }}-${{ hashFiles('frontend/next.config.ts', 'frontend/package.json', 'frontend/pnpm-lock.yaml') }} + path: frontend/.next/cache + + - name: Prepare environment variables for frontend + run: cp .env.e2e .env + working-directory: frontend + + - name: Build frontend + run: pnpm run build + working-directory: frontend + + - name: Copy public and static folders + run: | + cp -r public .next/standalone/ + cp -r .next/static .next/standalone/.next/ + working-directory: frontend + + - name: Run frontend in the background + env: + HOSTNAME: 0.0.0.0 + NEXT_PUBLIC_E2E_BACKEND_BASE_URL: http://localhost:9000 + NEXT_SERVER_CSRF_URL: http://localhost:9000/csrf/ + NEXT_SERVER_GRAPHQL_URL: http://localhost:9000/graphql/ + NEXT_TELEMETRY_DISABLED: '1' + NODE_ENV: production + PORT: '3000' + run: | + nohup node .next/standalone/server.js > /dev/null 2>&1 & + disown + working-directory: frontend + + - name: Wait for frontend to be ready + run: | + timeout 5m bash -c ' + until wget --spider http://localhost:3000/api/health; do + echo "Waiting for the frontend..." + sleep 5 + done + ' + echo "Frontend is up!" + + - name: Run frontend end-to-end tests + env: + CI: 'true' + FRONTEND_URL: http://localhost:3000 + run: pnpm run test:e2e + working-directory: e2e + + - name: Upload Playwright test results + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: playwright-test-results + path: e2e/test-results/ + retention-days: 7 + timeout-minutes: 30 diff --git a/.github/workflows/run-frontend-tests.yaml b/.github/workflows/run-frontend-tests.yaml new file mode 100644 index 0000000000..b279518a60 --- /dev/null +++ b/.github/workflows/run-frontend-tests.yaml @@ -0,0 +1,56 @@ +name: Frontend tests + +on: + workflow_call: + +env: + FORCE_COLOR: 1 + +permissions: {} + +jobs: + run-a11y-tests: + name: Frontend a11y tests + permissions: + contents: read + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install frontend dependencies + uses: ./.github/actions/install-frontend-dependencies + + - name: Run tests + run: pnpm run test:a11y + working-directory: frontend + timeout-minutes: 5 + + run-unit-tests: + name: Frontend unit tests + permissions: + actions: write + contents: read + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install frontend dependencies + uses: ./.github/actions/install-frontend-dependencies + + - name: Run tests + run: pnpm run test:unit + working-directory: frontend + + - name: Upload coverage artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + if-no-files-found: error + name: coverage-lcov + path: frontend/coverage/lcov.info + timeout-minutes: 10 diff --git a/.github/workflows/run-fuzz-tests.yaml b/.github/workflows/run-fuzz-tests.yaml index 7df5df0e9e..f7a29632b2 100644 --- a/.github/workflows/run-fuzz-tests.yaml +++ b/.github/workflows/run-fuzz-tests.yaml @@ -1,4 +1,4 @@ -name: Run fuzz tests +name: Fuzz tests on: workflow_call: @@ -13,12 +13,27 @@ on: type: string default: http://localhost:9500/api/v0 +env: + FORCE_COLOR: 1 + +permissions: {} + jobs: run-fuzz-tests: - name: Run Fuzz Tests + name: Fuzz tests + permissions: + contents: read runs-on: ubuntu-latest - timeout-minutes: 20 services: + cache: + image: redis:8.8.0-alpine3.23@sha256:09160599abd229764c0fb44cb6be640294e1d360a54b19985ab4843dcf2d90f1 + options: >- + --health-cmd="redis-cli ping" + --health-interval=5s + --health-retries=5 + --health-timeout=5s + ports: + - 6379:6379 db: image: pgvector/pgvector:pg16@sha256:00ba258a66dac104fd5171074a0084462a64a1369d8513f3d0a634e2f24d15bc env: @@ -32,89 +47,30 @@ jobs: --health-timeout=5s ports: - 5432:5432 - cache: - image: redis:8.8.0-alpine3.23@sha256:09160599abd229764c0fb44cb6be640294e1d360a54b19985ab4843dcf2d90f1 - options: >- - --health-cmd="redis-cli ping" - --health-interval=5s - --health-retries=5 - --health-timeout=5s - ports: - - 6379:6379 steps: - name: Check out repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - name: Set up Docker buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - - - name: Setup Backend environment - uses: ./.github/workflows/setup-backend-environment + - name: Set up backend environment + uses: ./.github/actions/setup-backend-environment with: - db_username: nest_user_fuzz + backend_port: 9500 db_name: nest_db_fuzz - - - name: Run backend with fuzz environment variables - run: | - docker run -d --rm --name fuzz-nest-backend \ - --env-file backend/.env.fuzz-tests \ - --network host \ - -e DJANGO_DB_HOST=localhost \ - -e DJANGO_REDIS_AUTH_ENABLED=False \ - -e DJANGO_REDIS_HOST=localhost \ - -p 9500:9500 \ - owasp/nest:test-backend-latest \ - sh -c ' - python manage.py migrate && - gunicorn wsgi:application --bind 0.0.0.0:9500 - ' - - - name: Waiting for the backend to be ready - run: | - timeout 5m bash -c ' - until wget --spider http://localhost:9500/a; do - echo "Waiting for backend..." - sleep 5 - done - ' - echo "Backend is up!" - - - name: Load Postgres data - env: - PGPASSWORD: nest_user_fuzz_password - run: | - set -euo pipefail - if ! pg_restore -h localhost -U nest_user_fuzz -d nest_db_fuzz < backend/data/nest.dump; then - echo "Data loading failed" - exit 1 - fi - echo "Data loading completed." - - - name: Build Fuzz-testing image - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 - with: - cache-from: | - type=gha - type=registry,ref=owasp/nest:test-fuzz-backend-cache - cache-to: | - type=gha,compression=zstd - context: backend - file: docker/backend/Dockerfile.fuzz-tests - load: true - platforms: linux/amd64 - tags: owasp/nest:test-fuzz-backend-latest + db_password: nest_user_fuzz_password + db_username: nest_user_fuzz + env_file: .env.fuzz-tests - name: Run fuzz tests env: - TEST_FILE: ${{ inputs.test-file }} + BASE_URL: http://localhost:9500 + CI: 'true' + PYTEST_ADDOPTS: --no-cov REST_URL: ${{ inputs.rest-url }} + TEST_FILE: fuzz/${{ inputs.test-file }} run: | - docker run \ - --network host \ - -e BASE_URL=http://localhost:9500 \ - -e CI=true \ - -e REST_URL="$REST_URL" \ - -e TEST_FILE="$TEST_FILE" \ - owasp/nest:test-fuzz-backend-latest + set -a && source ./.env.fuzz-tests && set +a + poetry run sh ./entrypoint.fuzz.sh + working-directory: backend + timeout-minutes: 20 diff --git a/.github/workflows/run-image-build.yaml b/.github/workflows/run-image-build.yaml new file mode 100644 index 0000000000..0e3c3c0dbf --- /dev/null +++ b/.github/workflows/run-image-build.yaml @@ -0,0 +1,211 @@ +name: Image build + +on: + workflow_call: + inputs: + aws_role_name: + description: AWS role name to assume + required: true + type: string + aws_role_session_name: + description: AWS role session name + required: true + type: string + backend_ecr_cache_repo: + description: Backend ECR repository name for build cache + required: true + type: string + backend_ecr_repo: + description: Backend ECR repository name + required: true + type: string + backend_gid: + description: Backend OWASP GID + required: true + type: string + backend_uid: + description: Backend OWASP UID + required: true + type: string + environment: + description: GitHub environment name + required: true + type: string + frontend_ecr_cache_repo: + description: Frontend ECR repository name for build cache + required: true + type: string + frontend_ecr_repo: + description: Frontend ECR repository name + required: true + type: string + release_version: + description: The release version to set + required: true + type: string + secrets: + AWS_ACCESS_KEY_ID: + required: true + AWS_ACCOUNT_ID: + required: true + AWS_ROLE_EXTERNAL_ID: + required: true + AWS_SECRET_ACCESS_KEY: + required: true + NEXT_PUBLIC_GTM_ID: + required: true + NEXT_PUBLIC_IS_PROJECT_HEALTH_ENABLED: + required: true + NEXT_PUBLIC_POSTHOG_HOST: + required: true + NEXT_PUBLIC_POSTHOG_KEY: + required: true + SENTRY_AUTH_TOKEN: + required: true + VITE_API_URL: + required: true + VITE_CSRF_URL: + required: true + VITE_ENVIRONMENT: + required: true + VITE_GRAPHQL_URL: + required: true + VITE_IDX_URL: + required: true + VITE_SENTRY_DSN: + required: true + +env: + FORCE_COLOR: 1 + +permissions: {} + +jobs: + run-image-build: + name: Image build + environment: ${{ inputs.environment }} + env: + BACKEND_CACHE_REF: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/${{ inputs.backend_ecr_cache_repo }}:cache + BACKEND_IMAGE: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/${{ inputs.backend_ecr_repo }}:${{ inputs.release_version + }} + FRONTEND_CACHE_REF: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/${{ inputs.frontend_ecr_cache_repo }}:cache + FRONTEND_IMAGE: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/${{ inputs.frontend_ecr_repo }}:${{ inputs.release_version + }} + RELEASE_VERSION: ${{ inputs.release_version }} + permissions: + contents: read + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Set up Docker buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-region: ${{ vars.AWS_REGION }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + role-duration-seconds: 3600 + role-external-id: ${{ secrets.AWS_ROLE_EXTERNAL_ID }} + role-session-name: ${{ inputs.aws_role_session_name }} + role-skip-session-tagging: true + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ inputs.aws_role_name }} + + - name: Login to Amazon ECR + uses: aws-actions/amazon-ecr-login@fa648b43de3d4d023bcb3f89ed6940096949c419 # v2.1.5 + + - name: Build backend image + uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 + with: + build-args: | + OWASP_GID=${{ inputs.backend_gid }} + OWASP_UID=${{ inputs.backend_uid }} + cache-from: | + type=gha + type=registry,ref=${{ env.BACKEND_CACHE_REF }} + cache-to: | + type=gha,mode=max + type=registry,ref=${{ env.BACKEND_CACHE_REF }},mode=max + context: backend + file: docker/backend/Dockerfile + load: true + platforms: linux/amd64 + push: true + tags: ${{ env.BACKEND_IMAGE }} + + - name: Prepare frontend public environment + env: + NEXT_PUBLIC_API_URL: ${{ secrets.VITE_API_URL }} + NEXT_PUBLIC_CSRF_URL: ${{ secrets.VITE_CSRF_URL }} + NEXT_PUBLIC_ENVIRONMENT: ${{ secrets.VITE_ENVIRONMENT }} + NEXT_PUBLIC_GRAPHQL_URL: ${{ secrets.VITE_GRAPHQL_URL }} + NEXT_PUBLIC_GTM_ID: ${{ secrets.NEXT_PUBLIC_GTM_ID }} + NEXT_PUBLIC_IDX_URL: ${{ secrets.VITE_IDX_URL }} + NEXT_PUBLIC_IS_PROJECT_HEALTH_ENABLED: ${{ secrets.NEXT_PUBLIC_IS_PROJECT_HEALTH_ENABLED }} + NEXT_PUBLIC_POSTHOG_HOST: ${{ secrets.NEXT_PUBLIC_POSTHOG_HOST }} + NEXT_PUBLIC_POSTHOG_KEY: ${{ secrets.NEXT_PUBLIC_POSTHOG_KEY }} + NEXT_PUBLIC_RELEASE_VERSION: ${{ env.RELEASE_VERSION }} + NEXT_PUBLIC_SENTRY_DSN: ${{ secrets.VITE_SENTRY_DSN }} + run: | + umask 377 + cat > frontend/.env <> $GITHUB_OUTPUT + + - name: Build frontend image + uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 + with: + cache-from: | + type=gha + type=registry,ref=${{ env.FRONTEND_CACHE_REF }} + cache-to: | + type=gha,mode=max + type=registry,ref=${{ env.FRONTEND_CACHE_REF }},mode=max + context: frontend + file: docker/frontend/Dockerfile + load: true + platforms: linux/amd64 + push: true + secrets: | + RELEASE_VERSION=${{ env.RELEASE_VERSION }} + SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} + tags: ${{ env.FRONTEND_IMAGE }} + + - name: Get frontend image size + id: frontend-size + run: | + RAW_SIZE=$(docker image inspect "$FRONTEND_IMAGE" --format='{{.Size}}') + DISPLAY_SIZE=$(numfmt --to=iec --suffix=B "$RAW_SIZE") + echo "human_readable=$DISPLAY_SIZE" >> $GITHUB_OUTPUT + + - name: Create Docker image size report + run: | + { + echo "## Docker Image Size Report" + echo "" + echo "**Backend:** ${{ steps.backend-size.outputs.human_readable }}" + echo "**Frontend:** ${{ steps.frontend-size.outputs.human_readable }}" + } >> $GITHUB_STEP_SUMMARY + timeout-minutes: 10 diff --git a/.github/workflows/run-image-scan.yaml b/.github/workflows/run-image-scan.yaml new file mode 100644 index 0000000000..c1c78ef1b6 --- /dev/null +++ b/.github/workflows/run-image-scan.yaml @@ -0,0 +1,137 @@ +name: Image scan + +on: + workflow_call: + inputs: + aws_role_name: + description: AWS role name to assume + required: true + type: string + aws_role_session_name: + description: AWS role session name + required: true + type: string + backend_ecr_repo: + description: Backend ECR repository name + required: true + type: string + environment: + description: The workflow environment to use for scanning (e.g., staging, production) + required: true + type: string + frontend_ecr_repo: + description: Frontend ECR repository name + required: true + type: string + release_tag: + description: Release tag for uploading SBOM (production only) + required: false + type: string + default: '' + release_version: + description: The release version to set + required: true + type: string + secrets: + AWS_ACCESS_KEY_ID: + required: true + AWS_ACCOUNT_ID: + required: true + AWS_ROLE_EXTERNAL_ID: + required: true + AWS_SECRET_ACCESS_KEY: + required: true + +env: + FORCE_COLOR: 1 + +permissions: {} + +jobs: + run-image-scan: + name: Image scan + environment: ${{ inputs.environment }} + env: + BACKEND_IMAGE: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/${{ inputs.backend_ecr_repo }}:${{ inputs.release_version + }} + FRONTEND_IMAGE: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/${{ inputs.frontend_ecr_repo }}:${{ inputs.release_version + }} + RELEASE_VERSION: ${{ inputs.release_version }} + permissions: + actions: write + contents: read + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-region: ${{ vars.AWS_REGION }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + role-duration-seconds: 3600 + role-external-id: ${{ secrets.AWS_ROLE_EXTERNAL_ID }} + role-session-name: ${{ inputs.aws_role_session_name }} + role-skip-session-tagging: true + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ inputs.aws_role_name }} + + - name: Login to Amazon ECR + uses: aws-actions/amazon-ecr-login@fa648b43de3d4d023bcb3f89ed6940096949c419 # v2.1.5 + + - name: Pull container images + run: | + docker pull "$BACKEND_IMAGE" + docker pull "$FRONTEND_IMAGE" + + - name: Run trivy scan + uses: ./.github/actions/run-trivy-scan + with: + command: | + make security-scan-backend-image BACKEND_IMAGE_NAME="$BACKEND_IMAGE" + make security-scan-frontend-image FRONTEND_IMAGE_NAME="$FRONTEND_IMAGE" + + - name: Generate SBOM for backend image + run: make sbom-backend-image BACKEND_IMAGE_NAME="$BACKEND_IMAGE" + + - name: Generate SBOM for frontend image + run: make sbom-frontend-image FRONTEND_IMAGE_NAME="$FRONTEND_IMAGE" + + - name: Upload SBOM as artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: sbom-${{ env.RELEASE_VERSION }} + path: | + backend-sbom-${{ env.RELEASE_VERSION }}.cdx.json + frontend-sbom-${{ env.RELEASE_VERSION }}.cdx.json + timeout-minutes: 10 + + upload-release-sbom: + name: SBOM upload + if: inputs.environment == 'production' + needs: + - run-image-scan + permissions: + actions: read + contents: write + runs-on: ubuntu-latest + env: + RELEASE_VERSION: ${{ inputs.release_version }} + steps: + - name: Download SBOM + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: sbom-${{ env.RELEASE_VERSION }} + + - name: Upload SBOM to release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RELEASE_TAG: ${{ inputs.release_tag }} + run: | + gh release upload "$RELEASE_TAG" --clobber \ + "backend-sbom-$RELEASE_VERSION.cdx.json" \ + "frontend-sbom-$RELEASE_VERSION.cdx.json" + timeout-minutes: 5 diff --git a/.github/workflows/run-infrastructure-bootstrap.yaml b/.github/workflows/run-infrastructure-bootstrap.yaml new file mode 100644 index 0000000000..ed8f9a5617 --- /dev/null +++ b/.github/workflows/run-infrastructure-bootstrap.yaml @@ -0,0 +1,64 @@ +name: Infrastructure bootstrap + +on: + workflow_call: + inputs: + environment: + description: GitHub environment name + required: true + type: string + secrets: + AWS_ROLE_EXTERNAL_ID: + required: true + BOOTSTRAP_AWS_ACCESS_KEY_ID: + required: true + BOOTSTRAP_AWS_SECRET_ACCESS_KEY: + required: true + BOOTSTRAP_TF_STATE_BUCKET_NAME: + required: true + +env: + FORCE_COLOR: 1 + TF_PLUGIN_CACHE_DIR: /home/runner/.terraform.d/plugin-cache + +permissions: {} + +jobs: + run-infrastructure-bootstrap: + name: Infrastructure bootstrap + environment: ${{ inputs.environment }} + env: + TF_IN_AUTOMATION: true + TF_INPUT: false + permissions: + contents: read + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@d979d5b3a71173a29b74b5b88418bfda9437d885 # v6.1.1 + with: + aws-access-key-id: ${{ secrets.BOOTSTRAP_AWS_ACCESS_KEY_ID }} + aws-region: ${{ vars.AWS_REGION }} + aws-secret-access-key: ${{ secrets.BOOTSTRAP_AWS_SECRET_ACCESS_KEY }} + + - name: Apply infrastructure changes + uses: ./.github/actions/apply-infrastructure-changes + with: + plugin-cache-dir: ${{ env.TF_PLUGIN_CACHE_DIR }} + summary-title: Bootstrap Terraform Plan Output + tfbackend-path: infrastructure/bootstrap/terraform.tfbackend + tfbackend-content: | + bucket="${{ secrets.BOOTSTRAP_TF_STATE_BUCKET_NAME }}" + region="${{ vars.AWS_REGION }}" + tfvars-path: infrastructure/bootstrap/terraform.tfvars + tfvars-content: | + aws_region="${{ vars.AWS_REGION }}" + project_name="nest" + aws_role_external_id="${{ secrets.AWS_ROLE_EXTERNAL_ID }}" + working-directory: infrastructure/bootstrap + timeout-minutes: 10 diff --git a/.github/workflows/run-infrastructure-tests.yaml b/.github/workflows/run-infrastructure-tests.yaml new file mode 100644 index 0000000000..879179c503 --- /dev/null +++ b/.github/workflows/run-infrastructure-tests.yaml @@ -0,0 +1,40 @@ +name: Infrastructure tests + +on: + workflow_call: + +env: + FORCE_COLOR: 1 + TF_PLUGIN_CACHE_DIR: /home/runner/.terraform.d/plugin-cache + +permissions: {} + +jobs: + run-infrastructure-tests: + name: Infrastructure tests + permissions: + contents: read + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install Terraform + uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 + with: + terraform_version: 1.14 + + - name: Create plugin cache directory + run: mkdir -p "$TF_PLUGIN_CACHE_DIR" + + - name: Cache Terraform plugins + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + key: ${{ runner.os }}-terraform-${{ hashFiles('infrastructure/**/.terraform.lock.hcl') }} + path: ${{ env.TF_PLUGIN_CACHE_DIR }} + + - name: Run tests + run: make test-infrastructure + timeout-minutes: 10 diff --git a/.github/workflows/run-lighthouse-ci.yaml b/.github/workflows/run-lighthouse-ci.yaml new file mode 100644 index 0000000000..bf7a19d052 --- /dev/null +++ b/.github/workflows/run-lighthouse-ci.yaml @@ -0,0 +1,36 @@ +name: Lighthouse CI + +on: + workflow_call: + inputs: + base_url: + description: Base URL for Lighthouse CI + required: true + type: string + +env: + FORCE_COLOR: 1 + +permissions: {} + +jobs: + run-lighthouse-ci: + name: Lighthouse CI + permissions: + contents: read + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install frontend dependencies + uses: ./.github/actions/install-frontend-dependencies + + - name: Run lighthouse-ci + env: + LHCI_BASE_URL: ${{ inputs.base_url }} + run: pnpm run lighthouse-ci + working-directory: frontend + timeout-minutes: 5 diff --git a/.github/workflows/run-release-version-resolution.yaml b/.github/workflows/run-release-version-resolution.yaml new file mode 100644 index 0000000000..877e7b1512 --- /dev/null +++ b/.github/workflows/run-release-version-resolution.yaml @@ -0,0 +1,39 @@ +name: Resolve release version + +on: + workflow_call: + outputs: + release_version: + description: The resolved release version + value: ${{ jobs.run-release-version-resolution.outputs.release_version }} + +env: + FORCE_COLOR: 1 + +permissions: {} + +jobs: + run-release-version-resolution: + name: Resolve release version + runs-on: ubuntu-latest + outputs: + release_version: ${{ steps.release-version-resolution.outputs.release_version }} + steps: + - name: Resolve release version + id: release-version-resolution + env: + RELEASE_TAG: ${{ github.event.release.tag_name }} + run: | + if [ -n "$RELEASE_TAG" ]; then + release_version="$RELEASE_TAG" + else + release_version="$(date '+%y.%-m.%-d')-${GITHUB_SHA::7}" + fi + + if [[ ! "$release_version" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]]; then + echo "::error::Invalid Docker tag: $release_version" + exit 1 + fi + + echo "release_version=$release_version" >> "$GITHUB_OUTPUT" + timeout-minutes: 5 diff --git a/.github/workflows/run-zap-baseline-scan.yaml b/.github/workflows/run-zap-baseline-scan.yaml new file mode 100644 index 0000000000..c099aecf4a --- /dev/null +++ b/.github/workflows/run-zap-baseline-scan.yaml @@ -0,0 +1,43 @@ +name: ZAP baseline scan + +on: + workflow_call: + inputs: + base_url: + description: Base URL for ZAP baseline scan + required: true + type: string + +env: + FORCE_COLOR: 1 + +permissions: {} + +jobs: + run-zap-baseline-scan: + name: ZAP baseline scan + permissions: + contents: read + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Run baseline scan + uses: zaproxy/action-baseline@de8ad967d3548d44ef623df22cf95c3b0baf8b25 # v0.15.0 + with: + allow_issue_writing: false + cmd_options: -a -c .zapconfig -r report_html.html + fail_action: true + target: ${{ inputs.base_url }} + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: zap-baseline-scan-report-${{ github.run_id }} + path: report_html.html + timeout-minutes: 5 diff --git a/.github/workflows/setup-backend-environment/action.yaml b/.github/workflows/setup-backend-environment/action.yaml deleted file mode 100644 index ff5bcc367e..0000000000 --- a/.github/workflows/setup-backend-environment/action.yaml +++ /dev/null @@ -1,65 +0,0 @@ -name: Set up Backend environment - -description: >- - Fetches nest.dump from S3 using the same Poetry environment as local Make targets, waits for - Postgres, installs the PostgreSQL client, and builds the backend test image. - -inputs: - db_username: - description: Database username - required: true - db_name: - description: Database name - required: true - -runs: - using: composite - steps: - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: '3.13' - - - name: Install Poetry - run: python -m pip install -r backend/requirements.build.txt - shell: bash - - - name: Install backend dependencies (fetch script) - run: | - cd backend - poetry install --no-root --without fuzz --without test --without video - shell: bash - - - name: Fetch nest.dump from S3 - run: cd backend && poetry run python -m scripts.fetch_nest_dump - shell: bash - - - name: Wait for database to be ready - env: - DB_USERNAME: ${{ inputs.db_username }} - DB_NAME: ${{ inputs.db_name }} - run: | - timeout 5m bash -c ' - until docker exec ${{ job.services.db.id }} pg_isready -U $DB_USERNAME -d $DB_NAME; do - echo "Waiting for database..." - sleep 5 - done - ' - shell: bash - - - name: Install PostgreSQL client - run: sudo apt-get install -y postgresql-client - shell: bash - - - name: Build backend image - uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 - with: - cache-from: | - type=gha - cache-to: | - type=gha,compression=zstd - context: backend - file: docker/backend/Dockerfile - load: true - platforms: linux/amd64 - tags: owasp/nest:test-backend-latest diff --git a/.github/workflows/update-nest-test-images.yaml b/.github/workflows/update-nest-test-images.yaml deleted file mode 100644 index a5cb483f87..0000000000 --- a/.github/workflows/update-nest-test-images.yaml +++ /dev/null @@ -1,92 +0,0 @@ -name: Update Nest Test Images - -on: - schedule: - - cron: 30 0 * * * - workflow_dispatch: - -permissions: {} - -env: - FORCE_COLOR: 1 - -jobs: - update-nest-test-images: - name: Update Nest test images - if: ${{ github.repository == 'OWASP/Nest' }} - permissions: - actions: write - contents: read - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Set up Docker buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - - - name: Login to Docker Hub - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Update backend test image - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 - with: - cache-from: | - type=gha - type=registry,ref=owasp/nest:test-backend-cache - cache-to: | - type=gha,compression=zstd - type=registry,ref=owasp/nest:test-backend-cache - context: backend - file: docker/backend/Dockerfile.unit-tests - platforms: linux/amd64 - push: true - tags: owasp/nest:test-backend-latest - - - name: Update frontend unit test image - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 - with: - cache-from: | - type=gha - type=registry,ref=owasp/nest:test-frontend-unit-cache - cache-to: | - type=gha,compression=zstd - type=registry,ref=owasp/nest:test-frontend-unit-cache - context: frontend - file: docker/frontend/Dockerfile.unit-tests - platforms: linux/amd64 - push: true - tags: owasp/nest:test-frontend-unit-latest - - - name: Update frontend end-to-end test image - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 - with: - cache-from: | - type=gha - type=registry,ref=owasp/nest:test-frontend-e2e-cache - cache-to: | - type=gha,compression=zstd - type=registry,ref=owasp/nest:test-frontend-e2e-cache - context: . - file: docker/e2e/Dockerfile - platforms: linux/amd64 - push: true - tags: owasp/nest:test-frontend-e2e-latest - - - name: Build and push fuzz-test-backend image - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 - with: - cache-from: type=registry,ref=owasp/nest:test-fuzz-backend-cache - cache-to: | - type=gha,compression=zstd - type=registry,ref=owasp/nest:test-fuzz-backend-cache - context: backend - file: docker/backend/Dockerfile.fuzz-tests - platforms: linux/amd64 - push: true - tags: owasp/nest:test-fuzz-backend-latest - timeout-minutes: 15 diff --git a/.trivyignore.yaml b/.trivyignore.yaml index 766d8cf342..f65e54a664 100644 --- a/.trivyignore.yaml +++ b/.trivyignore.yaml @@ -1,4 +1,10 @@ misconfigurations: + - id: AVD-AWS-0030 # BuildKit cache-only ECR; release images use scan-on-push. + paths: + - infrastructure/modules/ecr-cache/main.tf + - id: AVD-AWS-0031 # Mutable :cache tag required for BuildKit registry cache overwrite. + paths: + - infrastructure/modules/ecr-cache/main.tf - id: DS-0002 # Specify at least 1 USER command in Dockerfile with non-root user as argument. paths: - docker/e2e/Dockerfile diff --git a/README.md b/README.md index 17d16430be..af6b4fb63e 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![OWASP](https://img.shields.io/badge/production-blue?&label=level&style=for-the-badge)](https://owasp.org/www-project-nest/) [![OWASP](https://img.shields.io/badge/Code-blue?label=type&style=for-the-badge)](https://owasp.org/www-project-nest/) [![project-nest](https://img.shields.io/badge/%23project--nest-blue?label=slack&logoColor=white&style=for-the-badge)](https://owasp.slack.com/archives/project-nest) -[![CI/CD](https://img.shields.io/github/actions/workflow/status/owasp/nest/run-ci-cd.yaml?branch=main&color=blue&label=Build&style=for-the-badge)](https://github.com/owasp/nest/actions/workflows/run-ci-cd.yaml?query=branch%3Amain) [![CodeQL](https://img.shields.io/github/actions/workflow/status/owasp/nest/run-code-ql.yaml?branch=main&color=blue&label=CodeQL&style=for-the-badge)](https://github.com/owasp/nest/actions/workflows/run-code-ql.yaml?query=branch%3Amain) [![Sonarqube](https://img.shields.io/sonar/quality_gate/OWASP_Nest?color=blue&server=https://sonarcloud.io&style=for-the-badge&label=Sonarqube)](https://sonarcloud.io/summary/new_code?id=OWASP_Nest&branch=main) +[![CI](https://img.shields.io/github/actions/workflow/status/owasp/nest/ci.yaml?branch=main&color=blue&label=CI&style=for-the-badge)](https://github.com/owasp/nest/actions/workflows/ci.yaml?query=branch%3Amain) [![CodeQL](https://img.shields.io/github/actions/workflow/status/owasp/nest/code-ql.yaml?branch=main&color=blue&label=CodeQL&style=for-the-badge)](https://github.com/owasp/nest/actions/workflows/code-ql.yaml?query=branch%3Amain) [![CI/CD](https://img.shields.io/github/actions/workflow/status/owasp/nest/ci-cd-staging.yaml?branch=main&color=blue&label=CI%2FCD%20Staging&style=for-the-badge)](https://github.com/owasp/nest/actions/workflows/ci-cd-staging.yaml?query=branch%3Amain) [![Sonarqube](https://img.shields.io/sonar/quality_gate/OWASP_Nest?color=blue&server=https://sonarcloud.io&style=for-the-badge&label=Sonarqube)](https://sonarcloud.io/summary/new_code?id=OWASP_Nest&branch=main) [![License](https://img.shields.io/badge/license-%20MIT-blue?style=for-the-badge)](https://github.com/OWASP/Nest/blob/main/LICENSE) [![Last Commit](https://img.shields.io/github/last-commit/owasp/nest/main?color=blue&style=for-the-badge&label=Last%20commit)](https://github.com/OWASP/Nest/commits/main/) [![Contributors](https://img.shields.io/github/contributors/owasp/nest?style=for-the-badge&label=Contributors&color=blue)](https://github.com/OWASP/Nest/graphs/contributors) diff --git a/backend/entrypoint.fuzz.sh b/backend/entrypoint.fuzz.sh index bbfbab7c52..9f247a19e5 100644 --- a/backend/entrypoint.fuzz.sh +++ b/backend/entrypoint.fuzz.sh @@ -43,7 +43,7 @@ fi echo "Starting fuzzing process..." if [ -n "$CI" ]; then - pytest ./tests/${TEST_FILE} + pytest "./tests/${TEST_FILE}" else - pytest --log-cli-level=INFO -s ./tests/${TEST_FILE} + pytest --log-cli-level=INFO -s "./tests/${TEST_FILE}" fi diff --git a/cspell/custom-dict.txt b/cspell/custom-dict.txt index 447c3633a9..ff108d3bff 100644 --- a/cspell/custom-dict.txt +++ b/cspell/custom-dict.txt @@ -23,6 +23,7 @@ Emay FOSS GBP GFKs +GPG GRC GSOC GTM @@ -107,7 +108,6 @@ defectdojo demojize dismissable dkr -dockerhub dsn dvo elevenlabs @@ -169,6 +169,7 @@ mpim navlink nestbot ngx +nohup noinput nosniff notab diff --git a/e2e/components/Footer.spec.ts b/e2e/components/Footer.spec.ts index 37543693d5..7ebae9dd6b 100644 --- a/e2e/components/Footer.spec.ts +++ b/e2e/components/Footer.spec.ts @@ -1,4 +1,5 @@ -import { test, expect, devices } from '@playwright/test' +import { iphone13Chromium } from '@e2e/helpers/devices' +import { test, expect } from '@playwright/test' // Desktop tests test.describe('Footer - Desktop (Chrome)', () => { @@ -22,9 +23,9 @@ test.describe('Footer - Desktop (Chrome)', () => { }) }) -// Mobile tests (iPhone 13) +// Mobile tests (iPhone 13, Chromium) test.use({ - ...devices['iPhone 13'], + ...iphone13Chromium, isMobile: true, }) diff --git a/e2e/components/Header.spec.ts b/e2e/components/Header.spec.ts index 0b4492910e..e3df26b4b7 100644 --- a/e2e/components/Header.spec.ts +++ b/e2e/components/Header.spec.ts @@ -1,4 +1,5 @@ -import { test, expect, devices } from '@playwright/test' +import { iphone13Chromium } from '@e2e/helpers/devices' +import { test, expect } from '@playwright/test' // Desktop tests test.describe('Header - Desktop (Chrome)', () => { @@ -51,9 +52,9 @@ test.describe('Header - Desktop (Chrome)', () => { }) }) -// Mobile tests (iPhone 13) +// Mobile tests (iPhone 13, Chromium) test.use({ - ...devices['iPhone 13'], + ...iphone13Chromium, isMobile: true, }) diff --git a/e2e/helpers/devices.ts b/e2e/helpers/devices.ts new file mode 100644 index 0000000000..25f39cad55 --- /dev/null +++ b/e2e/helpers/devices.ts @@ -0,0 +1,9 @@ +import { devices } from '@playwright/test' + +// iPhone 13 viewport without WebKit; use Chromium for mobile emulation in CI and locally. +const { defaultBrowserType: _webkit, ...iphone13 } = devices['iPhone 13'] + +export const iphone13Chromium = { + ...iphone13, + browserName: 'chromium' as const, +} diff --git a/e2e/pages/About.spec.ts b/e2e/pages/About.spec.ts index 9d33270f67..bacb3f2953 100644 --- a/e2e/pages/About.spec.ts +++ b/e2e/pages/About.spec.ts @@ -3,7 +3,7 @@ import { test, expect } from '@playwright/test' test.describe('About Page', () => { test.beforeEach(async ({ page }) => { - await page.goto('/about', { timeout: 25000 }) + await page.goto('/about') }) test('renders main sections correctly', async ({ page }) => { diff --git a/e2e/pages/CalendarButton.spec.ts b/e2e/pages/CalendarButton.spec.ts index ee3eb39e50..468fa25de8 100644 --- a/e2e/pages/CalendarButton.spec.ts +++ b/e2e/pages/CalendarButton.spec.ts @@ -4,7 +4,7 @@ import slugify from 'utils/slugify' test.describe('Calendar Export Functionality', () => { test.beforeEach(async ({ page }) => { - await page.goto('/', { timeout: 25000 }) + await page.goto('/') }) test('should download a valid ICS file when clicked', async ({ page }) => { diff --git a/e2e/pages/ChapterDetails.spec.ts b/e2e/pages/ChapterDetails.spec.ts index 8b2c172d27..f3b5dce3c3 100644 --- a/e2e/pages/ChapterDetails.spec.ts +++ b/e2e/pages/ChapterDetails.spec.ts @@ -2,7 +2,7 @@ import { test, expect } from '@playwright/test' test.describe('Chapter Details Page', () => { test.beforeEach(async ({ page }) => { - await page.goto('/chapters/rosario', { timeout: 25000 }) + await page.goto('/chapters/rosario') }) test('should have a heading and summary', async ({ page }) => { diff --git a/e2e/pages/Chapters.spec.ts b/e2e/pages/Chapters.spec.ts index d4a7e40f86..1211422833 100644 --- a/e2e/pages/Chapters.spec.ts +++ b/e2e/pages/Chapters.spec.ts @@ -13,7 +13,7 @@ test.describe('Chapters Page', () => { }), }) }) - await page.goto('/chapters', { timeout: 25000 }) + await page.goto('/chapters') }) test('renders chapter data correctly', async ({ page }) => { diff --git a/e2e/pages/CommitteeDetails.spec.ts b/e2e/pages/CommitteeDetails.spec.ts index 2d115cb4d3..25e13b8bf4 100644 --- a/e2e/pages/CommitteeDetails.spec.ts +++ b/e2e/pages/CommitteeDetails.spec.ts @@ -2,7 +2,7 @@ import { test, expect } from '@playwright/test' test.describe('Committee Details Page', () => { test.beforeEach(async ({ page }) => { - await page.goto('/committees/events', { timeout: 25000 }) + await page.goto('/committees/events') }) test('should have a heading and summary', async ({ page }) => { diff --git a/e2e/pages/Committees.spec.ts b/e2e/pages/Committees.spec.ts index fb0ac12631..ffbbb3f2dd 100644 --- a/e2e/pages/Committees.spec.ts +++ b/e2e/pages/Committees.spec.ts @@ -13,7 +13,7 @@ test.describe('Committees Page', () => { }), }) }) - await page.goto('/committees', { timeout: 25000 }) + await page.goto('/committees') }) test('renders committee data correctly', async ({ page }) => { diff --git a/e2e/pages/Community.spec.ts b/e2e/pages/Community.spec.ts index b428adb53e..452ecef529 100644 --- a/e2e/pages/Community.spec.ts +++ b/e2e/pages/Community.spec.ts @@ -2,7 +2,7 @@ import { test, expect } from '@playwright/test' test.describe('Community Page', () => { test.beforeEach(async ({ page }) => { - await page.goto('/community', { timeout: 25000 }) + await page.goto('/community') }) test('renders main heading', async ({ page }) => { diff --git a/e2e/pages/Contribute.spec.ts b/e2e/pages/Contribute.spec.ts index f9e3a5b7ed..86b734d70e 100644 --- a/e2e/pages/Contribute.spec.ts +++ b/e2e/pages/Contribute.spec.ts @@ -13,7 +13,7 @@ test.describe('Contribute Page', () => { }), }) }) - await page.goto('/contribute', { timeout: 25000 }) + await page.goto('/contribute') }) test('renders issue data correctly', async ({ page }) => { diff --git a/e2e/pages/Home.spec.ts b/e2e/pages/Home.spec.ts index e7afe65cab..25c5f95ead 100644 --- a/e2e/pages/Home.spec.ts +++ b/e2e/pages/Home.spec.ts @@ -2,7 +2,7 @@ import { test, expect } from '@playwright/test' test.describe('Home Page', () => { test.beforeEach(async ({ page }) => { - await page.goto('/', { timeout: 25000 }) + await page.goto('/') }) test('should have a heading', async ({ page }) => { diff --git a/e2e/pages/MentorshipPrograms.spec.ts b/e2e/pages/MentorshipPrograms.spec.ts index 0c5703ac8d..ba5db65157 100644 --- a/e2e/pages/MentorshipPrograms.spec.ts +++ b/e2e/pages/MentorshipPrograms.spec.ts @@ -21,7 +21,7 @@ test.describe('Mentorship Programs Page', () => { body: JSON.stringify({ hits: mockPrograms, nbPages: 1 }), }) }) - await page.goto('/mentorship/programs', { timeout: 25000 }) + await page.goto('/mentorship/programs') }) test('renders program card from mock data', async ({ page }) => { @@ -48,7 +48,7 @@ test.describe('Mentorship Programs Page', () => { body: JSON.stringify({ hits: [], nbPages: 0 }), }) }) - await page.goto('/mentorship/programs', { timeout: 25000 }) + await page.goto('/mentorship/programs') await expect(page.getByText('No programs found')).toBeVisible() }) diff --git a/e2e/pages/OrganizationDetails.spec.ts b/e2e/pages/OrganizationDetails.spec.ts index b8dad8255d..a86f477f2c 100644 --- a/e2e/pages/OrganizationDetails.spec.ts +++ b/e2e/pages/OrganizationDetails.spec.ts @@ -2,7 +2,7 @@ import { test, expect } from '@playwright/test' test.describe('Organization Details Page', () => { test.beforeEach(async ({ page }) => { - await page.goto('/organizations/OWASP', { timeout: 25000 }) + await page.goto('/organizations/OWASP') }) test('should have a heading', async ({ page }) => { diff --git a/e2e/pages/Organizations.spec.ts b/e2e/pages/Organizations.spec.ts index 5ef469da74..4e1a74daf8 100644 --- a/e2e/pages/Organizations.spec.ts +++ b/e2e/pages/Organizations.spec.ts @@ -13,7 +13,7 @@ test.describe('Organization Page', () => { }), }) }) - await page.goto('/organizations', { timeout: 25000 }) + await page.goto('/organizations') }) test('renders organization data correctly', async ({ page }) => { diff --git a/e2e/pages/ProjectDetails.spec.ts b/e2e/pages/ProjectDetails.spec.ts index b3a5725832..b3ba717536 100644 --- a/e2e/pages/ProjectDetails.spec.ts +++ b/e2e/pages/ProjectDetails.spec.ts @@ -2,7 +2,7 @@ import { test, expect } from '@playwright/test' test.describe('Project Details Page', () => { test.beforeEach(async ({ page }) => { - await page.goto('/projects/nest', { timeout: 25000 }) + await page.goto('/projects/nest') }) test('should have a heading', async ({ page }) => { diff --git a/e2e/pages/UserDetails.spec.ts b/e2e/pages/UserDetails.spec.ts index 3012ed54cd..f19efcced4 100644 --- a/e2e/pages/UserDetails.spec.ts +++ b/e2e/pages/UserDetails.spec.ts @@ -2,7 +2,7 @@ import { test, expect } from '@playwright/test' test.describe('User Details Page', () => { test.beforeEach(async ({ page }) => { - await page.goto('/members/arkid15r', { timeout: 25000 }) + await page.goto('/members/arkid15r') }) test('should have a heading and summary', async ({ page }) => { diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts index 904b069ece..a6c35a07a8 100644 --- a/e2e/playwright.config.ts +++ b/e2e/playwright.config.ts @@ -1,3 +1,4 @@ +import { iphone13Chromium } from '@e2e/helpers/devices' import os from 'node:os' import { defineConfig, devices } from '@playwright/test' @@ -8,19 +9,22 @@ export default defineConfig({ fullyParallel: true, projects: [ { - name: 'chromium', + name: 'Desktop Chrome', use: { ...devices['Desktop Chrome'], }, }, { - name: 'Mobile Safari - iPhone 13', - use: { - ...devices['iPhone 13'], - }, + name: 'Mobile Chrome - iPhone 13', + use: iphone13Chromium, }, ], - reporter: [['list', { printSteps: true }]], + reporter: process.env.CI + ? [ + ['list', { printSteps: true }], + ['github'], + ] + : [['list', { printSteps: true }]], retries: 2, testDir: '.', timeout: 120_000, @@ -29,5 +33,5 @@ export default defineConfig({ headless: true, trace: 'off', }, - workers: os.cpus().length, + workers: Math.max(1, os.cpus().length - 1), }) diff --git a/frontend/next.config.ts b/frontend/next.config.ts index 9cf3da04d9..14b93222d9 100644 --- a/frontend/next.config.ts +++ b/frontend/next.config.ts @@ -2,7 +2,10 @@ import { withSentryConfig } from '@sentry/nextjs' import type { NextConfig } from 'next' const forceStandalone = process.env.FORCE_STANDALONE === 'yes' +const isEnd2End = Boolean(process.env.NEXT_PUBLIC_E2E_BACKEND_BASE_URL) const isLocal = process.env.NEXT_PUBLIC_ENVIRONMENT === 'local' +const isProduction = process.env.NEXT_PUBLIC_ENVIRONMENT === 'production' +const isStaging = process.env.NEXT_PUBLIC_ENVIRONMENT === 'staging' const headers = [ { key: 'Cross-Origin-Embedder-Policy', value: 'credentialless' }, @@ -51,20 +54,18 @@ const nextConfig: NextConfig = { // https://nextjs.org/docs/app/api-reference/config/next-config-js/poweredByHeader poweredByHeader: false, // https://nextjs.org/docs/app/api-reference/config/next-config-js/productionBrowserSourceMaps - productionBrowserSourceMaps: true, + productionBrowserSourceMaps: (isStaging || isProduction) && !isEnd2End, serverExternalPackages: ['import-in-the-middle', 'require-in-the-middle'], transpilePackages: ['@react-leaflet/core', 'leaflet', 'react-leaflet', 'react-leaflet-cluster'], + ...(isEnd2End ? { skipTrailingSlashRedirect: true } : {}), rewrites: process.env.NEXT_PUBLIC_E2E_BACKEND_BASE_URL - ? async () => [ - { - source: '/csrf', - destination: `${process.env.NEXT_PUBLIC_E2E_BACKEND_BASE_URL}/csrf/`, - }, - { - source: '/graphql', - destination: `${process.env.NEXT_PUBLIC_E2E_BACKEND_BASE_URL}/graphql/`, - }, - ] + ? async () => { + const backendBase = process.env.NEXT_PUBLIC_E2E_BACKEND_BASE_URL + return [ + { source: '/csrf/', destination: `${backendBase}/csrf/` }, + { source: '/graphql/', destination: `${backendBase}/graphql/` }, + ] + } : undefined, ...(isLocal && !forceStandalone ? {} : { output: 'standalone' }), } diff --git a/infrastructure/live/README.md b/infrastructure/live/README.md index 35f30a16eb..6247244e86 100644 --- a/infrastructure/live/README.md +++ b/infrastructure/live/README.md @@ -51,9 +51,11 @@ No providers. | ---- | ------ | ------- | | [alb](#module\_alb) | ../modules/alb | n/a | | [backend](#module\_backend) | ../modules/service | n/a | +| [backend\_build\_cache](#module\_backend\_build\_cache) | ../modules/ecr-cache | n/a | | [cache](#module\_cache) | ../modules/cache | n/a | | [database](#module\_database) | ../modules/database | n/a | | [frontend](#module\_frontend) | ../modules/service | n/a | +| [frontend\_build\_cache](#module\_frontend\_build\_cache) | ../modules/ecr-cache | n/a | | [kms](#module\_kms) | ../modules/kms | n/a | | [networking](#module\_networking) | ../modules/networking | n/a | | [parameters](#module\_parameters) | ../modules/parameters | n/a | diff --git a/infrastructure/live/main.tf b/infrastructure/live/main.tf index 700e54e03f..1172d26472 100644 --- a/infrastructure/live/main.tf +++ b/infrastructure/live/main.tf @@ -124,6 +124,20 @@ module "frontend" { use_fargate_spot = var.frontend_use_fargate_spot } +module "backend_build_cache" { + source = "../modules/ecr-cache" + + common_tags = local.common_tags + name = "${var.project_name}-${var.environment}-backend-cache" +} + +module "frontend_build_cache" { + source = "../modules/ecr-cache" + + common_tags = local.common_tags + name = "${var.project_name}-${var.environment}-frontend-cache" +} + module "kms" { source = "../modules/kms" diff --git a/infrastructure/modules/ecr-cache/.terraform.lock.hcl b/infrastructure/modules/ecr-cache/.terraform.lock.hcl new file mode 100644 index 0000000000..432a39cf0a --- /dev/null +++ b/infrastructure/modules/ecr-cache/.terraform.lock.hcl @@ -0,0 +1,25 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.36.0" + constraints = "~> 6.36.0" + hashes = [ + "h1:UYt6mrz0d3PfTJRbJAxe+fLcJt7voXJCLLdwfHrApGk=", + "zh:0eb4481315564aaeec4905a804fd0df22c40f509ad2af63615eeaa90abacf81c", + "zh:12c3cddc461a8dbaa04387fe83420b64c4c05cb5479d181674168ca7daefcc38", + "zh:1b55a09661e80acf6826faa38dd8fbff24c2ef620d2a0a16918491a222c55370", + "zh:269cb1a406d0cac762bce82119247395a0bbf0d4ad2492fb2ea5653b4f44bc05", + "zh:3bfb78e3345f0c3846e76578952a09fb5dda05d2d73e19473fb0af0000469a66", + "zh:3ead4f4388c7dd78ed198082a981746324da0d7a51460c9b455fd884d86fc82c", + "zh:44906654199991b3f1a21c6a984bc5f9f556ff4baa4e5f77e168968e941c2725", + "zh:4803d050d581b05b0fd0ae5cce95ec1784d66e2bc9da4b1f7663df0ce7914609", + "zh:4cf9fe8fae58b62e83c0672a9c66e0963b7289aaf768a250e9bc44570d82cbd5", + "zh:5bfd7a1fb3116164b411777115dd4b272a68984fa949c687e41a3041318c82f1", + "zh:77cbcf2db512617f10b81e11c20d40fa534ef07163171cbe35214fa8f74b4e85", + "zh:8201cabed01f1434bf9ea7fbcf2a95612a87a0398b870b2643bd1a5119793d2d", + "zh:9aaded4cf36ec2abbe35086733a4510e08819698180b21a9387ba4112aee02e0", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:f594ef2683a0d23d3a6f0ad6c84a55ed79368c158ee08c2f3b7c41ec446a701f", + ] +} diff --git a/infrastructure/modules/ecr-cache/README.md b/infrastructure/modules/ecr-cache/README.md new file mode 100644 index 0000000000..485a4f5afc --- /dev/null +++ b/infrastructure/modules/ecr-cache/README.md @@ -0,0 +1,39 @@ + +## Requirements + +| Name | Version | +| ---- | ------- | +| [terraform](#requirement\_terraform) | ~> 1.14.0 | +| [aws](#requirement\_aws) | ~> 6.36.0 | + +## Providers + +| Name | Version | +| ---- | ------- | +| [aws](#provider\_aws) | 6.36.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +| ---- | ---- | +| [aws_ecr_lifecycle_policy.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecr_lifecycle_policy) | resource | +| [aws_ecr_repository.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecr_repository) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +| ---- | ----------- | ---- | ------- | :------: | +| [common\_tags](#input\_common\_tags) | Common tags to apply to all resources. | `map(string)` | `{}` | no | +| [name](#input\_name) | The ECR repository name for build cache manifests. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +| ---- | ----------- | +| [repository\_name](#output\_repository\_name) | The name of the ECR cache repository. | +| [repository\_url](#output\_repository\_url) | The URL of the ECR cache repository. | + diff --git a/infrastructure/modules/ecr-cache/main.tf b/infrastructure/modules/ecr-cache/main.tf new file mode 100644 index 0000000000..2eca89f8da --- /dev/null +++ b/infrastructure/modules/ecr-cache/main.tf @@ -0,0 +1,46 @@ +terraform { + required_version = "~> 1.14.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.36.0" + } + } +} + +# BuildKit registry cache reuses a single tag (e.g. :cache) and overwrites it each build. +# App release images use IMMUTABLE repos with scan-on-push in modules/service; this repo is cache-only. +# NOSEMGREP: terraform.aws.security.aws-ecr-mutable-image-tags.aws-ecr-mutable-image-tags, terraform.lang.security.ecr-image-scan-on-push.ecr-image-scan-on-push +resource "aws_ecr_repository" "main" { + image_tag_mutability = "MUTABLE" + name = var.name + + tags = merge(var.common_tags, { + Name = "${var.name}-ecr" + }) + + image_scanning_configuration { + scan_on_push = false + } +} + +resource "aws_ecr_lifecycle_policy" "main" { + policy = jsonencode({ + rules = [ + { + action = { + type = "expire" + } + description = "Keep the last 3 build cache images." + rulePriority = 1 + selection = { + countNumber = 3 + countType = "imageCountMoreThan" + tagStatus = "any" + } + } + ] + }) + repository = aws_ecr_repository.main.name +} diff --git a/infrastructure/modules/ecr-cache/outputs.tf b/infrastructure/modules/ecr-cache/outputs.tf new file mode 100644 index 0000000000..6b229f8ddf --- /dev/null +++ b/infrastructure/modules/ecr-cache/outputs.tf @@ -0,0 +1,9 @@ +output "repository_name" { + description = "The name of the ECR cache repository." + value = aws_ecr_repository.main.name +} + +output "repository_url" { + description = "The URL of the ECR cache repository." + value = aws_ecr_repository.main.repository_url +} diff --git a/infrastructure/modules/ecr-cache/tests/ecr-cache.tftest.hcl b/infrastructure/modules/ecr-cache/tests/ecr-cache.tftest.hcl new file mode 100644 index 0000000000..7f3e2761f7 --- /dev/null +++ b/infrastructure/modules/ecr-cache/tests/ecr-cache.tftest.hcl @@ -0,0 +1,45 @@ +mock_provider "aws" {} + +variables { + common_tags = { + Environment = "test" + Project = "nest" + } + name = "nest-test-backend-cache" +} + +run "test_repository_name" { + command = plan + + assert { + condition = aws_ecr_repository.main.name == var.name + error_message = "ECR cache repository name must match the configured name." + } +} + +run "test_repository_tag_mutability" { + command = plan + + assert { + condition = aws_ecr_repository.main.image_tag_mutability == "MUTABLE" + error_message = "ECR cache repository must allow mutable tags for build cache manifests." + } +} + +run "test_ecr_repository_scan_on_push_disabled" { + command = plan + + assert { + condition = aws_ecr_repository.main.image_scanning_configuration[0].scan_on_push == false + error_message = "ECR cache repository must not scan cache manifests on push." + } +} + +run "test_lifecycle_policy_retains_three_images" { + command = plan + + assert { + condition = jsondecode(aws_ecr_lifecycle_policy.main.policy).rules[0].selection.countNumber == 3 + error_message = "ECR cache lifecycle policy must retain the last 3 images." + } +} diff --git a/infrastructure/modules/ecr-cache/variables.tf b/infrastructure/modules/ecr-cache/variables.tf new file mode 100644 index 0000000000..7becf03cba --- /dev/null +++ b/infrastructure/modules/ecr-cache/variables.tf @@ -0,0 +1,10 @@ +variable "common_tags" { + description = "Common tags to apply to all resources." + type = map(string) + default = {} +} + +variable "name" { + description = "The ECR repository name for build cache manifests." + type = string +}