From 96dda2d6a52ccc737cbe7e787ec654ea9871a4a6 Mon Sep 17 00:00:00 2001
From: Christopher Harrop <Christopher.W.Harrop@noaa.gov>
Date: Thu, 21 May 2026 15:53:06 -0600
Subject: [PATCH 01/11] Parameterize ubuntu versions and improve ssh handling

---
 .env                                        |   2 +
 .github/workflows/docker.yml                | 608 +++-----------------
 .github/workflows/refresh-ubuntu-digest.yml |  67 ---
 docker-compose.yml                          | 143 ++---
 frontend/Dockerfile                         |   2 +-
 frontend/docker-entrypoint.sh               |  13 +-
 mom/Dockerfile                              |   6 +-
 server/Dockerfile                           |  84 ++-
 server/docker-entrypoint.sh                 |   6 -
 9 files changed, 194 insertions(+), 737 deletions(-)
 create mode 100644 .env
 delete mode 100644 .github/workflows/refresh-ubuntu-digest.yml

diff --git a/.env b/.env
new file mode 100644
index 0000000..a7421a0
--- /dev/null
+++ b/.env
@@ -0,0 +1,2 @@
+UBUNTU_BASE=ubuntu:26.04
+IMAGE_TAG=latest
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 4282328..fd5a078 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -9,24 +9,27 @@ on:
 
 env:
   OPENPBS_VERSION: 23.06.06
-  UBUNTU_BASE_TAG: 26.04
-  REGISTRY_SERVER_IMAGE: ghcr.io/noaa-gsl/dockerpbscluster/pbs-server
-  REGISTRY_FRONTEND_IMAGE: ghcr.io/noaa-gsl/dockerpbscluster/pbs-frontend
-  REGISTRY_MOM_IMAGE: ghcr.io/noaa-gsl/dockerpbscluster/pbs-mom
+  REGISTRY: ghcr.io/noaa-gsl/dockerpbscluster
+  LATEST_UBUNTU: '26.04'
 
 jobs:
 
   docker_compose_test_amd64:
-    name: Docker Compose Test (amd64)
-    runs-on: ubuntu2204-4c-16g-150ssd
-
+    name: Compose Test (ubuntu-${{ matrix.ubuntu }}, amd64)
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ubuntu: &ubuntu_versions
+          - '24.04'
+          - '26.04'
     steps:
       -
         name: Checkout Repository
         uses: actions/checkout@v4
       -
         name: Build and start containers
-        run: docker compose -f docker-compose.yml up --build -d
+        run: UBUNTU_BASE=ubuntu:${{ matrix.ubuntu }} docker compose -f docker-compose.yml up -d --build --pull never --wait
 
       -
         name: Check cluster logs
@@ -59,7 +62,7 @@ jobs:
         name: Run a PBS job
         run: |
           docker exec pbs-frontend bash -lc "echo hostname | qsub"
-          sleep 15
+          sleep 5
           docker exec pbs-server bash -lc "qstat -x"
 
       -
@@ -74,16 +77,19 @@ jobs:
         run: docker compose -f docker-compose.yml down
 
   docker_compose_test_arm64:
-    name: Docker Compose Test (arm64)
+    name: Compose Test (ubuntu-${{ matrix.ubuntu }}, arm64)
     runs-on: ubuntu-24.04-arm
-
+    strategy:
+      fail-fast: false
+      matrix:
+        ubuntu: *ubuntu_versions
     steps:
       -
         name: Checkout Repository
         uses: actions/checkout@v4
       -
         name: Build and start containers
-        run: docker compose -f docker-compose.yml up --build -d
+        run: UBUNTU_BASE=ubuntu:${{ matrix.ubuntu }} docker compose -f docker-compose.yml up -d --build --pull never --wait
 
       -
         name: Check cluster logs
@@ -130,503 +136,28 @@ jobs:
         name: Shut down PBS cluster containers
         run: docker compose -f docker-compose.yml down
 
-  resolve_versions:
-    name: Resolve Version Tags
-    runs-on: ubuntu-latest
-    outputs:
-      ubuntu_version: ${{ steps.resolve.outputs.ubuntu_version }}
-    steps:
-      -
-        name: Resolve concrete ubuntu base version
-        id: resolve
-        run: |
-          UBUNTU_VERSION=$(docker run --rm ubuntu:${{ env.UBUNTU_BASE_TAG }} bash -lc '. /etc/os-release && echo "$VERSION_ID"')
-          echo "ubuntu_version=${UBUNTU_VERSION}" >> "$GITHUB_OUTPUT"
-          echo "Resolved ubuntu:${{ env.UBUNTU_BASE_TAG }} to VERSION_ID=${UBUNTU_VERSION}"
-
-  build-server-arm64:
-    runs-on: ubuntu-24.04-arm
-    needs:
-      - docker_compose_test_amd64
-      - docker_compose_test_arm64
-      - resolve_versions
-    timeout-minutes: 360
-    permissions:
-      packages: write
-      contents: read
-      id-token: write
-    steps:
-      -
-        name: Checkout repository
-        uses: actions/checkout@v4
-      -
-        name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY_SERVER_IMAGE }}
-          tags: |
-            type=raw,value=latest
-            type=raw,value=ubuntu-${{ needs.resolve_versions.outputs.ubuntu_version }}-openpbs-${{ env.OPENPBS_VERSION }}
-          flavor: |
-            latest=true
-            prefix=
-            suffix=
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      -
-        name: Login to GHCR
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-          logout: false
-      -
-        name: Prune pre-loaded GHA docker images
-        run: |
-          docker images
-          docker image prune -a -f
-          docker images
-      -
-        name: Build and push by digest
-        id: build
-        uses: docker/build-push-action@v5
-        with:
-          context: ./server
-          file: ./server/Dockerfile
-          platforms: linux/arm64
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=registry,ref=ghcr.io/noaa-gsl/dockerpbscluster/server-cache-arm64:cache
-          cache-to: type=registry,ref=ghcr.io/noaa-gsl/dockerpbscluster/server-cache-arm64:cache,mode=max
-          outputs: type=image,name=${{ env.REGISTRY_SERVER_IMAGE }},push-by-digest=true,name-canonical=true,push=true
-      -
-        name: Export digest
-        run: |
-          mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-      -
-        name: Upload digest
-        uses: actions/upload-artifact@v4
-        with:
-          name: server-digests-linux-arm64
-          path: /tmp/digests/*
-          if-no-files-found: error
-          retention-days: 1
-
-  build-server-amd64:
-    runs-on: ubuntu-latest
-    needs:
-      - docker_compose_test_amd64
-      - docker_compose_test_arm64
-      - resolve_versions
-    timeout-minutes: 360
-    permissions:
-      packages: write
-      contents: read
-      id-token: write
-    steps:
-      -
-        name: Checkout repository
-        uses: actions/checkout@v4
-      -
-        name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY_SERVER_IMAGE }}
-          tags: |
-            type=raw,value=latest
-            type=raw,value=ubuntu-${{ needs.resolve_versions.outputs.ubuntu_version }}-openpbs-${{ env.OPENPBS_VERSION }}
-          flavor: |
-            latest=true
-            prefix=
-            suffix=
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      -
-        name: Login to GHCR
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-          logout: false
-      -
-        name: Prune pre-loaded GHA docker images
-        run: |
-          docker images
-          docker image prune -a -f
-          docker images
-      -
-        name: Build and push by digest
-        id: build
-        uses: docker/build-push-action@v5
-        with:
-          context: ./server
-          file: ./server/Dockerfile
-          platforms: linux/amd64
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=registry,ref=ghcr.io/noaa-gsl/dockerpbscluster/server-cache-amd64:cache
-          cache-to: type=registry,ref=ghcr.io/noaa-gsl/dockerpbscluster/server-cache-amd64:cache,mode=max
-          outputs: type=image,name=${{ env.REGISTRY_SERVER_IMAGE }},push-by-digest=true,name-canonical=true,push=true
-      -
-        name: Export digest
-        run: |
-          mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-      -
-        name: Upload digest
-        uses: actions/upload-artifact@v4
-        with:
-          name: server-digests-linux-amd64
-          path: /tmp/digests/*
-          if-no-files-found: error
-          retention-days: 1
-
-  merge-server:
-    runs-on: ubuntu-latest
-    needs:
-      - build-server-amd64
-      - build-server-arm64
-      - resolve_versions
-    permissions:
-      packages: write
-      contents: read
-      id-token: write
-    steps:
-      -
-        name: Checkout repository
-        uses: actions/checkout@v4
-      -
-        name: Download digests
-        uses: actions/download-artifact@v4
-        with:
-          path: /tmp/digests
-          pattern: server-digests-*
-          merge-multiple: true
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      -
-        name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY_SERVER_IMAGE }}
-          tags: |
-            type=raw,value=latest
-            type=raw,value=ubuntu-${{ needs.resolve_versions.outputs.ubuntu_version }}-openpbs-${{ env.OPENPBS_VERSION }}
-          flavor: |
-            latest=true
-            prefix=
-            suffix=
-      -
-        name: Login to GHCR
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-          logout: false
-      -
-        name: Create manifest list and push
-        working-directory: /tmp/digests
-        run: |
-          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-            $(printf '${{ env.REGISTRY_SERVER_IMAGE }}@sha256:%s ' *)
-      -
-        name: Inspect image
-        run: |
-          docker buildx imagetools inspect ${{ env.REGISTRY_SERVER_IMAGE }}:${{ steps.meta.outputs.version }}
-
-  build-mom-arm64:
-    runs-on: ubuntu-24.04-arm
-    needs:
-      - docker_compose_test_amd64
-      - docker_compose_test_arm64
-      - resolve_versions
-    timeout-minutes: 360
-    permissions:
-      packages: write
-      contents: read
-      id-token: write
-    steps:
-      -
-        name: Checkout repository
-        uses: actions/checkout@v4
-      -
-        name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY_MOM_IMAGE }}
-          tags: |
-            type=raw,value=latest
-            type=raw,value=ubuntu-${{ needs.resolve_versions.outputs.ubuntu_version }}-openpbs-${{ env.OPENPBS_VERSION }}
-          flavor: |
-            latest=true
-            prefix=
-            suffix=
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      -
-        name: Login to GHCR
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-          logout: false
-      -
-        name: Prune pre-loaded GHA docker images
-        run: |
-          docker images
-          docker image prune -a -f
-          docker images
-      -
-        name: Build and push by digest
-        id: build
-        uses: docker/build-push-action@v5
-        with:
-          context: ./mom
-          file: ./mom/Dockerfile
-          platforms: linux/arm64
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=registry,ref=ghcr.io/noaa-gsl/dockerpbscluster/mom-cache-arm64:cache
-          cache-to: type=registry,ref=ghcr.io/noaa-gsl/dockerpbscluster/mom-cache-arm64:cache,mode=max
-          outputs: type=image,name=${{ env.REGISTRY_MOM_IMAGE }},push-by-digest=true,name-canonical=true,push=true
-      -
-        name: Export digest
-        run: |
-          mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-      -
-        name: Upload digest
-        uses: actions/upload-artifact@v4
-        with:
-          name: mom-digests-linux-arm64
-          path: /tmp/digests/*
-          if-no-files-found: error
-          retention-days: 1
-
-  build-mom-amd64:
-    runs-on: ubuntu-latest
-    needs:
-      - docker_compose_test_amd64
-      - docker_compose_test_arm64
-      - resolve_versions
-    timeout-minutes: 360
-    permissions:
-      packages: write
-      contents: read
-      id-token: write
-    steps:
-      -
-        name: Checkout repository
-        uses: actions/checkout@v4
-      -
-        name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY_MOM_IMAGE }}
-          tags: |
-            type=raw,value=latest
-            type=raw,value=ubuntu-${{ needs.resolve_versions.outputs.ubuntu_version }}-openpbs-${{ env.OPENPBS_VERSION }}
-          flavor: |
-            latest=true
-            prefix=
-            suffix=
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      -
-        name: Login to GHCR
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-          logout: false
-      -
-        name: Prune pre-loaded GHA docker images
-        run: |
-          docker images
-          docker image prune -a -f
-          docker images
-      -
-        name: Build and push by digest
-        id: build
-        uses: docker/build-push-action@v5
-        with:
-          context: ./mom
-          file: ./mom/Dockerfile
-          platforms: linux/amd64
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=registry,ref=ghcr.io/noaa-gsl/dockerpbscluster/mom-cache-amd64:cache
-          cache-to: type=registry,ref=ghcr.io/noaa-gsl/dockerpbscluster/mom-cache-amd64:cache,mode=max
-          outputs: type=image,name=${{ env.REGISTRY_MOM_IMAGE }},push-by-digest=true,name-canonical=true,push=true
-      -
-        name: Export digest
-        run: |
-          mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-      -
-        name: Upload digest
-        uses: actions/upload-artifact@v4
-        with:
-          name: mom-digests-linux-amd64
-          path: /tmp/digests/*
-          if-no-files-found: error
-          retention-days: 1
-
-  merge-mom:
-    runs-on: ubuntu-latest
-    needs:
-      - build-mom-amd64
-      - build-mom-arm64
-      - resolve_versions
-    permissions:
-      packages: write
-      contents: read
-      id-token: write
-    steps:
-      -
-        name: Checkout repository
-        uses: actions/checkout@v4
-      -
-        name: Download digests
-        uses: actions/download-artifact@v4
-        with:
-          path: /tmp/digests
-          pattern: mom-digests-*
-          merge-multiple: true
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      -
-        name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY_MOM_IMAGE }}
-          tags: |
-            type=raw,value=latest
-            type=raw,value=ubuntu-${{ needs.resolve_versions.outputs.ubuntu_version }}-openpbs-${{ env.OPENPBS_VERSION }}
-          flavor: |
-            latest=true
-            prefix=
-            suffix=
-      -
-        name: Login to GHCR
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-          logout: false
-      -
-        name: Create manifest list and push
-        working-directory: /tmp/digests
-        run: |
-          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-            $(printf '${{ env.REGISTRY_MOM_IMAGE }}@sha256:%s ' *)
-      -
-        name: Inspect image
-        run: |
-          docker buildx imagetools inspect ${{ env.REGISTRY_MOM_IMAGE }}:${{ steps.meta.outputs.version }}
-
-  build-frontend-arm64:
-    runs-on: ubuntu-24.04-arm
-    needs:
-      - docker_compose_test_amd64
-      - docker_compose_test_arm64
-      - resolve_versions
-    timeout-minutes: 360
-    permissions:
-      packages: write
-      contents: read
-      id-token: write
-    steps:
-      -
-        name: Checkout repository
-        uses: actions/checkout@v4
-      -
-        name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY_FRONTEND_IMAGE }}
-          tags: |
-            type=raw,value=latest
-            type=raw,value=ubuntu-${{ needs.resolve_versions.outputs.ubuntu_version }}-openpbs-${{ env.OPENPBS_VERSION }}
-          flavor: |
-            latest=true
-            prefix=
-            suffix=
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      -
-        name: Login to GHCR
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-          logout: false
-      -
-        name: Prune pre-loaded GHA docker images
-        run: |
-          docker images
-          docker image prune -a -f
-          docker images
-      -
-        name: Build and push by digest
-        id: build
-        uses: docker/build-push-action@v5
-        with:
-          context: ./frontend
-          file: ./frontend/Dockerfile
-          platforms: linux/arm64
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=registry,ref=ghcr.io/noaa-gsl/dockerpbscluster/frontend-cache-arm64:cache
-          cache-to: type=registry,ref=ghcr.io/noaa-gsl/dockerpbscluster/frontend-cache-arm64:cache,mode=max
-          outputs: type=image,name=${{ env.REGISTRY_FRONTEND_IMAGE }},push-by-digest=true,name-canonical=true,push=true
-      -
-        name: Export digest
-        run: |
-          mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-      -
-        name: Upload digest
-        uses: actions/upload-artifact@v4
-        with:
-          name: frontend-digests-linux-arm64
-          path: /tmp/digests/*
-          if-no-files-found: error
-          retention-days: 1
-
-  build-frontend-amd64:
-    runs-on: ubuntu-latest
-    needs:
-      - docker_compose_test_amd64
-      - docker_compose_test_arm64
-      - resolve_versions
+  build:
+    name: Build ${{ matrix.container }} (ubuntu-${{ matrix.ubuntu }}, ${{ matrix.arch }})
+    runs-on: ${{ matrix.runner }}
+    needs: [docker_compose_test_amd64, docker_compose_test_arm64]
     timeout-minutes: 360
     permissions:
       packages: write
       contents: read
       id-token: write
+    strategy:
+      fail-fast: false
+      matrix:
+        container: [frontend, server, mom]
+        ubuntu: *ubuntu_versions
+        include:
+          - arch: amd64
+            runner: ubuntu-latest
+            platform: linux/amd64
+          - arch: arm64
+            runner: ubuntu-24.04-arm
+            platform: linux/arm64
+        arch: [amd64, arm64]
     steps:
       -
         name: Checkout repository
@@ -636,14 +167,12 @@ jobs:
         id: meta
         uses: docker/metadata-action@v5
         with:
-          images: ${{ env.REGISTRY_FRONTEND_IMAGE }}
+          images: ${{ env.REGISTRY }}/pbs-${{ matrix.container }}
           tags: |
-            type=raw,value=latest
-            type=raw,value=ubuntu-${{ needs.resolve_versions.outputs.ubuntu_version }}-openpbs-${{ env.OPENPBS_VERSION }}
+            type=raw,value=ubuntu-${{ matrix.ubuntu }}-openpbs-${{ env.OPENPBS_VERSION }}
+            ${{ matrix.ubuntu == env.LATEST_UBUNTU && 'type=raw,value=latest' || '' }}
           flavor: |
-            latest=true
-            prefix=
-            suffix=
+            latest=false
       -
         name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -666,13 +195,14 @@ jobs:
         id: build
         uses: docker/build-push-action@v5
         with:
-          context: ./frontend
-          file: ./frontend/Dockerfile
-          platforms: linux/amd64
+          context: ./${{ matrix.container }}
+          file: ./${{ matrix.container }}/Dockerfile
+          build-args: UBUNTU_BASE=ubuntu:${{ matrix.ubuntu }}
+          platforms: ${{ matrix.platform }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=registry,ref=ghcr.io/noaa-gsl/dockerpbscluster/frontend-cache-amd64:cache
-          cache-to: type=registry,ref=ghcr.io/noaa-gsl/dockerpbscluster/frontend-cache-amd64:cache,mode=max
-          outputs: type=image,name=${{ env.REGISTRY_FRONTEND_IMAGE }},push-by-digest=true,name-canonical=true,push=true
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ matrix.container }}-cache-${{ matrix.ubuntu }}-${{ matrix.arch }}:cache
+          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ matrix.container }}-cache-${{ matrix.ubuntu }}-${{ matrix.arch }}:cache,mode=max
+          outputs: type=image,name=${{ env.REGISTRY }}/pbs-${{ matrix.container }},push-by-digest=true,name-canonical=true,push=true
       -
         name: Export digest
         run: |
@@ -683,21 +213,24 @@ jobs:
         name: Upload digest
         uses: actions/upload-artifact@v4
         with:
-          name: frontend-digests-linux-amd64
+          name: ${{ matrix.container }}-${{ matrix.ubuntu }}-digests-${{ matrix.arch }}
           path: /tmp/digests/*
           if-no-files-found: error
           retention-days: 1
 
-  merge-frontend:
+  merge:
+    name: Merge ${{ matrix.container }} (ubuntu-${{ matrix.ubuntu }})
     runs-on: ubuntu-latest
-    needs:
-      - build-frontend-amd64
-      - build-frontend-arm64
-      - resolve_versions
+    needs: build
     permissions:
       packages: write
       contents: read
       id-token: write
+    strategy:
+      fail-fast: false
+      matrix:
+        container: [frontend, server, mom]
+        ubuntu: *ubuntu_versions
     steps:
       -
         name: Checkout repository
@@ -707,7 +240,7 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           path: /tmp/digests
-          pattern: frontend-digests-*
+          pattern: ${{ matrix.container }}-${{ matrix.ubuntu }}-digests-*
           merge-multiple: true
       -
         name: Set up Docker Buildx
@@ -717,14 +250,12 @@ jobs:
         id: meta
         uses: docker/metadata-action@v5
         with:
-          images: ${{ env.REGISTRY_FRONTEND_IMAGE }}
+          images: ${{ env.REGISTRY }}/pbs-${{ matrix.container }}
           tags: |
-            type=raw,value=latest
-            type=raw,value=ubuntu-${{ needs.resolve_versions.outputs.ubuntu_version }}-openpbs-${{ env.OPENPBS_VERSION }}
+            type=raw,value=ubuntu-${{ matrix.ubuntu }}-openpbs-${{ env.OPENPBS_VERSION }}
+            ${{ matrix.ubuntu == env.LATEST_UBUNTU && 'type=raw,value=latest' || '' }}
           flavor: |
-            latest=true
-            prefix=
-            suffix=
+            latest=false
       -
         name: Login to GHCR
         uses: docker/login-action@v3
@@ -738,29 +269,24 @@ jobs:
         working-directory: /tmp/digests
         run: |
           docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-            $(printf '${{ env.REGISTRY_FRONTEND_IMAGE }}@sha256:%s ' *)
+            $(printf '${{ env.REGISTRY }}/pbs-${{ matrix.container }}@sha256:%s ' *)
       -
         name: Inspect image
         run: |
-          docker buildx imagetools inspect ${{ env.REGISTRY_FRONTEND_IMAGE }}:${{ steps.meta.outputs.version }}
+          docker buildx imagetools inspect ${{ env.REGISTRY }}/pbs-${{ matrix.container }}:${{ steps.meta.outputs.version }}
 
   scan-images:
-    name: Scan images for fixable High/Critical CVEs
+    name: Scan ${{ matrix.container }} (ubuntu-${{ matrix.ubuntu }})
     runs-on: ubuntu-latest
-    needs:
-      - merge-server
-      - merge-mom
-      - merge-frontend
+    needs: merge
     permissions:
       packages: read
       contents: read
     strategy:
       fail-fast: false
       matrix:
-        image:
-          - ghcr.io/noaa-gsl/dockerpbscluster/pbs-server:latest
-          - ghcr.io/noaa-gsl/dockerpbscluster/pbs-frontend:latest
-          - ghcr.io/noaa-gsl/dockerpbscluster/pbs-mom:latest
+        container: [frontend, server, mom]
+        ubuntu: *ubuntu_versions
     steps:
       -
         name: Checkout repository
@@ -776,7 +302,7 @@ jobs:
         name: Run Trivy scan
         uses: aquasecurity/trivy-action@v0.36.0
         with:
-          image-ref: ${{ matrix.image }}
+          image-ref: ${{ env.REGISTRY }}/pbs-${{ matrix.container }}:ubuntu-${{ matrix.ubuntu }}-openpbs-${{ env.OPENPBS_VERSION }}
           format: table
           vuln-type: os,library
           severity: HIGH,CRITICAL
diff --git a/.github/workflows/refresh-ubuntu-digest.yml b/.github/workflows/refresh-ubuntu-digest.yml
deleted file mode 100644
index a1e94bd..0000000
--- a/.github/workflows/refresh-ubuntu-digest.yml
+++ /dev/null
@@ -1,67 +0,0 @@
-name: Refresh Ubuntu Base Digest
-
-on:
-  schedule:
-    - cron: '0 8 * * 1'
-  workflow_dispatch:
-
-permissions:
-  contents: write
-  pull-requests: write
-
-jobs:
-  refresh-ubuntu-digest:
-    runs-on: ubuntu-latest
-    steps:
-      -
-        name: Checkout repository
-        uses: actions/checkout@v4
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      -
-        name: Resolve latest digest for ubuntu:26.04
-        id: resolve
-        run: |
-          set -euo pipefail
-          DIGEST=$(docker buildx imagetools inspect ubuntu:26.04 --format '{{.Digest}}')
-          if [[ -z "${DIGEST}" ]]; then
-            echo "Unable to resolve digest for ubuntu:26.04"
-            exit 1
-          fi
-          FULL_REF="ubuntu:26.04@${DIGEST}"
-          echo "full_ref=${FULL_REF}" >> "$GITHUB_OUTPUT"
-      -
-        name: Update pinned digest in Dockerfiles and README
-        id: update
-        run: |
-          set -euo pipefail
-          FULL_REF='${{ steps.resolve.outputs.full_ref }}'
-
-          for dockerfile in frontend/Dockerfile mom/Dockerfile server/Dockerfile; do
-            sed -i "s|^ARG UBUNTU_BASE=.*$|ARG UBUNTU_BASE=${FULL_REF}|g" "${dockerfile}"
-          done
-
-          sed -i "s|^Pinned Ubuntu base image: .*|Pinned Ubuntu base image: ${FULL_REF}|g" README.md
-
-          if git diff --quiet; then
-            echo "changed=false" >> "$GITHUB_OUTPUT"
-          else
-            echo "changed=true" >> "$GITHUB_OUTPUT"
-          fi
-      -
-        name: Create pull request
-        if: steps.update.outputs.changed == 'true'
-        uses: peter-evans/create-pull-request@v6
-        with:
-          branch: chore/refresh-ubuntu-base-digest
-          commit-message: "chore: refresh ubuntu 26.04 base digest"
-          title: "chore: refresh ubuntu 26.04 base digest"
-          body: |
-            Automated update of the pinned ubuntu:26.04 digest used by all Dockerfiles.
-
-            - Updates ARG UBUNTU_BASE in frontend/mom/server Dockerfiles
-            - Updates documentation in README.md
-          labels: |
-            dependencies
-            security
diff --git a/docker-compose.yml b/docker-compose.yml
index c7c6519..52e7608 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,76 +1,79 @@
+x-node-common: &node-common
+  image: ghcr.io/noaa-gsl/dockerpbscluster/pbs-mom:${IMAGE_TAG}
+  shm_size: '4g'
+  user: admin
+  volumes:
+    - shared-vol:/home/admin
+  depends_on:
+    pbsserver:
+      condition: service_started
+    pbsfrontend:
+      condition: service_healthy
+  healthcheck:
+    test: ["CMD", "ssh", "-o", "StrictHostKeyChecking=no", "admin@localhost", "hostname"]
+    interval: 10s
+    timeout: 5s
+    retries: 5
+
 services:
-  pbsserver:
-        build:
-          context: ./server
-          dockerfile: Dockerfile
-        image: ghcr.io/noaa-gsl/dockerpbscluster/pbs-server:latest
-        container_name: pbs-server
-        shm_size: '4g'
-        hostname: pbsserver
-        user: admin
-        volumes:
-                - shared-vol:/home/admin
-        ports:
-                - 15001:15001
-                - 15003:15003
-                - 17001:17001
   pbsfrontend:
-        build:
-          context: ./frontend
-          dockerfile: Dockerfile
-        image: ghcr.io/noaa-gsl/dockerpbscluster/pbs-frontend:latest
-        container_name: pbs-frontend
-        shm_size: '4g'
-        hostname: pbsfrontend
-        user: admin
-        volumes:
-                - shared-vol:/home/admin:nocopy
-        links:
-                - pbsserver
+    build:
+      context: ./frontend
+      args:
+        UBUNTU_BASE: ${UBUNTU_BASE}
+    image: ghcr.io/noaa-gsl/dockerpbscluster/pbs-frontend:${IMAGE_TAG}
+    container_name: pbs-frontend
+    shm_size: '4g'
+    hostname: pbsfrontend
+    user: admin
+    volumes:
+      - shared-vol:/home/admin
+    healthcheck:
+      test: ["CMD", "test", "-f", "/tmp/ssh-ready"]
+      interval: 2s
+      timeout: 2s
+      retries: 5
+      start_period: 5s
+  pbsserver:
+    build:
+      context: ./server
+      args:
+        UBUNTU_BASE: ${UBUNTU_BASE}
+    image: ghcr.io/noaa-gsl/dockerpbscluster/pbs-server:${IMAGE_TAG}
+    container_name: pbs-server
+    shm_size: '4g'
+    hostname: pbsserver
+    user: admin
+    volumes:
+      - shared-vol:/home/admin
+    ports:
+      - 15001:15001
+      - 15003:15003
+      - 17001:17001
+    depends_on:
+      pbsfrontend:
+        condition: service_healthy
   pbsnode1:
-        build:
-          context: ./mom
-          dockerfile: Dockerfile
-        image: ghcr.io/noaa-gsl/dockerpbscluster/pbs-mom:latest
-        container_name: pbs-node1
-        shm_size: '4g'
-        hostname: pbsnode1
-        user: admin
-        volumes:
-                - shared-vol:/home/admin:nocopy
-        environment:
-                - PBS_NODENAME=pbsnode1
-        links:
-                - pbsserver
+    <<: *node-common
+    build:
+      context: ./mom
+      args:
+        UBUNTU_BASE: ${UBUNTU_BASE}
+    container_name: pbs-node1
+    hostname: pbsnode1
+    environment:
+      - PBS_NODENAME=pbsnode1
   pbsnode2:
-        build:
-          context: ./mom
-          dockerfile: Dockerfile
-        image: ghcr.io/noaa-gsl/dockerpbscluster/pbs-mom:latest
-        container_name: pbs-node2
-        shm_size: '4g'
-        hostname: pbsnode2
-        user: admin
-        volumes:
-                - shared-vol:/home/admin:nocopy
-        environment:
-                - PBS_NODENAME=pbsnode2
-        links:
-                - pbsserver
+    <<: *node-common
+    container_name: pbs-node2
+    hostname: pbsnode2
+    environment:
+      - PBS_NODENAME=pbsnode2
   pbsnode3:
-        build:
-          context: ./mom
-          dockerfile: Dockerfile
-        image: ghcr.io/noaa-gsl/dockerpbscluster/pbs-mom:latest
-        container_name: pbs-node3
-        shm_size: '4g'
-        hostname: pbsnode3
-        user: admin
-        volumes:
-                - shared-vol:/home/admin:nocopy
-        environment:
-                - PBS_NODENAME=pbsnode3
-        links:
-                - pbsserver
+    <<: *node-common
+    container_name: pbs-node3
+    hostname: pbsnode3
+    environment:
+      - PBS_NODENAME=pbsnode3
 volumes:
-        shared-vol:
+  shared-vol:
diff --git a/frontend/Dockerfile b/frontend/Dockerfile
index eb3c730..df0f0d4 100644
--- a/frontend/Dockerfile
+++ b/frontend/Dockerfile
@@ -1,4 +1,4 @@
-ARG UBUNTU_BASE=ubuntu:26.04@sha256:f3d28607ddd78734bb7f71f117f3c6706c666b8b76cbff7c9ff6e5718d46ff64
+ARG UBUNTU_BASE=ubuntu:26.04
 FROM ${UBUNTU_BASE}
 
 ARG OPENPBS_VERSION=23.06.06
diff --git a/frontend/docker-entrypoint.sh b/frontend/docker-entrypoint.sh
index 7949984..4cb5013 100644
--- a/frontend/docker-entrypoint.sh
+++ b/frontend/docker-entrypoint.sh
@@ -8,8 +8,15 @@ sudo ssh-keygen -A
 # Start SSH
 sudo service ssh start
 
-# Generate SSH keys for admin user
-ssh-keygen -t rsa -f /home/admin/.ssh/id_rsa -N ""
-cp /home/admin/.ssh/id_rsa.pub /home/admin/.ssh/authorized_keys
+# Generate SSH key and set up shared authorized_keys
+if [ ! -f /home/admin/.ssh/id_rsa ]; then
+    ssh-keygen -t rsa -f /home/admin/.ssh/id_rsa -N ""
+    cp /home/admin/.ssh/id_rsa.pub /home/admin/.ssh/authorized_keys
+    chown -R admin:admin /home/admin/.ssh
+    chmod 700 /home/admin/.ssh
+    chmod 600 /home/admin/.ssh/authorized_keys
+fi
+
+touch /tmp/ssh-ready
 
 tail -f /dev/null
diff --git a/mom/Dockerfile b/mom/Dockerfile
index e2f61cd..6e4a650 100644
--- a/mom/Dockerfile
+++ b/mom/Dockerfile
@@ -1,4 +1,4 @@
-ARG UBUNTU_BASE=ubuntu:26.04@sha256:f3d28607ddd78734bb7f71f117f3c6706c666b8b76cbff7c9ff6e5718d46ff64
+ARG UBUNTU_BASE=ubuntu:26.04
 FROM ${UBUNTU_BASE}
 
 ARG OPENPBS_VERSION=23.06.06
@@ -82,10 +82,6 @@ RUN chmod 4755 /opt/pbs/sbin/pbs_iff \
 COPY docker-entrypoint.sh /opt/pbs/docker-entrypoint.sh
 RUN chmod +x /opt/pbs/docker-entrypoint.sh
 
-COPY ssh /home/admin/.ssh
-RUN chown -R admin:admin /home/admin/.ssh \
- && chmod -R 700 /home/admin/.ssh
-
 WORKDIR /home/admin
 
 EXPOSE 22 15002
diff --git a/server/Dockerfile b/server/Dockerfile
index 3e7875c..7e073d3 100644
--- a/server/Dockerfile
+++ b/server/Dockerfile
@@ -1,47 +1,48 @@
-ARG UBUNTU_BASE=ubuntu:26.04@sha256:f3d28607ddd78734bb7f71f117f3c6706c666b8b76cbff7c9ff6e5718d46ff64
+ARG UBUNTU_BASE=ubuntu:26.04
 FROM ${UBUNTU_BASE}
 
 ARG OPENPBS_VERSION=23.06.06
 ARG DEBIAN_FRONTEND=noninteractive
 
-# Pin PostgreSQL 16: ubuntu 26.04 ships pg18, which OpenPBS 23.x does not
-# support.  Add the official PGDG repo so we can install a compatible version.
-RUN apt-get update -y && apt-get install -y wget ca-certificates gnupg \
+
+# Add the official PGDG repo for the detected Ubuntu codename and always install PostgreSQL 15
+RUN apt-get update -y && apt-get install -y wget ca-certificates gnupg lsb-release \
  && install -d /usr/share/keyrings \
  && wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc \
-    | gpg --dearmor -o /usr/share/keyrings/postgresql-keyring.gpg \
- && echo "deb [signed-by=/usr/share/keyrings/postgresql-keyring.gpg] https://apt.postgresql.org/pub/repos/apt resolute-pgdg main" \
-    > /etc/apt/sources.list.d/pgdg.list \
+      | gpg --dearmor -o /usr/share/keyrings/postgresql-keyring.gpg \
+ && UBUNTU_CODENAME=$(lsb_release -cs) \
+ && echo "deb [signed-by=/usr/share/keyrings/postgresql-keyring.gpg] https://apt.postgresql.org/pub/repos/apt $UBUNTU_CODENAME-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
  && rm -rf /var/lib/apt/lists/*
 
-# Install dependencies
-RUN apt-get update -y && apt-get install -y \
-    gcc \
-    g++ \
-    make \
-    autoconf \
-    automake \
-    libtool \
-    git \
-    wget \
-    vim \
-    sudo \
-    openssh-server \
-    python3 \
-    python3-dev \
-    libpq-dev \
-    postgresql-16 \
-    libical-dev \
-    libedit-dev \
-    libhwloc-dev \
-    libssl-dev \
-    swig \
-    pkg-config \
-    expat \
-    libexpat1-dev \
-    tcl-dev \
-    tk-dev \
-    libxt-dev \
+# Install dependencies, always using PostgreSQL 15
+RUN apt-get update -y \
+ && apt-get install -y \
+         gcc \
+         g++ \
+         make \
+         autoconf \
+         automake \
+         libtool \
+         git \
+         wget \
+         vim \
+         sudo \
+         openssh-server \
+         python3 \
+         python3-dev \
+         libpq-dev \
+         postgresql-15 \
+         libical-dev \
+         libedit-dev \
+         libhwloc-dev \
+         libssl-dev \
+         swig \
+         pkg-config \
+         expat \
+         libexpat1-dev \
+         tcl-dev \
+         tk-dev \
+         libxt-dev \
  && rm -rf /var/lib/apt/lists/* \
  && rm -f /etc/ssh/ssh_host_* \
  && rm -f /etc/ssl/private/ssl-cert-snakeoil.key \
@@ -67,12 +68,11 @@ RUN useradd -r -m -d /var/lib/pbsdata -s /bin/sh -G postgres pbsdata
 # Fix the db_user file: pbs_postinstall writes "postgres" but we use pbsdata
 RUN echo -n pbsdata > /var/spool/pbs/server_priv/db_user
 
-# Symlink PostgreSQL 16 server binaries into /usr/bin so pbs_db_utility can
-# find them via PATH under sudo.  Hardcode the pg16 path so this is not
-# affected by whichever pg_config libpq-dev exposes.
+# Symlink PostgreSQL 15 server binaries into /usr/bin so pbs_db_utility can
+# find them via PATH under sudo.
 RUN for bin in initdb pg_ctl createdb psql pg_config; do \
-        ln -sf /usr/lib/postgresql/16/bin/${bin} /usr/bin/${bin}; \
-    done
+            ln -sf /usr/lib/postgresql/15/bin/${bin} /usr/bin/${bin}; \
+      done
 
 # Configure PBS as server
 RUN echo "PBS_EXEC=/opt/pbs"           >  /etc/pbs.conf \
@@ -107,10 +107,6 @@ RUN chmod 4755 /opt/pbs/sbin/pbs_iff \
 COPY docker-entrypoint.sh /opt/pbs/docker-entrypoint.sh
 RUN chmod +x /opt/pbs/docker-entrypoint.sh
 
-COPY ssh /home/admin/.ssh
-RUN chown -R admin:admin /home/admin/.ssh \
- && chmod -R 700 /home/admin/.ssh
-
 WORKDIR /home/admin
 
 EXPOSE 22 15001 15003 17001
diff --git a/server/docker-entrypoint.sh b/server/docker-entrypoint.sh
index aed13b9..5ceb1cd 100644
--- a/server/docker-entrypoint.sh
+++ b/server/docker-entrypoint.sh
@@ -64,10 +64,4 @@ sudo ssh-keygen -A
 # Start SSH
 sudo service ssh start
 
-# Generate SSH keys for admin user
-if [ ! -f /home/admin/.ssh/id_rsa ]; then
-    ssh-keygen -t rsa -f /home/admin/.ssh/id_rsa -N ""
-    cp /home/admin/.ssh/id_rsa.pub /home/admin/.ssh/authorized_keys
-fi
-
 tail -f /dev/null

From 961ef1f75460c26377a772daadfa46f7a291464d Mon Sep 17 00:00:00 2001
From: Christopher Harrop <Christopher.W.Harrop@noaa.gov>
Date: Thu, 21 May 2026 16:10:56 -0600
Subject: [PATCH 02/11] Add package cleanup CI job

---
 .github/workflows/package-cleanup.yaml | 88 ++++++++++++++++++++++++++
 1 file changed, 88 insertions(+)
 create mode 100644 .github/workflows/package-cleanup.yaml

diff --git a/.github/workflows/package-cleanup.yaml b/.github/workflows/package-cleanup.yaml
new file mode 100644
index 0000000..b5b2f46
--- /dev/null
+++ b/.github/workflows/package-cleanup.yaml
@@ -0,0 +1,88 @@
+name: PackageCleanup
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch:
+
+jobs:
+  cleanup-packages:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      -
+        name: Remove untagged versions of dockerpbscluster/pbs-frontend
+        uses: actions/delete-package-versions@v5
+        with: 
+          package-name: 'dockerpbscluster/pbs-frontend'
+          package-type: 'container'
+          min-versions-to-keep: 0
+          delete-only-untagged-versions: 'true'
+      -
+        name: Remove untagged versions of dockerpbscluster/pbs-server
+        uses: actions/delete-package-versions@v5
+        with: 
+          package-name: 'dockerpbscluster/pbs-server'
+          package-type: 'container'
+          min-versions-to-keep: 0
+          delete-only-untagged-versions: 'true'
+      -
+        name: Remove untagged versions of dockerpbscluster/pbs-mom
+        uses: actions/delete-package-versions@v5
+        with: 
+          package-name: 'dockerpbscluster/pbs-mom'
+          package-type: 'container'
+          min-versions-to-keep: 0
+          delete-only-untagged-versions: 'true'
+      -
+        name: Remove untagged versions of dockerpbscluster/frontend-cache-amd64
+        uses: actions/delete-package-versions@v5
+        with: 
+          package-name: 'dockerpbscluster/frontend-cache-amd64'
+          package-type: 'container'
+          min-versions-to-keep: 0
+          delete-only-untagged-versions: 'true'
+      -
+        name: Remove untagged versions of dockerpbscluster/frontend-cache-arm64
+        uses: actions/delete-package-versions@v5
+        with: 
+          package-name: 'dockerpbscluster/frontend-cache-arm64'
+          package-type: 'container'
+          min-versions-to-keep: 0
+          delete-only-untagged-versions: 'true'
+      -
+        name: Remove untagged versions of dockerpbscluster/server-cache-amd64
+        uses: actions/delete-package-versions@v5
+        with: 
+          package-name: 'dockerpbscluster/server-cache-amd64'
+          package-type: 'container'
+          min-versions-to-keep: 0
+          delete-only-untagged-versions: 'true'
+      -
+        name: Remove untagged versions of dockerpbscluster/server-cache-arm64
+        uses: actions/delete-package-versions@v5
+        with: 
+          package-name: 'dockerpbscluster/server-cache-arm64'
+          package-type: 'container'
+          min-versions-to-keep: 0
+          delete-only-untagged-versions: 'true'
+      -
+        name: Remove untagged versions of dockerpbscluster/mom-cache-amd64
+        uses: actions/delete-package-versions@v5
+        with: 
+          package-name: 'dockerpbscluster/mom-cache-amd64'
+          package-type: 'container'
+          min-versions-to-keep: 0
+          delete-only-untagged-versions: 'true'
+      -
+        name: Remove untagged versions of dockerpbscluster/mom-cache-arm64
+        uses: actions/delete-package-versions@v5
+        with: 
+          package-name: 'dockerpbscluster/mom-cache-arm64'
+          package-type: 'container'
+          min-versions-to-keep: 0
+          delete-only-untagged-versions: 'true'

From 66eb72b3598ea78fa548d9760c7a2cfe23cd57f1 Mon Sep 17 00:00:00 2001
From: Christopher Harrop <Christopher.W.Harrop@noaa.gov>
Date: Thu, 21 May 2026 16:35:24 -0600
Subject: [PATCH 03/11] Make sure postgres is ready before trying to initialize
 pbs datastore

---
 server/docker-entrypoint.sh | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/server/docker-entrypoint.sh b/server/docker-entrypoint.sh
index 5ceb1cd..39f40d4 100644
--- a/server/docker-entrypoint.sh
+++ b/server/docker-entrypoint.sh
@@ -2,6 +2,14 @@
 
 export PATH=/opt/pbs/bin:/opt/pbs/sbin:$PATH
 
+
+# Wait for PostgreSQL to be ready
+echo "Waiting for PostgreSQL to be ready..."
+for i in $(seq 1 30); do
+    pg_isready -U pbsdata -h /var/run/postgresql -p 15007 && break
+    sleep 1
+done
+
 # Initialize the PBS PostgreSQL datastore
 echo "Initializing PBS datastore..."
 sudo /opt/pbs/libexec/pbs_db_utility install_db

From 72855df4237464770f84bac568de81283bdf0226 Mon Sep 17 00:00:00 2001
From: Christopher Harrop <Christopher.W.Harrop@noaa.gov>
Date: Fri, 22 May 2026 09:34:15 -0600
Subject: [PATCH 04/11] Add debug steps and fix bug in pbs server wait loop

---
 .github/workflows/docker.yml | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index fd5a078..b4f5e7e 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -44,7 +44,8 @@ jobs:
         run: |
           echo "Waiting for PBS nodes to become free..."
           for i in $(seq 1 60); do
-            free=$(docker exec pbs-server bash -lc "pbsnodes -a 2>/dev/null | grep -c 'state = free'" 2>/dev/null || echo 0)
+            # free=$(docker exec pbs-server bash -lc "pbsnodes -a 2>/dev/null | grep -c 'state = free'" 2>/dev/null || echo 0)
+            free=$(docker exec pbs-server bash -lc "pbsnodes -a 2>/dev/null | grep -c 'state = free' || echo 0" 2>/dev/null)
             echo "Attempt $i: $free nodes free"
             [ "$free" -ge 3 ] && echo "All 3 nodes ready" && break
             sleep 5
@@ -75,6 +76,14 @@ jobs:
       -
         name: Shut down PBS cluster containers
         run: docker compose -f docker-compose.yml down
+      -
+        name: Debug session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
+        timeout-minutes: 60
+        with:
+          limit-access-to-actor: true
+
 
   docker_compose_test_arm64:
     name: Compose Test (ubuntu-${{ matrix.ubuntu }}, arm64)
@@ -104,7 +113,8 @@ jobs:
         run: |
           echo "Waiting for PBS nodes to become free..."
           for i in $(seq 1 60); do
-            free=$(docker exec pbs-server bash -lc "pbsnodes -a 2>/dev/null | grep -c 'state = free'" 2>/dev/null || echo 0)
+            # free=$(docker exec pbs-server bash -lc "pbsnodes -a 2>/dev/null | grep -c 'state = free'" 2>/dev/null || echo 0)
+            free=$(docker exec pbs-server bash -lc "pbsnodes -a 2>/dev/null | grep -c 'state = free' || echo 0" 2>/dev/null)
             echo "Attempt $i: $free nodes free"
             [ "$free" -ge 3 ] && echo "All 3 nodes ready" && break
             sleep 5
@@ -135,6 +145,13 @@ jobs:
       -
         name: Shut down PBS cluster containers
         run: docker compose -f docker-compose.yml down
+      -
+        name: Debug session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
+        timeout-minutes: 60
+        with:
+          limit-access-to-actor: true
 
   build:
     name: Build ${{ matrix.container }} (ubuntu-${{ matrix.ubuntu }}, ${{ matrix.arch }})

From 5f51393d36e7324d24ea8707bd33bf2b4a563c00 Mon Sep 17 00:00:00 2001
From: christopherwharrop <christopher.w.harrop@gmail.com>
Date: Fri, 22 May 2026 11:54:31 -0600
Subject: [PATCH 05/11] Harden the container startup steps

---
 .github/workflows/docker.yml |   4 ++
 docker-compose.yml           |   6 ++
 server/docker-entrypoint.sh  | 122 ++++++++++++++++++++++++++++-------
 3 files changed, 108 insertions(+), 24 deletions(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index b4f5e7e..3dee20c 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -33,10 +33,12 @@ jobs:
 
       -
         name: Check cluster logs
+        if: ${{ always() }}
         run: docker compose -f docker-compose.yml logs
 
       -
         name: Check status of the cluster containers
+        if: ${{ always() }}
         run: docker compose -f docker-compose.yml ps
 
       -
@@ -102,10 +104,12 @@ jobs:
 
       -
         name: Check cluster logs
+        if: ${{ always() }}
         run: docker compose -f docker-compose.yml logs
 
       -
         name: Check status of the cluster containers
+        if: ${{ always() }}
         run: docker compose -f docker-compose.yml ps
 
       -
diff --git a/docker-compose.yml b/docker-compose.yml
index 52e7608..c622f8f 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -53,6 +53,12 @@ services:
     depends_on:
       pbsfrontend:
         condition: service_healthy
+    healthcheck:
+      test: ["CMD", "test", "-f", "/tmp/pbs-server-ready"]
+      interval: 5s
+      timeout: 3s
+      retries: 60
+      start_period: 10s
   pbsnode1:
     <<: *node-common
     build:
diff --git a/server/docker-entrypoint.sh b/server/docker-entrypoint.sh
index 39f40d4..13c5259 100644
--- a/server/docker-entrypoint.sh
+++ b/server/docker-entrypoint.sh
@@ -1,38 +1,104 @@
 #!/bin/bash
 
+set -e
+set -o pipefail
+
 export PATH=/opt/pbs/bin:/opt/pbs/sbin:$PATH
 
+dump_diagnostics() {
+    echo "================== PBS SERVER FAILURE DIAGNOSTICS =================="
+    echo "--- uname / lsb-release ---"
+    uname -a || true
+    cat /etc/os-release 2>/dev/null || true
+    echo "--- /etc/pbs.conf ---"
+    sudo cat /etc/pbs.conf 2>&1 || true
+    echo "--- processes ---"
+    sudo ps -efw 2>&1 || true
+    echo "--- postgres / pbs_ processes ---"
+    sudo pgrep -af 'postgres|pbs_' 2>&1 || echo "(none)"
+    echo "--- listening sockets ---"
+    sudo ss -lntp 2>&1 || sudo netstat -lntp 2>&1 || true
+    echo "--- /var/spool/pbs ---"
+    sudo ls -la /var/spool/pbs/ 2>&1 || true
+    echo "--- /var/spool/pbs/server_priv ---"
+    sudo ls -la /var/spool/pbs/server_priv/ 2>&1 || true
+    echo "--- /var/spool/pbs/server_priv/db_* ---"
+    sudo sh -c 'for f in /var/spool/pbs/server_priv/db_*; do echo "=== $f ==="; cat "$f" 2>&1 || true; done' || true
+    echo "--- /var/spool/pbs/datastore ---"
+    sudo ls -la /var/spool/pbs/datastore/ 2>&1 || true
+    echo "--- PBS data service log (pg_log) ---"
+    sudo sh -c 'tail -n 200 /var/spool/pbs/datastore/pg_log/* 2>&1' || echo "(no pg_log)"
+    echo "--- PBS server_logs ---"
+    sudo sh -c 'tail -n 200 /var/spool/pbs/server_logs/* 2>&1' || echo "(no server_logs)"
+    echo "--- PBS comm_logs ---"
+    sudo sh -c 'tail -n 100 /var/spool/pbs/comm_logs/* 2>&1' || echo "(no comm_logs)"
+    echo "--- PBS sched_logs ---"
+    sudo sh -c 'tail -n 100 /var/spool/pbs/sched_logs/* 2>&1' || echo "(no sched_logs)"
+    echo "===================================================================="
+}
 
-# Wait for PostgreSQL to be ready
-echo "Waiting for PostgreSQL to be ready..."
-for i in $(seq 1 30); do
-    pg_isready -U pbsdata -h /var/run/postgresql -p 15007 && break
-    sleep 1
-done
+on_error() {
+    rc=$?
+    echo "ERROR: docker-entrypoint.sh failed (exit code $rc) at line ${1:-?}"
+    dump_diagnostics
+    echo "Keeping container alive for inspection (docker exec / tmate)."
+    # Don't exit -- keep PID 1 alive so the container doesn't restart and lose state.
+    exec tail -f /dev/null
+}
+trap 'on_error $LINENO' ERR
 
-# Initialize the PBS PostgreSQL datastore
-echo "Initializing PBS datastore..."
+# Initialize the PBS PostgreSQL datastore.
+# This is responsible for initdb, creating the pbsdata role+db, applying the
+# schema, and starting the PBS data service (postgres on port 15007).
+echo "Initializing PBS datastore (pbs_db_utility install_db)..."
 sudo /opt/pbs/libexec/pbs_db_utility install_db
+echo "install_db returned $?"
+
+# Verify the PBS data service is actually accepting connections.
+echo "Waiting for PBS data service on port 15007..."
+for i in $(seq 1 60); do
+    if pg_isready -U pbsdata -h /var/run/postgresql -p 15007 >/dev/null 2>&1; then
+        echo "PBS data service is ready (after ${i}s)."
+        break
+    fi
+    sleep 1
+    if [ "$i" -eq 60 ]; then
+        echo "ERROR: PBS data service did not come up within 60s after install_db."
+        exit 1
+    fi
+done
 
 # Start pbs_comm first (pbs_server requires it)
+echo "Starting pbs_comm..."
 sudo /opt/pbs/sbin/pbs_comm
 
-# Start PBS server with -t create (creates schema if needed, then runs as daemon)
-echo "Starting PBS server..."
+# Start PBS server with -t create (creates schema if needed, then daemonizes)
+echo "Starting pbs_server -t create..."
 sudo /opt/pbs/sbin/pbs_server -t create
 
-# Start scheduler
-sudo /opt/pbs/sbin/pbs_sched
-
-# Wait for PBS server to be ready
-echo "Waiting for PBS server..."
-for i in $(seq 1 30); do
-    sudo /opt/pbs/bin/qstat -Bf > /dev/null 2>&1 && break
+# Verify pbs_server is actually alive and responding
+echo "Waiting for pbs_server to accept requests..."
+for i in $(seq 1 60); do
+    if sudo /opt/pbs/bin/qstat -Bf >/dev/null 2>&1; then
+        echo "pbs_server is responding (after ${i}s)."
+        break
+    fi
     sleep 1
+    if [ "$i" -eq 60 ]; then
+        echo "ERROR: pbs_server did not become responsive within 60s."
+        sudo pgrep -af pbs_server || echo "(no pbs_server process running)"
+        exit 1
+    fi
 done
 
-# Configure server (|| true so restarts don't fail on "already exists")
-sudo /opt/pbs/bin/qmgr -c "create queue workq queue_type=execution" 2>/dev/null || true
+# Start scheduler
+echo "Starting pbs_sched..."
+sudo /opt/pbs/sbin/pbs_sched
+
+# Configure server. These are idempotent-on-restart; some commands are
+# expected to "fail" (e.g. queue already exists), so don't trip the ERR trap.
+set +e
+sudo /opt/pbs/bin/qmgr -c "create queue workq queue_type=execution" 2>/dev/null
 sudo /opt/pbs/bin/qmgr -c "set queue workq enabled = True"
 sudo /opt/pbs/bin/qmgr -c "set queue workq started = True"
 sudo /opt/pbs/bin/qmgr -c "set server default_queue = workq"
@@ -40,10 +106,10 @@ sudo /opt/pbs/bin/qmgr -c "set server scheduling = True"
 sudo /opt/pbs/bin/qmgr -c "set server flatuid = True"
 sudo /opt/pbs/bin/qmgr -c "set server job_history_enable = True"
 
-# Register compute nodes
-sudo /opt/pbs/bin/qmgr -c "create node pbsnode1" 2>/dev/null || true
-sudo /opt/pbs/bin/qmgr -c "create node pbsnode2" 2>/dev/null || true
-sudo /opt/pbs/bin/qmgr -c "create node pbsnode3" 2>/dev/null || true
+sudo /opt/pbs/bin/qmgr -c "create node pbsnode1" 2>/dev/null
+sudo /opt/pbs/bin/qmgr -c "create node pbsnode2" 2>/dev/null
+sudo /opt/pbs/bin/qmgr -c "create node pbsnode3" 2>/dev/null
+set -e
 
 # Create a queuejob hook that defaults output/error paths to the submission
 # directory (PBS_O_WORKDIR) instead of $HOME on the submission host.
@@ -62,9 +128,11 @@ except Exception as ex:
     pbs.logmsg(pbs.LOG_DEBUG, 'default_output_dir: ' + str(ex))
 e.accept()
 PYEOF
-sudo /opt/pbs/bin/qmgr -c "create hook default_output_dir" 2>/dev/null || true
+set +e
+sudo /opt/pbs/bin/qmgr -c "create hook default_output_dir" 2>/dev/null
 sudo /opt/pbs/bin/qmgr -c "set hook default_output_dir event = queuejob"
 sudo /opt/pbs/bin/qmgr -c "import hook default_output_dir application/x-python default /tmp/default_output_dir.py"
+set -e
 
 # Regenerate SSH host keys (keys are not baked into the image)
 sudo ssh-keygen -A
@@ -72,4 +140,10 @@ sudo ssh-keygen -A
 # Start SSH
 sudo service ssh start
 
+# Marker file used by the docker-compose healthcheck so `up --wait` blocks
+# until the server is actually ready (not just until the container started).
+sudo touch /tmp/pbs-server-ready
+
+echo "PBS server startup completed successfully."
+
 tail -f /dev/null

From 2cb803e1ae01ee927e8d8a39eca9c3fe5f5c78ac Mon Sep 17 00:00:00 2001
From: christopherwharrop <christopher.w.harrop@gmail.com>
Date: Fri, 22 May 2026 12:27:10 -0600
Subject: [PATCH 06/11] Add diagnostics and deal with missing socket directory

---
 server/docker-entrypoint.sh | 58 +++++++++++++++++++++++--------------
 1 file changed, 37 insertions(+), 21 deletions(-)

diff --git a/server/docker-entrypoint.sh b/server/docker-entrypoint.sh
index 13c5259..0a08d19 100644
--- a/server/docker-entrypoint.sh
+++ b/server/docker-entrypoint.sh
@@ -7,7 +7,7 @@ export PATH=/opt/pbs/bin:/opt/pbs/sbin:$PATH
 
 dump_diagnostics() {
     echo "================== PBS SERVER FAILURE DIAGNOSTICS =================="
-    echo "--- uname / lsb-release ---"
+    echo "--- uname / os-release ---"
     uname -a || true
     cat /etc/os-release 2>/dev/null || true
     echo "--- /etc/pbs.conf ---"
@@ -18,6 +18,10 @@ dump_diagnostics() {
     sudo pgrep -af 'postgres|pbs_' 2>&1 || echo "(none)"
     echo "--- listening sockets ---"
     sudo ss -lntp 2>&1 || sudo netstat -lntp 2>&1 || true
+    echo "--- /var/run/postgresql ---"
+    sudo ls -la /var/run/postgresql/ 2>&1 || echo "(missing)"
+    echo "--- /run ---"
+    sudo ls -la /run/ 2>&1 || true
     echo "--- /var/spool/pbs ---"
     sudo ls -la /var/spool/pbs/ 2>&1 || true
     echo "--- /var/spool/pbs/server_priv ---"
@@ -25,7 +29,7 @@ dump_diagnostics() {
     echo "--- /var/spool/pbs/server_priv/db_* ---"
     sudo sh -c 'for f in /var/spool/pbs/server_priv/db_*; do echo "=== $f ==="; cat "$f" 2>&1 || true; done' || true
     echo "--- /var/spool/pbs/datastore ---"
-    sudo ls -la /var/spool/pbs/datastore/ 2>&1 || true
+    sudo ls -la /var/spool/pbs/datastore/ 2>&1 || echo "(missing)"
     echo "--- PBS data service log (pg_log) ---"
     sudo sh -c 'tail -n 200 /var/spool/pbs/datastore/pg_log/* 2>&1' || echo "(no pg_log)"
     echo "--- PBS server_logs ---"
@@ -42,31 +46,42 @@ on_error() {
     echo "ERROR: docker-entrypoint.sh failed (exit code $rc) at line ${1:-?}"
     dump_diagnostics
     echo "Keeping container alive for inspection (docker exec / tmate)."
-    # Don't exit -- keep PID 1 alive so the container doesn't restart and lose state.
     exec tail -f /dev/null
 }
 trap 'on_error $LINENO' ERR
 
+# Ensure the PostgreSQL runtime socket directory exists with the right perms.
+# In Docker there is no systemd-tmpfiles to create /run/postgresql, and PBS's
+# data service uses /var/run/postgresql as its unix socket dir. Without this,
+# `pg_db_utility install_db` can fail with:
+#   createdb: connection to server on socket "/var/run/postgresql/.s.PGSQL.15007"
+#   failed: No such file or directory
+sudo install -d -o postgres -g postgres -m 2775 /var/run/postgresql
+
 # Initialize the PBS PostgreSQL datastore.
-# This is responsible for initdb, creating the pbsdata role+db, applying the
-# schema, and starting the PBS data service (postgres on port 15007).
+# install_db has a known race condition between starting postgres and running
+# createdb: pbs_dataservice's status poll can return success before postgres
+# has actually bound its unix socket, which causes createdb to fail. Retry a
+# few times. If it still fails, continue anyway: `pbs_server -t create` is
+# able to bootstrap the datastore from scratch and is the source of truth
+# for whether startup succeeded.
 echo "Initializing PBS datastore (pbs_db_utility install_db)..."
-sudo /opt/pbs/libexec/pbs_db_utility install_db
-echo "install_db returned $?"
-
-# Verify the PBS data service is actually accepting connections.
-echo "Waiting for PBS data service on port 15007..."
-for i in $(seq 1 60); do
-    if pg_isready -U pbsdata -h /var/run/postgresql -p 15007 >/dev/null 2>&1; then
-        echo "PBS data service is ready (after ${i}s)."
+install_db_rc=1
+for attempt in 1 2 3; do
+    set +e
+    sudo /opt/pbs/libexec/pbs_db_utility install_db
+    install_db_rc=$?
+    set -e
+    if [ "$install_db_rc" -eq 0 ]; then
+        echo "install_db succeeded on attempt $attempt."
         break
     fi
-    sleep 1
-    if [ "$i" -eq 60 ]; then
-        echo "ERROR: PBS data service did not come up within 60s after install_db."
-        exit 1
-    fi
+    echo "install_db attempt $attempt failed (rc=$install_db_rc); retrying in 5s..."
+    sleep 5
 done
+if [ "$install_db_rc" -ne 0 ]; then
+    echo "WARN: install_db failed after 3 attempts; will rely on pbs_server -t create to bootstrap."
+fi
 
 # Start pbs_comm first (pbs_server requires it)
 echo "Starting pbs_comm..."
@@ -76,7 +91,8 @@ sudo /opt/pbs/sbin/pbs_comm
 echo "Starting pbs_server -t create..."
 sudo /opt/pbs/sbin/pbs_server -t create
 
-# Verify pbs_server is actually alive and responding
+# Verify pbs_server is actually alive and responding. This is the real gate:
+# if pbs_server can't talk to its datastore, qstat will keep failing.
 echo "Waiting for pbs_server to accept requests..."
 for i in $(seq 1 60); do
     if sudo /opt/pbs/bin/qstat -Bf >/dev/null 2>&1; then
@@ -95,8 +111,8 @@ done
 echo "Starting pbs_sched..."
 sudo /opt/pbs/sbin/pbs_sched
 
-# Configure server. These are idempotent-on-restart; some commands are
-# expected to "fail" (e.g. queue already exists), so don't trip the ERR trap.
+# Configure server. These are idempotent-on-restart; some commands intentionally
+# error if a queue/node/hook already exists, so don't trip the ERR trap.
 set +e
 sudo /opt/pbs/bin/qmgr -c "create queue workq queue_type=execution" 2>/dev/null
 sudo /opt/pbs/bin/qmgr -c "set queue workq enabled = True"

From d69cae097337eea6418db2d247dc0cf5f987f569 Mon Sep 17 00:00:00 2001
From: christopherwharrop <christopher.w.harrop@gmail.com>
Date: Fri, 22 May 2026 12:45:49 -0600
Subject: [PATCH 07/11] Fix retry loop for startup failures

---
 server/docker-entrypoint.sh | 76 ++++++++++++++++++++-----------------
 1 file changed, 42 insertions(+), 34 deletions(-)

diff --git a/server/docker-entrypoint.sh b/server/docker-entrypoint.sh
index 0a08d19..def9ed4 100644
--- a/server/docker-entrypoint.sh
+++ b/server/docker-entrypoint.sh
@@ -59,28 +59,38 @@ trap 'on_error $LINENO' ERR
 sudo install -d -o postgres -g postgres -m 2775 /var/run/postgresql
 
 # Initialize the PBS PostgreSQL datastore.
-# install_db has a known race condition between starting postgres and running
-# createdb: pbs_dataservice's status poll can return success before postgres
-# has actually bound its unix socket, which causes createdb to fail. Retry a
-# few times. If it still fails, continue anyway: `pbs_server -t create` is
-# able to bootstrap the datastore from scratch and is the source of truth
-# for whether startup succeeded.
+# install_db has a known race condition: pbs_dataservice considers postgres
+# "started" as soon as the process is up, but createdb can hit "the database
+# system is starting up" if it connects before postgres finishes WAL recovery.
+# We retry the whole install_db invocation; each retry runs initdb from
+# scratch, so the race window is re-rolled. If we still fail after a few
+# attempts, we continue anyway: `pbs_server -t create` is able to bootstrap
+# the datastore itself and is the real source of truth for startup success.
+#
+# Calling install_db inside an `if` (or via `|| true`) is essential: the
+# `trap ... ERR` above fires on *any* simple-command non-zero exit, even
+# under `set +e`. Only the conditional context suppresses it.
 echo "Initializing PBS datastore (pbs_db_utility install_db)..."
 install_db_rc=1
-for attempt in 1 2 3; do
-    set +e
-    sudo /opt/pbs/libexec/pbs_db_utility install_db
-    install_db_rc=$?
-    set -e
-    if [ "$install_db_rc" -eq 0 ]; then
+for attempt in 1 2 3 4 5; do
+    # Make sure no stale data-service state from a failed previous attempt is
+    # left around: pbs_dataservice's "cleanup" doesn't always tear down the
+    # pbs_ds_monitor process, which can confuse the next attempt.
+    sudo pkill -9 -f 'pbs_ds_monitor' 2>/dev/null || true
+    sudo pkill -9 -f 'postgres.*-D /var/spool/pbs/datastore' 2>/dev/null || true
+    sudo rm -rf /var/spool/pbs/datastore 2>/dev/null || true
+
+    if sudo /opt/pbs/libexec/pbs_db_utility install_db; then
+        install_db_rc=0
         echo "install_db succeeded on attempt $attempt."
         break
     fi
-    echo "install_db attempt $attempt failed (rc=$install_db_rc); retrying in 5s..."
-    sleep 5
+    install_db_rc=$?
+    echo "install_db attempt $attempt failed (rc=$install_db_rc); retrying in 10s..."
+    sleep 10
 done
 if [ "$install_db_rc" -ne 0 ]; then
-    echo "WARN: install_db failed after 3 attempts; will rely on pbs_server -t create to bootstrap."
+    echo "WARN: install_db failed after 5 attempts; will rely on pbs_server -t create to bootstrap."
 fi
 
 # Start pbs_comm first (pbs_server requires it)
@@ -112,20 +122,20 @@ echo "Starting pbs_sched..."
 sudo /opt/pbs/sbin/pbs_sched
 
 # Configure server. These are idempotent-on-restart; some commands intentionally
-# error if a queue/node/hook already exists, so don't trip the ERR trap.
-set +e
-sudo /opt/pbs/bin/qmgr -c "create queue workq queue_type=execution" 2>/dev/null
-sudo /opt/pbs/bin/qmgr -c "set queue workq enabled = True"
-sudo /opt/pbs/bin/qmgr -c "set queue workq started = True"
-sudo /opt/pbs/bin/qmgr -c "set server default_queue = workq"
-sudo /opt/pbs/bin/qmgr -c "set server scheduling = True"
-sudo /opt/pbs/bin/qmgr -c "set server flatuid = True"
-sudo /opt/pbs/bin/qmgr -c "set server job_history_enable = True"
-
-sudo /opt/pbs/bin/qmgr -c "create node pbsnode1" 2>/dev/null
-sudo /opt/pbs/bin/qmgr -c "create node pbsnode2" 2>/dev/null
-sudo /opt/pbs/bin/qmgr -c "create node pbsnode3" 2>/dev/null
-set -e
+# error if a queue/node/hook already exists. The ERR trap fires on *any*
+# non-zero exit (not gated by `set -e`), so each command must be `|| true`'d
+# rather than relying on set +e.
+sudo /opt/pbs/bin/qmgr -c "create queue workq queue_type=execution" 2>/dev/null || true
+sudo /opt/pbs/bin/qmgr -c "set queue workq enabled = True" || true
+sudo /opt/pbs/bin/qmgr -c "set queue workq started = True" || true
+sudo /opt/pbs/bin/qmgr -c "set server default_queue = workq" || true
+sudo /opt/pbs/bin/qmgr -c "set server scheduling = True" || true
+sudo /opt/pbs/bin/qmgr -c "set server flatuid = True" || true
+sudo /opt/pbs/bin/qmgr -c "set server job_history_enable = True" || true
+
+sudo /opt/pbs/bin/qmgr -c "create node pbsnode1" 2>/dev/null || true
+sudo /opt/pbs/bin/qmgr -c "create node pbsnode2" 2>/dev/null || true
+sudo /opt/pbs/bin/qmgr -c "create node pbsnode3" 2>/dev/null || true
 
 # Create a queuejob hook that defaults output/error paths to the submission
 # directory (PBS_O_WORKDIR) instead of $HOME on the submission host.
@@ -144,11 +154,9 @@ except Exception as ex:
     pbs.logmsg(pbs.LOG_DEBUG, 'default_output_dir: ' + str(ex))
 e.accept()
 PYEOF
-set +e
-sudo /opt/pbs/bin/qmgr -c "create hook default_output_dir" 2>/dev/null
-sudo /opt/pbs/bin/qmgr -c "set hook default_output_dir event = queuejob"
-sudo /opt/pbs/bin/qmgr -c "import hook default_output_dir application/x-python default /tmp/default_output_dir.py"
-set -e
+sudo /opt/pbs/bin/qmgr -c "create hook default_output_dir" 2>/dev/null || true
+sudo /opt/pbs/bin/qmgr -c "set hook default_output_dir event = queuejob" || true
+sudo /opt/pbs/bin/qmgr -c "import hook default_output_dir application/x-python default /tmp/default_output_dir.py" || true
 
 # Regenerate SSH host keys (keys are not baked into the image)
 sudo ssh-keygen -A

From b14b6d995b3ab51617492b7c55e9fc9fa44675e8 Mon Sep 17 00:00:00 2001
From: christopherwharrop <christopher.w.harrop@gmail.com>
Date: Fri, 22 May 2026 13:11:12 -0600
Subject: [PATCH 08/11] Fix extraneous output in datastore retry loop

---
 server/docker-entrypoint.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/server/docker-entrypoint.sh b/server/docker-entrypoint.sh
index def9ed4..d96d973 100644
--- a/server/docker-entrypoint.sh
+++ b/server/docker-entrypoint.sh
@@ -76,8 +76,10 @@ for attempt in 1 2 3 4 5; do
     # Make sure no stale data-service state from a failed previous attempt is
     # left around: pbs_dataservice's "cleanup" doesn't always tear down the
     # pbs_ds_monitor process, which can confuse the next attempt.
-    sudo pkill -9 -f 'pbs_ds_monitor' 2>/dev/null || true
-    sudo pkill -9 -f 'postgres.*-D /var/spool/pbs/datastore' 2>/dev/null || true
+    # Match by process name only (not full cmdline) so pkill doesn't match
+    # its own sudo wrapper.
+    sudo pkill -9 -x pbs_ds_monitor 2>/dev/null || true
+    sudo pkill -9 -x postgres 2>/dev/null || true
     sudo rm -rf /var/spool/pbs/datastore 2>/dev/null || true
 
     if sudo /opt/pbs/libexec/pbs_db_utility install_db; then

From 87cfc246eae7720271786e613c6b93cac564e547 Mon Sep 17 00:00:00 2001
From: christopherwharrop <christopher.w.harrop@gmail.com>
Date: Fri, 22 May 2026 14:04:09 -0600
Subject: [PATCH 09/11] Patch PBS db_utiity to wait for postgres to accept
 connections

---
 server/Dockerfile           | 16 ++++++++++++++++
 server/docker-entrypoint.sh |  9 ++++++---
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/server/Dockerfile b/server/Dockerfile
index 7e073d3..f3cf05b 100644
--- a/server/Dockerfile
+++ b/server/Dockerfile
@@ -62,6 +62,22 @@ RUN git clone --branch v${OPENPBS_VERSION} --depth 1 https://github.com/openpbs/
  && /opt/pbs/libexec/pbs_postinstall \
  && rm -rf /tmp/openpbs
 
+# Fix race in pbs_db_utility install_db: the upstream script polls
+# `pbs_dataservice status`, which returns success as soon as the postgres
+# process exists -- before WAL recovery completes. createdb then hits
+# "FATAL: the database system is starting up". Replace that probe with
+# pg_isready (returns 0 only when postgres is accepting connections) and
+# bump the retry budget from ~12s to ~60s for slower CI runners.
+# Fail the build if any anchor changes upstream so we notice early.
+RUN grep -q '^	tries=5$' /opt/pbs/libexec/pbs_db_utility \
+ && grep -q '\${server_ctl} status > /dev/null 2>&1' /opt/pbs/libexec/pbs_db_utility \
+ && sed -i \
+        -e '0,/^	tries=5$/{s||	tries=30|}' \
+        -e 's|\${server_ctl} status > /dev/null 2>&1|\${bin_dir}/pg_isready -h /var/run/postgresql -p \${port} -q > /dev/null 2>&1|' \
+        /opt/pbs/libexec/pbs_db_utility \
+ && grep -q 'pg_isready' /opt/pbs/libexec/pbs_db_utility \
+ && grep -q '^	tries=30$' /opt/pbs/libexec/pbs_db_utility
+
 # Create pbsdata system user required by PBS dataservice (normally done by package manager)
 RUN useradd -r -m -d /var/lib/pbsdata -s /bin/sh -G postgres pbsdata
 
diff --git a/server/docker-entrypoint.sh b/server/docker-entrypoint.sh
index d96d973..931feb9 100644
--- a/server/docker-entrypoint.sh
+++ b/server/docker-entrypoint.sh
@@ -82,12 +82,15 @@ for attempt in 1 2 3 4 5; do
     sudo pkill -9 -x postgres 2>/dev/null || true
     sudo rm -rf /var/spool/pbs/datastore 2>/dev/null || true
 
-    if sudo /opt/pbs/libexec/pbs_db_utility install_db; then
-        install_db_rc=0
+    # Use `|| install_db_rc=$?` rather than `if cmd; then ... fi`: after an
+    # `if`, $? is the exit status of the if statement itself (0 when the
+    # then-branch didn't run), so we'd report the wrong rc on failure.
+    install_db_rc=0
+    sudo /opt/pbs/libexec/pbs_db_utility install_db || install_db_rc=$?
+    if [ "$install_db_rc" -eq 0 ]; then
         echo "install_db succeeded on attempt $attempt."
         break
     fi
-    install_db_rc=$?
     echo "install_db attempt $attempt failed (rc=$install_db_rc); retrying in 10s..."
     sleep 10
 done

From 68d2d3383784a3c9ad2ededdcb8e18a0264c47bb Mon Sep 17 00:00:00 2001
From: christopherwharrop <christopher.w.harrop@gmail.com>
Date: Fri, 22 May 2026 14:24:32 -0600
Subject: [PATCH 10/11] Fix broken sed command by escaping &

---
 server/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/Dockerfile b/server/Dockerfile
index f3cf05b..4aa57f6 100644
--- a/server/Dockerfile
+++ b/server/Dockerfile
@@ -73,9 +73,9 @@ RUN grep -q '^	tries=5$' /opt/pbs/libexec/pbs_db_utility \
  && grep -q '\${server_ctl} status > /dev/null 2>&1' /opt/pbs/libexec/pbs_db_utility \
  && sed -i \
         -e '0,/^	tries=5$/{s||	tries=30|}' \
-        -e 's|\${server_ctl} status > /dev/null 2>&1|\${bin_dir}/pg_isready -h /var/run/postgresql -p \${port} -q > /dev/null 2>&1|' \
+        -e 's|\${server_ctl} status > /dev/null 2>&1|\${bin_dir}/pg_isready -h /var/run/postgresql -p \${port} -q > /dev/null 2>\&1|' \
         /opt/pbs/libexec/pbs_db_utility \
- && grep -q 'pg_isready' /opt/pbs/libexec/pbs_db_utility \
+ && grep -q 'pg_isready -h /var/run/postgresql -p \${port} -q > /dev/null 2>&1' /opt/pbs/libexec/pbs_db_utility \
  && grep -q '^	tries=30$' /opt/pbs/libexec/pbs_db_utility
 
 # Create pbsdata system user required by PBS dataservice (normally done by package manager)

From 3dde4c5d78d51d73d09532b480a8c398f7accb5f Mon Sep 17 00:00:00 2001
From: christopherwharrop <christopher.w.harrop@gmail.com>
Date: Fri, 22 May 2026 14:42:29 -0600
Subject: [PATCH 11/11] Cleanup and document PBS patch

---
 .github/workflows/docker.yml | 14 ------------
 server/Dockerfile            | 42 +++++++++++++++++++++++++++++------
 server/docker-entrypoint.sh  | 43 +++++-------------------------------
 3 files changed, 40 insertions(+), 59 deletions(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 3dee20c..c472220 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -78,13 +78,6 @@ jobs:
       -
         name: Shut down PBS cluster containers
         run: docker compose -f docker-compose.yml down
-      -
-        name: Debug session
-        if: ${{ failure() }}
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
-        with:
-          limit-access-to-actor: true
 
 
   docker_compose_test_arm64:
@@ -149,13 +142,6 @@ jobs:
       -
         name: Shut down PBS cluster containers
         run: docker compose -f docker-compose.yml down
-      -
-        name: Debug session
-        if: ${{ failure() }}
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 60
-        with:
-          limit-access-to-actor: true
 
   build:
     name: Build ${{ matrix.container }} (ubuntu-${{ matrix.ubuntu }}, ${{ matrix.arch }})
diff --git a/server/Dockerfile b/server/Dockerfile
index 4aa57f6..863286f 100644
--- a/server/Dockerfile
+++ b/server/Dockerfile
@@ -62,13 +62,41 @@ RUN git clone --branch v${OPENPBS_VERSION} --depth 1 https://github.com/openpbs/
  && /opt/pbs/libexec/pbs_postinstall \
  && rm -rf /tmp/openpbs
 
-# Fix race in pbs_db_utility install_db: the upstream script polls
-# `pbs_dataservice status`, which returns success as soon as the postgres
-# process exists -- before WAL recovery completes. createdb then hits
-# "FATAL: the database system is starting up". Replace that probe with
-# pg_isready (returns 0 only when postgres is accepting connections) and
-# bump the retry budget from ~12s to ~60s for slower CI runners.
-# Fail the build if any anchor changes upstream so we notice early.
+# Fix a race in pbs_db_utility install_db.
+#
+# Upstream's pbs_install_db() starts the data service and then waits for it
+# to come up with this loop (verbatim from v23.06.06 src/lib/Libdb/pgsql/pbs_db_utility):
+#
+#     ${server_ctl} start
+#     ...
+#     tries=5
+#     while [ $tries -ge 0 ]
+#     do
+#         ${server_ctl} status > /dev/null 2>&1   # <-- only checks "process exists"
+#         ret=$?
+#         if [ $ret -eq 0 ]; then break; fi
+#         tries=$((tries-1))
+#         sleep 2
+#     done
+#     # ... then runs createdb immediately ...
+#
+# `pbs_dataservice status` reports success as soon as the postgres process
+# is up, but postgres may still be in startup (WAL recovery, listen-socket
+# bind) and refusing new connections. createdb then fails with:
+#   FATAL:  the database system is starting up
+# followed by `Error creating PBS datastore` and install_db exits.
+#
+# This race was timing-sensitive: it manifested deterministically on
+# Ubuntu 24.04 / arm64 in CI but went unnoticed on the other three platform
+# combos. We close it by patching two lines in the installed script:
+#   - tries=5  -> tries=30   (bump the wait budget from ~12s to ~62s)
+#   - ${server_ctl} status   -> pg_isready, which only returns 0 when
+#                               postgres is actually accepting connections
+#
+# The grep -q calls before/after sed assert that the anchor lines exist
+# and that the substitution produced the expected result, so a future
+# upstream change or a botched sed will fail the build loudly instead of
+# silently producing a broken image.
 RUN grep -q '^	tries=5$' /opt/pbs/libexec/pbs_db_utility \
  && grep -q '\${server_ctl} status > /dev/null 2>&1' /opt/pbs/libexec/pbs_db_utility \
  && sed -i \
diff --git a/server/docker-entrypoint.sh b/server/docker-entrypoint.sh
index 931feb9..6217115 100644
--- a/server/docker-entrypoint.sh
+++ b/server/docker-entrypoint.sh
@@ -58,45 +58,12 @@ trap 'on_error $LINENO' ERR
 #   failed: No such file or directory
 sudo install -d -o postgres -g postgres -m 2775 /var/run/postgresql
 
-# Initialize the PBS PostgreSQL datastore.
-# install_db has a known race condition: pbs_dataservice considers postgres
-# "started" as soon as the process is up, but createdb can hit "the database
-# system is starting up" if it connects before postgres finishes WAL recovery.
-# We retry the whole install_db invocation; each retry runs initdb from
-# scratch, so the race window is re-rolled. If we still fail after a few
-# attempts, we continue anyway: `pbs_server -t create` is able to bootstrap
-# the datastore itself and is the real source of truth for startup success.
-#
-# Calling install_db inside an `if` (or via `|| true`) is essential: the
-# `trap ... ERR` above fires on *any* simple-command non-zero exit, even
-# under `set +e`. Only the conditional context suppresses it.
+# Initialize the PBS PostgreSQL datastore. The historical race in install_db
+# (where createdb ran before postgres was accepting connections) is patched
+# at build time in the Dockerfile, so this is expected to succeed on the
+# first try. If it ever fails, the ERR trap will dump diagnostics.
 echo "Initializing PBS datastore (pbs_db_utility install_db)..."
-install_db_rc=1
-for attempt in 1 2 3 4 5; do
-    # Make sure no stale data-service state from a failed previous attempt is
-    # left around: pbs_dataservice's "cleanup" doesn't always tear down the
-    # pbs_ds_monitor process, which can confuse the next attempt.
-    # Match by process name only (not full cmdline) so pkill doesn't match
-    # its own sudo wrapper.
-    sudo pkill -9 -x pbs_ds_monitor 2>/dev/null || true
-    sudo pkill -9 -x postgres 2>/dev/null || true
-    sudo rm -rf /var/spool/pbs/datastore 2>/dev/null || true
-
-    # Use `|| install_db_rc=$?` rather than `if cmd; then ... fi`: after an
-    # `if`, $? is the exit status of the if statement itself (0 when the
-    # then-branch didn't run), so we'd report the wrong rc on failure.
-    install_db_rc=0
-    sudo /opt/pbs/libexec/pbs_db_utility install_db || install_db_rc=$?
-    if [ "$install_db_rc" -eq 0 ]; then
-        echo "install_db succeeded on attempt $attempt."
-        break
-    fi
-    echo "install_db attempt $attempt failed (rc=$install_db_rc); retrying in 10s..."
-    sleep 10
-done
-if [ "$install_db_rc" -ne 0 ]; then
-    echo "WARN: install_db failed after 5 attempts; will rely on pbs_server -t create to bootstrap."
-fi
+sudo /opt/pbs/libexec/pbs_db_utility install_db
 
 # Start pbs_comm first (pbs_server requires it)
 echo "Starting pbs_comm..."