ARO-HCP: add optional in-place upgrade E2E workflow

ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml

@@ -463,6 +463,18 @@ tests:
       MULTISTAGE_PARAM_OVERRIDE_LOCATION: westus3
     workflow: aro-hcp-upgrade-e2e
   timeout: 10h0m0s
+- always_run: false
+  as: e2e-parallel-inplace-upgrade
+  optional: true
+  steps:
+    env:
+      ALLOWED_SUBSCRIPTIONS: ARO HCP E2E Hosted Clusters (EA Subscription),ARO HCP
+        E2E Hosted Clusters 2 (EA Subscription),ARO HCP E2E Hosted Clusters - Dev
+        - 02,ARO HCP E2E Hosted Clusters - Dev - 03
+      ARO_HCP_CLOUD: dev
+      ARO_HCP_DEPLOY_ENV: ci01
+      MULTISTAGE_PARAM_OVERRIDE_LOCATION: westus3
+    workflow: aro-hcp-local-e2e-upgrade
 - as: global-pipeline-postsubmit
   max_concurrency: 1
   postsubmit: true

ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml

@@ -685,6 +685,91 @@ presubmits:
         secret:
           secretName: result-aggregator
     trigger: (?m)^/test( | .* )e2e-parallel,?($|\s.*)
+  - agent: kubernetes
+    always_run: false
+    branches:
+    - ^main$
+    - ^main-
+    cluster: build05
+    context: ci/prow/e2e-parallel-inplace-upgrade
+    decorate: true
+    decoration_config:
+      sparse_checkout_files:
+      - admin/Dockerfile
+      - backend/Dockerfile
+      - fleet/Dockerfile
+      - frontend/Dockerfile
+      - hcp-recovery/Dockerfile
+      - image-sync/oc-mirror/Dockerfile
+      - kube-applier/Dockerfile
+      - mgmt-agent/Dockerfile
+      - sessiongate/Dockerfile
+      - test/Containerfile.e2e
+      - tooling/aro-hcp-exporter/Dockerfile
+    labels:
+      ci.openshift.io/generator: prowgen
+      pj-rehearse.openshift.io/can-be-rehearsed: "true"
+    name: pull-ci-Azure-ARO-HCP-main-e2e-parallel-inplace-upgrade
+    optional: true
+    rerun_command: /test e2e-parallel-inplace-upgrade
+    spec:
+      containers:
+      - args:
+        - --gcs-upload-secret=/secrets/gcs/service-account.json
+        - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+        - --lease-server-credentials-file=/etc/boskos/credentials
+        - --report-credentials-file=/etc/report/credentials
+        - --target=e2e-parallel-inplace-upgrade
+        command:
+        - ci-operator
+        env:
+        - name: HTTP_SERVER_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
+        imagePullPolicy: Always
+        name: ""
+        ports:
+        - containerPort: 8080
+          name: http
+        resources:
+          requests:
+            cpu: 10m
+        volumeMounts:
+        - mountPath: /etc/boskos
+          name: boskos
+          readOnly: true
+        - mountPath: /secrets/gcs
+          name: gcs-credentials
+          readOnly: true
+        - mountPath: /secrets/manifest-tool
+          name: manifest-tool-local-pusher
+          readOnly: true
+        - mountPath: /etc/pull-secret
+          name: pull-secret
+          readOnly: true
+        - mountPath: /etc/report
+          name: result-aggregator
+          readOnly: true
+      serviceAccountName: ci-operator
+      volumes:
+      - name: boskos
+        secret:
+          items:
+          - key: credentials
+            path: credentials
+          secretName: boskos-credentials
+      - name: manifest-tool-local-pusher
+        secret:
+          secretName: manifest-tool-local-pusher
+      - name: pull-secret
+        secret:
+          secretName: registry-pull-credentials
+      - name: result-aggregator
+        secret:
+          secretName: result-aggregator
+    trigger: (?m)^/test( | .* )e2e-parallel-inplace-upgrade,?($|\s.*)
   - agent: kubernetes
     always_run: true
     branches:

ci-operator/step-registry/aro-hcp/gather/provision-failure/aro-hcp-gather-provision-failure-commands.sh

@@ -21,7 +21,7 @@ echo "MGMT_RESOURCEGROUP: ${MGMT_RESOURCEGROUP}"
 echo "REGIONAL_RESOURCEGROUP: ${REGIONAL_RESOURCEGROUP}"
 
 # Check if provisioning completed successfully
-if [[ -f "${SHARED_DIR}/provision-complete" ]]; then
+if [[ -f "${SHARED_DIR}/provision-complete" || -f "${SHARED_DIR}/provision-from-main-complete" ]]; then
   echo "Provisioning completed successfully, skipping failure data gathering"
   exit 0
 fi

ci-operator/step-registry/aro-hcp/local-e2e-upgrade/OWNERS

@@ -0,0 +1,8 @@
+approvers:
+- aro-hcp-sl-approvers
+- geoberle
+- deads2k
+reviewers:
+- aro-hcp-sl-reviewers
+- geoberle
+- deads2k

ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.metadata.json

@@ -0,0 +1,15 @@
+{
+	"path": "aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml",
+	"owners": {
+		"approvers": [
+			"aro-hcp-sl-approvers",
+			"geoberle",
+			"deads2k"
+		],
+		"reviewers": [
+			"aro-hcp-sl-reviewers",
+			"geoberle",
+			"deads2k"
+		]
+	}
+}

ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml

@@ -0,0 +1,28 @@
+workflow:
+  as: aro-hcp-local-e2e-upgrade
+  steps:
+    allow_best_effort_post_steps: true
+    leases:
+      - env: LEASED_MSI_MOCK_SP
+        resource_type: aro-hcp-msi-mock-cs-sp-dev
+    pre:
+      - ref: aro-hcp-lease-acquire
+      - ref: aro-hcp-write-config
+      - ref: aro-hcp-provision-from-main
+    test:
+      - ref: aro-hcp-test-local-upgrade
+    post:
+      - ref: aro-hcp-deprovision-tracked-resource-groups
+      - ref: aro-hcp-gather-provision-failure
+      - ref: aro-hcp-gather-visualization
+      - ref: aro-hcp-gather-test-visualization
+      - ref: aro-hcp-gather-custom-link-tools
+      - ref: aro-hcp-gather-observability
+      - ref: aro-hcp-gather-snapshot
+      - ref: aro-hcp-deprovision-environment
+      - ref: aro-hcp-lease-release
+  documentation: |-
+    Acquire runtime leases, provision a main-branch baseline via
+    aro-hcp-provision-from-main (ACR images from images-push), run
+    upgrade/in-place on the PR head with OVERRIDE_CONFIG_FILE built from PR
+    pipeline images and hypershift config.

ci-operator/step-registry/aro-hcp/test/local-upgrade/OWNERS

@@ -0,0 +1,8 @@
+approvers:
+- aro-hcp-sl-approvers
+- geoberle
+- deads2k
+reviewers:
+- aro-hcp-sl-reviewers
+- geoberle
+- deads2k

ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-commands.sh

@@ -0,0 +1,171 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+: "${BACKEND_IMAGE:?BACKEND_IMAGE must be set}"
+: "${FRONTEND_IMAGE:?FRONTEND_IMAGE must be set}"
+: "${ADMIN_API_IMAGE:?ADMIN_API_IMAGE must be set}"
+: "${SESSIONGATE_IMAGE:?SESSIONGATE_IMAGE must be set}"
+: "${HCP_RECOVERY_IMAGE:?HCP_RECOVERY_IMAGE must be set}"
+: "${FLEET_IMAGE:?FLEET_IMAGE must be set}"
+: "${MGMT_AGENT_IMAGE:?MGMT_AGENT_IMAGE must be set}"
+: "${KUBE_APPLIER_IMAGE:?KUBE_APPLIER_IMAGE must be set}"
+
+if [[ ! -f "${SHARED_DIR}/config.yaml" ]]; then
+  echo "ERROR: ${SHARED_DIR}/config.yaml missing; run aro-hcp-provision-from-main first"
+  exit 1
+fi
+
+env_file="${SHARED_DIR}/aro-hcp-slot.env"
+if [[ ! -f "${env_file}" ]]; then
+  printf 'Missing runtime lease export file: %s\n' "${env_file}" >&2
+  exit 1
+fi
+
+# shellcheck disable=SC1090
+source "${env_file}"
+
+export LOCATION="${SELECTED_LOCATION:-${LOCATION:-}}"
+: "${LOCATION:?LOCATION must be provided by SELECTED_LOCATION or the legacy runtime slot export file}"
+: "${CUSTOMER_SUBSCRIPTION:?CUSTOMER_SUBSCRIPTION must be provided by the runtime slot export file}"
+
+export CLUSTER_PROFILE_DIR="/var/run/aro-hcp-${VAULT_SECRET_PROFILE}"
+
+export AZURE_CLIENT_ID; AZURE_CLIENT_ID=$(cat "${CLUSTER_PROFILE_DIR}/client-id")
+export AZURE_TENANT_ID; AZURE_TENANT_ID=$(cat "${CLUSTER_PROFILE_DIR}/tenant")
+export AZURE_CLIENT_SECRET; AZURE_CLIENT_SECRET=$(cat "${CLUSTER_PROFILE_DIR}/client-secret")
+export INFRA_SUBSCRIPTION_ID; INFRA_SUBSCRIPTION_ID=$(cat "${CLUSTER_PROFILE_DIR}/infra-${ARO_HCP_DEPLOY_ENV}-subscription-id")
+export DEPLOY_ENV="${ARO_HCP_DEPLOY_ENV}"
+export AZURE_TOKEN_CREDENTIALS=prod
+export SKIP_CONFIRM=true
+export PERSIST=true
+export DETECT_DIRTY_GIT_WORKTREE=0
+
+az login --service-principal -u "${AZURE_CLIENT_ID}" -p "${AZURE_CLIENT_SECRET}" --tenant "${AZURE_TENANT_ID}" --output none
+
+# Hypershift images from PR config; regional service images from PR pipeline builds.
+if ! yq -e '.defaults.hypershift.image.registry' config/config.yaml >/dev/null 2>&1 \
+  || ! yq -e '.defaults.hypershift.image.repository' config/config.yaml >/dev/null 2>&1 \
+  || ! yq -e '.defaults.hypershift.image.digest' config/config.yaml >/dev/null 2>&1; then
+  echo "ERROR: hypershift operator image missing in config/config.yaml (.defaults.hypershift.image)" >&2
+  exit 1
+fi
+if ! yq -e '.defaults.hypershift.sharedIngressImage.registry' config/config.yaml >/dev/null 2>&1 \
+  || ! yq -e '.defaults.hypershift.sharedIngressImage.repository' config/config.yaml >/dev/null 2>&1 \
+  || ! yq -e '.defaults.hypershift.sharedIngressImage.digest' config/config.yaml >/dev/null 2>&1; then
+  echo "ERROR: hypershift sharedIngressImage missing in config/config.yaml (.defaults.hypershift.sharedIngressImage)" >&2
+  exit 1
+fi
+
+HO_IMAGE_REGISTRY=$(yq '.defaults.hypershift.image.registry' config/config.yaml)
+HO_IMAGE_REPOSITORY=$(yq '.defaults.hypershift.image.repository' config/config.yaml)
+HO_IMAGE_DIGEST=$(yq '.defaults.hypershift.image.digest' config/config.yaml)
+HO_SHARED_INGRESS_REGISTRY=$(yq '.defaults.hypershift.sharedIngressImage.registry' config/config.yaml)
+HO_SHARED_INGRESS_REPOSITORY=$(yq '.defaults.hypershift.sharedIngressImage.repository' config/config.yaml)
+HO_SHARED_INGRESS_DIGEST=$(yq '.defaults.hypershift.sharedIngressImage.digest' config/config.yaml)
+
+BACKEND_DIGEST=$(echo "${BACKEND_IMAGE}" | cut -d'@' -f2)
+BACKEND_REPOSITORY=$(echo "${BACKEND_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f2-)
+BACKEND_SOURCE_REGISTRY=$(echo "${BACKEND_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f1)
+
+FRONTEND_DIGEST=$(echo "${FRONTEND_IMAGE}" | cut -d'@' -f2)
+FRONTEND_REPOSITORY=$(echo "${FRONTEND_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f2-)
+FRONTEND_SOURCE_REGISTRY=$(echo "${FRONTEND_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f1)
+
+ADMIN_API_DIGEST=$(echo "${ADMIN_API_IMAGE}" | cut -d'@' -f2)
+ADMIN_API_REPOSITORY=$(echo "${ADMIN_API_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f2-)
+ADMIN_API_SOURCE_REGISTRY=$(echo "${ADMIN_API_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f1)
+
+SESSIONGATE_DIGEST=$(echo "${SESSIONGATE_IMAGE}" | cut -d'@' -f2)
+SESSIONGATE_REPOSITORY=$(echo "${SESSIONGATE_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f2-)
+SESSIONGATE_SOURCE_REGISTRY=$(echo "${SESSIONGATE_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f1)
+
+HCP_RECOVERY_DIGEST=$(echo "${HCP_RECOVERY_IMAGE}" | cut -d'@' -f2)
+HCP_RECOVERY_REPOSITORY=$(echo "${HCP_RECOVERY_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f2-)
+HCP_RECOVERY_SOURCE_REGISTRY=$(echo "${HCP_RECOVERY_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f1)
+
+FLEET_DIGEST=$(echo "${FLEET_IMAGE}" | cut -d'@' -f2)
+FLEET_REPOSITORY=$(echo "${FLEET_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f2-)
+FLEET_SOURCE_REGISTRY=$(echo "${FLEET_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f1)
+
+MGMT_AGENT_DIGEST=$(echo "${MGMT_AGENT_IMAGE}" | cut -d'@' -f2)
+MGMT_AGENT_REPOSITORY=$(echo "${MGMT_AGENT_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f2-)
+MGMT_AGENT_SOURCE_REGISTRY=$(echo "${MGMT_AGENT_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f1)
+
+KUBE_APPLIER_DIGEST=$(echo "${KUBE_APPLIER_IMAGE}" | cut -d'@' -f2)
+KUBE_APPLIER_REPOSITORY=$(echo "${KUBE_APPLIER_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f2-)
+KUBE_APPLIER_SOURCE_REGISTRY=$(echo "${KUBE_APPLIER_IMAGE}" | cut -d'@' -f1 | cut -d '/' -f1)
+
+if [[ -n "${USE_OC_LOGIN_REGISTRIES:-}" ]]; then
+  USE_OC_LOGIN_REGISTRIES="${USE_OC_LOGIN_REGISTRIES} ${BACKEND_SOURCE_REGISTRY} ${FRONTEND_SOURCE_REGISTRY} ${ADMIN_API_SOURCE_REGISTRY} ${SESSIONGATE_SOURCE_REGISTRY} ${HCP_RECOVERY_SOURCE_REGISTRY} ${FLEET_SOURCE_REGISTRY} ${MGMT_AGENT_SOURCE_REGISTRY} ${KUBE_APPLIER_SOURCE_REGISTRY}"
+else
+  USE_OC_LOGIN_REGISTRIES="${BACKEND_SOURCE_REGISTRY} ${FRONTEND_SOURCE_REGISTRY} ${ADMIN_API_SOURCE_REGISTRY} ${SESSIONGATE_SOURCE_REGISTRY} ${HCP_RECOVERY_SOURCE_REGISTRY} ${FLEET_SOURCE_REGISTRY} ${MGMT_AGENT_SOURCE_REGISTRY} ${KUBE_APPLIER_SOURCE_REGISTRY}"
+fi
+export USE_OC_LOGIN_REGISTRIES
+
+export OVERRIDE_CONFIG_FILE="${SHARED_DIR}/config-override-upgrade.yaml"
+
+yq eval -n "
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.backend.image.registry = \"${BACKEND_SOURCE_REGISTRY}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.backend.image.repository = \"${BACKEND_REPOSITORY}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.backend.image.digest = \"${BACKEND_DIGEST}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.frontend.image.registry = \"${FRONTEND_SOURCE_REGISTRY}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.frontend.image.repository = \"${FRONTEND_REPOSITORY}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.frontend.image.digest = \"${FRONTEND_DIGEST}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.adminApi.image.registry = \"${ADMIN_API_SOURCE_REGISTRY}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.adminApi.image.repository = \"${ADMIN_API_REPOSITORY}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.adminApi.image.digest = \"${ADMIN_API_DIGEST}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.sessiongate.image.registry = \"${SESSIONGATE_SOURCE_REGISTRY}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.sessiongate.image.repository = \"${SESSIONGATE_REPOSITORY}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.sessiongate.image.digest = \"${SESSIONGATE_DIGEST}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.hcpRecovery.image.registry = \"${HCP_RECOVERY_SOURCE_REGISTRY}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.hcpRecovery.image.repository = \"${HCP_RECOVERY_REPOSITORY}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.hcpRecovery.image.digest = \"${HCP_RECOVERY_DIGEST}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.fleet.image.registry = \"${FLEET_SOURCE_REGISTRY}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.fleet.image.repository = \"${FLEET_REPOSITORY}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.fleet.image.digest = \"${FLEET_DIGEST}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.mgmtAgent.image.registry = \"${MGMT_AGENT_SOURCE_REGISTRY}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.mgmtAgent.image.repository = \"${MGMT_AGENT_REPOSITORY}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.mgmtAgent.image.digest = \"${MGMT_AGENT_DIGEST}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.kubeApplier.image.registry = \"${KUBE_APPLIER_SOURCE_REGISTRY}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.kubeApplier.image.repository = \"${KUBE_APPLIER_REPOSITORY}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.kubeApplier.image.digest = \"${KUBE_APPLIER_DIGEST}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift.image.registry = \"${HO_IMAGE_REGISTRY}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift.image.repository = \"${HO_IMAGE_REPOSITORY}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift.image.digest = \"${HO_IMAGE_DIGEST}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift.sharedIngressImage.registry = \"${HO_SHARED_INGRESS_REGISTRY}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift.sharedIngressImage.repository = \"${HO_SHARED_INGRESS_REPOSITORY}\" |
+  .clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift.sharedIngressImage.digest = \"${HO_SHARED_INGRESS_DIGEST}\"
+" > "${OVERRIDE_CONFIG_FILE}"
+
+cp "${OVERRIDE_CONFIG_FILE}" "${SHARED_DIR}/config-override.yaml"
+
+echo "Created upgrade override config at: ${OVERRIDE_CONFIG_FILE}"
+cat "${OVERRIDE_CONFIG_FILE}"
+
+unset GOFLAGS
+
+# Prepare svc cluster access for the test harness (customer tests run below).
+az account set --subscription "${INFRA_SUBSCRIPTION_ID}"
+make -C dev-infrastructure/ svc.aks.kubeconfig.pipeline SVC_KUBECONFIG_FILE=../kubeconfig DEPLOY_ENV="${DEPLOY_ENV}"
+export KUBECONFIG=kubeconfig
+FRONTEND_ADDRESS="https://$(kubectl get virtualservice -n aro-hcp aro-hcp-vs-frontend -o jsonpath='{.spec.hosts[0]}')"
+make frontend-grant-ingress DEPLOY_ENV="${DEPLOY_ENV}"
+
+make -C dev-infrastructure/ mgmt.aks.kubeconfig MGMT_KUBECONFIG_FILE=../mgmt-kubeconfig DEPLOY_ENV="${DEPLOY_ENV}"
+export KUBECONFIG=mgmt-kubeconfig
+
+az account set --subscription "${CUSTOMER_SUBSCRIPTION}"
+CUSTOMER_SUBSCRIPTION="$(az account show --output tsv --query 'name')"
+make e2e-local/setup FRONTEND_ADDRESS="${FRONTEND_ADDRESS}"
+
+SKIP_CERT_VERIFICATION=true \
+FRONTEND_ADDRESS="${FRONTEND_ADDRESS}" \
+CUSTOMER_SUBSCRIPTION="${CUSTOMER_SUBSCRIPTION}" \
+  ./test/aro-hcp-tests run-suite upgrade/in-place \
+  --junit-path="${ARTIFACT_DIR}/junit.xml" \
+  --html-path="${ARTIFACT_DIR}/extension-test-result-summary.html" \
+  --max-concurrency 100
+
+gzip -c "${ARTIFACT_DIR}/junit.xml" > "${SHARED_DIR}/junit-e2e-upgrade.xml.gz"

ci-operator/step-registry/aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.metadata.json

@@ -0,0 +1,15 @@
+{
+	"path": "aro-hcp/test/local-upgrade/aro-hcp-test-local-upgrade-ref.yaml",
+	"owners": {
+		"approvers": [
+			"aro-hcp-sl-approvers",
+			"geoberle",
+			"deads2k"
+		],
+		"reviewers": [
+			"aro-hcp-sl-reviewers",
+			"geoberle",
+			"deads2k"
+		]
+	}
+}