From e22028caff132042a2e4230beae7e5f84a495b03 Mon Sep 17 00:00:00 2001 From: Wenqi He Date: Thu, 30 Jul 2026 04:12:28 +0000 Subject: [PATCH] feat: set evidence-based memory requests for backend and mgmt-agent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backend: 500Mi → 1Gi (actual steady-state: 750MB–1GB in uksouth) mgmt-agent: 64Mi → 768Mi (actual steady-state: 256–512MB, peak <768MB) Values determined by querying container_memory_working_set_bytes across 3 prod regions for 7 days using alert-tester. Formula: peak × 1.25 margin. Ref: AROSLSRE-1033, AROSLSRE-1654 --- backend/deploy/templates/backend.deployment.yaml | 1 + ...ackend_clstr_scoped_identities_role_set_name_public.yaml | 3 ++- ..._backend_mi_mock_and_arm_perms_mgr_identities_unset.yaml | 3 ++- ...tHelmTemplate_dev_westus3_svc_1_aro_hcp_backend_dev.yaml | 3 ++- config/config.yaml | 6 +++++- config/rendered/dev/ci00/centralus.yaml | 6 +++++- config/rendered/dev/ci01/centralus.yaml | 6 +++++- config/rendered/dev/cspr/westus3.yaml | 6 +++++- config/rendered/dev/dev/westus3.yaml | 6 +++++- config/rendered/dev/perf/westus3.yaml | 6 +++++- config/rendered/dev/pers/westus3.yaml | 6 +++++- mgmt-agent/deploy/templates/deployment.yaml | 1 + mgmt-agent/values.yaml | 4 ++-- ...ture_TestHelmTemplate_dev_westus3_mgmt_1_mgmt_agent.yaml | 3 ++- 14 files changed, 47 insertions(+), 13 deletions(-) diff --git a/backend/deploy/templates/backend.deployment.yaml b/backend/deploy/templates/backend.deployment.yaml index 666be88627d..45c4f5b3da9 100644 --- a/backend/deploy/templates/backend.deployment.yaml +++ b/backend/deploy/templates/backend.deployment.yaml @@ -40,6 +40,7 @@ spec: matchLabels: app: aro-hcp-backend serviceAccountName: '{{ .Values.serviceAccount.name }}' + priorityClassName: service-lifecycle-critical volumes: - name: backend-service-key-vault csi: diff --git a/backend/testdata/zz_fixture_TestHelmTemplate_backend_clstr_scoped_identities_role_set_name_public.yaml b/backend/testdata/zz_fixture_TestHelmTemplate_backend_clstr_scoped_identities_role_set_name_public.yaml index cac546d7fff..0cf1b627ab0 100644 --- a/backend/testdata/zz_fixture_TestHelmTemplate_backend_clstr_scoped_identities_role_set_name_public.yaml +++ b/backend/testdata/zz_fixture_TestHelmTemplate_backend_clstr_scoped_identities_role_set_name_public.yaml @@ -154,6 +154,7 @@ spec: matchLabels: app: aro-hcp-backend serviceAccountName: 'backend' + priorityClassName: service-lifecycle-critical volumes: - name: backend-service-key-vault csi: @@ -239,7 +240,7 @@ spec: resources: requests: cpu: 100m - memory: 500Mi + memory: 1Gi securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/backend/testdata/zz_fixture_TestHelmTemplate_backend_mi_mock_and_arm_perms_mgr_identities_unset.yaml b/backend/testdata/zz_fixture_TestHelmTemplate_backend_mi_mock_and_arm_perms_mgr_identities_unset.yaml index a5580550e62..7c5ca58bebd 100644 --- a/backend/testdata/zz_fixture_TestHelmTemplate_backend_mi_mock_and_arm_perms_mgr_identities_unset.yaml +++ b/backend/testdata/zz_fixture_TestHelmTemplate_backend_mi_mock_and_arm_perms_mgr_identities_unset.yaml @@ -154,6 +154,7 @@ spec: matchLabels: app: aro-hcp-backend serviceAccountName: 'backend' + priorityClassName: service-lifecycle-critical volumes: - name: backend-service-key-vault csi: @@ -222,7 +223,7 @@ spec: resources: requests: cpu: 100m - memory: 500Mi + memory: 1Gi securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/backend/zz_fixture_TestHelmTemplate_dev_westus3_svc_1_aro_hcp_backend_dev.yaml b/backend/zz_fixture_TestHelmTemplate_dev_westus3_svc_1_aro_hcp_backend_dev.yaml index 15d120a7e6d..ee7eb9dfeaa 100644 --- a/backend/zz_fixture_TestHelmTemplate_dev_westus3_svc_1_aro_hcp_backend_dev.yaml +++ b/backend/zz_fixture_TestHelmTemplate_dev_westus3_svc_1_aro_hcp_backend_dev.yaml @@ -154,6 +154,7 @@ spec: matchLabels: app: aro-hcp-backend serviceAccountName: 'backend' + priorityClassName: service-lifecycle-critical volumes: - name: backend-service-key-vault csi: @@ -239,7 +240,7 @@ spec: resources: requests: cpu: 100m - memory: 500Mi + memory: 1Gi securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/config/config.yaml b/config/config.yaml index b6fa1593338..fc55768c443 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -219,6 +219,10 @@ defaults: namespace: mgmt-agent serviceAccountName: mgmt-agent replicas: 2 + resources: + requests: + cpu: 50m + memory: 768Mi # Kube Applier Controller kubeApplier: image: @@ -758,7 +762,7 @@ defaults: resources: requests: cpu: 100m - memory: 500Mi + memory: 1Gi limits: memory: unlimited azureRuntimeConfig: diff --git a/config/rendered/dev/ci00/centralus.yaml b/config/rendered/dev/ci00/centralus.yaml index 8fa42b23349..cf53530f3b1 100755 --- a/config/rendered/dev/ci00/centralus.yaml +++ b/config/rendered/dev/ci00/centralus.yaml @@ -120,7 +120,7 @@ backend: memory: unlimited requests: cpu: 100m - memory: 500Mi + memory: 1Gi serviceAccountName: backend logVerbosity: 4 maestro: @@ -751,6 +751,10 @@ mgmtAgent: k8s: namespace: mgmt-agent replicas: 2 + resources: + requests: + cpu: 50m + memory: 768Mi serviceAccountName: mgmt-agent managedIdentityName: mgmt-agent mgmtKeyVault: diff --git a/config/rendered/dev/ci01/centralus.yaml b/config/rendered/dev/ci01/centralus.yaml index cac2ebf91cf..7afe2f7422e 100755 --- a/config/rendered/dev/ci01/centralus.yaml +++ b/config/rendered/dev/ci01/centralus.yaml @@ -120,7 +120,7 @@ backend: memory: unlimited requests: cpu: 100m - memory: 500Mi + memory: 1Gi serviceAccountName: backend logVerbosity: 4 maestro: @@ -751,6 +751,10 @@ mgmtAgent: k8s: namespace: mgmt-agent replicas: 2 + resources: + requests: + cpu: 50m + memory: 768Mi serviceAccountName: mgmt-agent managedIdentityName: mgmt-agent mgmtKeyVault: diff --git a/config/rendered/dev/cspr/westus3.yaml b/config/rendered/dev/cspr/westus3.yaml index b9fe309fa74..a5338c45f95 100755 --- a/config/rendered/dev/cspr/westus3.yaml +++ b/config/rendered/dev/cspr/westus3.yaml @@ -120,7 +120,7 @@ backend: memory: unlimited requests: cpu: 100m - memory: 500Mi + memory: 1Gi serviceAccountName: backend logVerbosity: 4 maestro: @@ -749,6 +749,10 @@ mgmtAgent: k8s: namespace: mgmt-agent replicas: 2 + resources: + requests: + cpu: 50m + memory: 768Mi serviceAccountName: mgmt-agent managedIdentityName: mgmt-agent mgmtKeyVault: diff --git a/config/rendered/dev/dev/westus3.yaml b/config/rendered/dev/dev/westus3.yaml index 3c656dfacdf..91ba4e75c3b 100755 --- a/config/rendered/dev/dev/westus3.yaml +++ b/config/rendered/dev/dev/westus3.yaml @@ -120,7 +120,7 @@ backend: memory: unlimited requests: cpu: 100m - memory: 500Mi + memory: 1Gi serviceAccountName: backend logVerbosity: 4 maestro: @@ -749,6 +749,10 @@ mgmtAgent: k8s: namespace: mgmt-agent replicas: 2 + resources: + requests: + cpu: 50m + memory: 768Mi serviceAccountName: mgmt-agent managedIdentityName: mgmt-agent mgmtKeyVault: diff --git a/config/rendered/dev/perf/westus3.yaml b/config/rendered/dev/perf/westus3.yaml index d41a1d2959e..d65ef36e808 100755 --- a/config/rendered/dev/perf/westus3.yaml +++ b/config/rendered/dev/perf/westus3.yaml @@ -120,7 +120,7 @@ backend: memory: unlimited requests: cpu: 100m - memory: 500Mi + memory: 1Gi serviceAccountName: backend logVerbosity: 4 maestro: @@ -749,6 +749,10 @@ mgmtAgent: k8s: namespace: mgmt-agent replicas: 2 + resources: + requests: + cpu: 50m + memory: 768Mi serviceAccountName: mgmt-agent managedIdentityName: mgmt-agent mgmtKeyVault: diff --git a/config/rendered/dev/pers/westus3.yaml b/config/rendered/dev/pers/westus3.yaml index 4901ec9567c..b0947a68ca3 100755 --- a/config/rendered/dev/pers/westus3.yaml +++ b/config/rendered/dev/pers/westus3.yaml @@ -120,7 +120,7 @@ backend: memory: unlimited requests: cpu: 100m - memory: 500Mi + memory: 1Gi serviceAccountName: backend logVerbosity: 4 maestro: @@ -751,6 +751,10 @@ mgmtAgent: k8s: namespace: mgmt-agent replicas: 2 + resources: + requests: + cpu: 50m + memory: 768Mi serviceAccountName: mgmt-agent managedIdentityName: mgmt-agent mgmtKeyVault: diff --git a/mgmt-agent/deploy/templates/deployment.yaml b/mgmt-agent/deploy/templates/deployment.yaml index ce69d684107..840fd1966c8 100644 --- a/mgmt-agent/deploy/templates/deployment.yaml +++ b/mgmt-agent/deploy/templates/deployment.yaml @@ -69,6 +69,7 @@ spec: volumeMounts: [] volumes: [] serviceAccountName: {{ .Values.serviceAccount.name }} + priorityClassName: service-lifecycle-critical terminationGracePeriodSeconds: 10 nodeSelector: {{- with .Values.nodeSelector }} diff --git a/mgmt-agent/values.yaml b/mgmt-agent/values.yaml index cf6a742634b..a72289c9c32 100644 --- a/mgmt-agent/values.yaml +++ b/mgmt-agent/values.yaml @@ -9,8 +9,8 @@ ksmImage: deployment: replicas: "{{ .mgmtAgent.k8s.replicas }}" requests: - cpu: 50m - memory: 64Mi + cpu: "{{ .mgmtAgent.k8s.resources.requests.cpu }}" + memory: "{{ .mgmtAgent.k8s.resources.requests.memory }}" logVerbosity: 2 health: port: 8080 diff --git a/mgmt-agent/zz_fixture_TestHelmTemplate_dev_westus3_mgmt_1_mgmt_agent.yaml b/mgmt-agent/zz_fixture_TestHelmTemplate_dev_westus3_mgmt_1_mgmt_agent.yaml index 814ea7d1e74..d3c67e61bf1 100644 --- a/mgmt-agent/zz_fixture_TestHelmTemplate_dev_westus3_mgmt_1_mgmt_agent.yaml +++ b/mgmt-agent/zz_fixture_TestHelmTemplate_dev_westus3_mgmt_1_mgmt_agent.yaml @@ -251,10 +251,11 @@ spec: resources: requests: cpu: 50m - memory: 64Mi + memory: 768Mi volumeMounts: [] volumes: [] serviceAccountName: mgmt-agent + priorityClassName: service-lifecycle-critical terminationGracePeriodSeconds: 10 nodeSelector: ---