diff --git a/dev-infrastructure/region-pipeline.yaml b/dev-infrastructure/region-pipeline.yaml index 58492b39f87..9c221658fa9 100644 --- a/dev-infrastructure/region-pipeline.yaml +++ b/dev-infrastructure/region-pipeline.yaml @@ -11,6 +11,22 @@ $schema: "pipeline.schema.v1" serviceGroup: Microsoft.Azure.ARO.HCP.Region rolloutName: Region Rollout +# Pre-build the Go helper invoked by the ACR replication Shell steps before the +# rollout runs. The EV2 Shell runner image has no Azure CLI, so replica +# create/drift-reconcile is done in Go (see scripts/acr-replication), same +# pattern as scripts/postgres-access in svc-pipeline.yaml. +buildStep: + command: bash + args: + - -c + - | + set -euo pipefail + tmp="$(mktemp ./scripts/acr-replication/acr-replication.XXXXXX)" + trap 'rm -f "${tmp}"' EXIT + CGO_ENABLED=0 go build -o "${tmp}" ./scripts/acr-replication + chmod 0755 "${tmp}" + mv "${tmp}" ./scripts/acr-replication/acr-replication + trap - EXIT resourceGroups: - name: global resourceGroup: '{{ .global.rg }}' @@ -25,9 +41,16 @@ resourceGroups: outputOnly: true - name: ocp-acr-replication action: Shell - command: ./manage-acr-replication.sh - workingDir: ./scripts + command: ./scripts/acr-replication/acr-replication + workingDir: . variables: + - name: SUBSCRIPTION_ID + input: + resourceGroup: global + step: output + name: subscriptionId + - name: RESOURCE_GROUP + configRef: global.rg - name: ACR_NAME configRef: acr.ocp.name - name: REPLICATION_REGION @@ -41,9 +64,16 @@ resourceGroups: name: globalMSIId - name: svc-acr-replication action: Shell - command: ./manage-acr-replication.sh - workingDir: ./scripts + command: ./scripts/acr-replication/acr-replication + workingDir: . variables: + - name: SUBSCRIPTION_ID + input: + resourceGroup: global + step: output + name: subscriptionId + - name: RESOURCE_GROUP + configRef: global.rg - name: ACR_NAME configRef: acr.svc.name - name: REPLICATION_REGION diff --git a/dev-infrastructure/scripts/acr-replication/.gitignore b/dev-infrastructure/scripts/acr-replication/.gitignore new file mode 100644 index 00000000000..fe1719deb35 --- /dev/null +++ b/dev-infrastructure/scripts/acr-replication/.gitignore @@ -0,0 +1,2 @@ +# Binary produced by the region-pipeline buildStep. +/acr-replication diff --git a/dev-infrastructure/scripts/acr-replication/go.mod b/dev-infrastructure/scripts/acr-replication/go.mod new file mode 100644 index 00000000000..fccd4a481d4 --- /dev/null +++ b/dev-infrastructure/scripts/acr-replication/go.mod @@ -0,0 +1,26 @@ +module github.com/Azure/ARO-HCP/dev-infrastructure/scripts/acr-replication + +go 1.25.7 + +require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0 +) + +require ( + github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + golang.org/x/crypto v0.52.0 // indirect + golang.org/x/net v0.55.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.37.0 // indirect +) diff --git a/dev-infrastructure/scripts/acr-replication/go.sum b/dev-infrastructure/scripts/acr-replication/go.sum new file mode 100644 index 00000000000..c5ca81f2169 --- /dev/null +++ b/dev-infrastructure/scripts/acr-replication/go.sum @@ -0,0 +1,45 @@ +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1 h1:jHb/wfvRikGdxMXYV3QG/SzUOPYN9KEUUuC0Yd0/vC0= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1/go.mod h1:pzBXCYn05zvYIrwLgtK8Ap8QcjRg+0i76tMQdWN6wOk= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0 h1:xFaZZ+IubdftrDHnGGwZ6QvQ3KHTtWl2MCK+GMt2vxs= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0/go.mod h1:mCBhUhlMjLLJKr5aqw2TNS/VqJOie8MzWq3DAMJeKso= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0 h1:DWlwvVV5r/Wy1561nZ3wrpI1/vDIBRY/Wd1HWaRBZWA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0/go.mod h1:E7ltexgRDmeJ0fJWv0D/HLwY2xbDdN+uv+X2uZtOx3w= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 h1:Dd+RhdJn0OTtVGaeDLZpcumkIVCtA/3/Fo42+eoYvVM= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= +github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= +golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/dev-infrastructure/scripts/acr-replication/main.go b/dev-infrastructure/scripts/acr-replication/main.go new file mode 100644 index 00000000000..1792eade664 --- /dev/null +++ b/dev-infrastructure/scripts/acr-replication/main.go @@ -0,0 +1,348 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// acr-replication manages a single regional replica of an Azure Container +// Registry: it creates the replica if missing, deletes and recreates it if it +// is stuck in a Failed state, and reconciles its regional data-endpoint to the +// desired state on drift. +// +// It replaces the manage-acr-replication.sh shell script invoked per-region by +// the ocp-acr-replication and svc-acr-replication steps in region-pipeline.yaml. +// That script grew conditional branching and drift comparison with no +// automated tests; bugs there directly affect prod ACR data-path routing (root +// cause of incident AROSLSRE-1592). Reimplementing it in Go with unit test +// coverage lets that logic evolve safely. +// +// Inputs (environment variables, set by region-pipeline.yaml): +// +// SUBSCRIPTION_ID - subscription holding the ACR +// RESOURCE_GROUP - resource group holding the ACR +// ACR_NAME - name of the Azure Container Registry +// REPLICATION_REGION - Azure region to check/create a replica in; +// the replica is named after the region +// ENDPOINT_DISABLED_REGIONS - optional space-separated list of regions whose +// regional data endpoint must be kept disabled +// (e.g. a co-located canary replica). Defaults to +// none, i.e. the endpoint is enabled everywhere. +// LOG_VERBOSITY - optional slog verbosity (default 0) +// DRY_RUN - if set to any non-empty value, mutating calls +// (create/delete/update) are logged instead of +// executed +package main + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "strconv" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry" +) + +func main() { + verbosity := 0 + if v := os.Getenv("LOG_VERBOSITY"); v != "" { + if n, err := strconv.Atoi(v); err == nil { + verbosity = n + } + } + handler := slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{ + Level: slog.Level(verbosity * -4), + }) + slog.SetDefault(slog.New(handler).With("component", "acr-replication")) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + if err := run(ctx); err != nil { + slog.Error("run failed", "error", err.Error()) + os.Exit(1) + } +} + +// config holds all inputs sourced from environment variables. +type config struct { + subscriptionID string + resourceGroup string + acrName string + region string + disabledRegions map[string]bool + dryRun bool +} + +// parseEnvConfig builds a config from environment variables only. It does not +// call any external tools or APIs, which makes it safe to unit-test. +func parseEnvConfig(env func(string) string) (*config, error) { + c := &config{ + subscriptionID: env("SUBSCRIPTION_ID"), + resourceGroup: env("RESOURCE_GROUP"), + acrName: env("ACR_NAME"), + region: env("REPLICATION_REGION"), + } + + missing := []string{} + for k, v := range map[string]string{ + "SUBSCRIPTION_ID": c.subscriptionID, + "RESOURCE_GROUP": c.resourceGroup, + "ACR_NAME": c.acrName, + "REPLICATION_REGION": c.region, + } { + if v == "" { + missing = append(missing, k) + } + } + if len(missing) > 0 { + return nil, fmt.Errorf("missing required environment variables: %s", strings.Join(missing, ", ")) + } + + c.disabledRegions = map[string]bool{} + for _, r := range strings.Fields(env("ENDPOINT_DISABLED_REGIONS")) { + c.disabledRegions[r] = true + } + + // Mirrors the replaced shell script: any non-empty DRY_RUN enables + // dry-run mode, in which mutating calls (create/delete/update) are + // logged instead of executed. This keeps `make region.what-if` + // non-mutating. + c.dryRun = env("DRY_RUN") != "" + return c, nil +} + +// desiredEndpointEnabled reports whether the replica's regional data endpoint +// should be enabled for the configured replication region. +func (c *config) desiredEndpointEnabled() bool { + return !c.disabledRegions[c.region] +} + +func run(ctx context.Context) error { + cfg, err := parseEnvConfig(os.Getenv) + if err != nil { + return err + } + + // DefaultAzureCredential resolves to the rollout managed identity in EV2 + // and to the operator's `az login` locally; it never prompts interactively. + cred, err := azidentity.NewDefaultAzureCredential(nil) + if err != nil { + return fmt.Errorf("azidentity: %w", err) + } + + registriesClient, err := armcontainerregistry.NewRegistriesClient(cfg.subscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("new registries client: %w", err) + } + replicationsClient, err := armcontainerregistry.NewReplicationsClient(cfg.subscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("new replications client: %w", err) + } + + slog.Info("managing ACR replication", "acr", cfg.acrName, "region", cfg.region) + + registry, err := registriesClient.Get(ctx, cfg.resourceGroup, cfg.acrName, nil) + if err != nil { + return fmt.Errorf("get registry %q: %w", cfg.acrName, err) + } + if registry.Location == nil { + return fmt.Errorf("registry %q has no location", cfg.acrName) + } + homeRegion := *registry.Location + slog.Info("resolved registry home region", "acr", cfg.acrName, "homeRegion", homeRegion) + + if strings.EqualFold(cfg.region, homeRegion) { + slog.Info("registry is homed in the target region; replication is only needed for different regions", + "acr", cfg.acrName, "region", cfg.region) + return nil + } + + desiredEnabled := cfg.desiredEndpointEnabled() + slog.Info("desired regional endpoint state", "region", cfg.region, "enabled", desiredEnabled) + + return reconcile(ctx, replicationsClient, cfg, desiredEnabled) +} + +// reconcile creates, recreates, or updates the replica located in +// cfg.region so it ends up in a Succeeded state with the desired regional +// endpoint setting. +func reconcile(ctx context.Context, client *armcontainerregistry.ReplicationsClient, cfg *config, desiredEnabled bool) error { + name, err := findReplicationNameByLocation(ctx, client, cfg) + if err != nil { + return err + } + if name == "" { + slog.Info("no replication exists in region; creating", "region", cfg.region) + return createReplication(ctx, client, cfg, desiredEnabled) + } + + // Fetch authoritative state via Get on the discovered name rather than + // trusting the List response's Properties: the replaced shell script + // explicitly avoided reading state off the list/resource-list output + // because of known bugs where it reports the wrong provisioning/endpoint + // state, and instead did a separate `az resource show` on the replica ID. + existing, err := client.Get(ctx, cfg.resourceGroup, cfg.acrName, name, nil) + if isNotFound(err) { + // Replica disappeared between the list and the get; treat as missing. + slog.Info("no replication exists in region; creating", "region", cfg.region) + return createReplication(ctx, client, cfg, desiredEnabled) + } + if err != nil { + return fmt.Errorf("get replication %q: %w", name, err) + } + + state := armcontainerregistry.ProvisioningState("") + if existing.Properties != nil && existing.Properties.ProvisioningState != nil { + state = *existing.Properties.ProvisioningState + } + currentEnabled := true + if existing.Properties != nil && existing.Properties.RegionEndpointEnabled != nil { + currentEnabled = *existing.Properties.RegionEndpointEnabled + } + slog.Info("found existing replication", "region", cfg.region, "name", name, "state", state, "endpointEnabled", currentEnabled) + + switch state { + case armcontainerregistry.ProvisioningStateFailed: + slog.Info("replication is in failed state; deleting and recreating", "region", cfg.region, "name", name) + if err := deleteReplication(ctx, client, cfg, name); err != nil { + return err + } + return createReplication(ctx, client, cfg, desiredEnabled) + case armcontainerregistry.ProvisioningStateSucceeded: + // Mirrors the replaced shell script: reconciliation is only ever + // forced when the desired state is disabled (i.e. a canary region + // must be forced back to disabled on drift). When the desired state + // is enabled, an existing disabled replica is left untouched rather + // than re-enabled, since disabling it may have been an intentional, + // separately-managed mitigation. + if !desiredEnabled && currentEnabled != desiredEnabled { + return reconcileEndpoint(ctx, client, cfg, name, desiredEnabled) + } + slog.Info("endpoint reconciliation not requested or already satisfied; leaving existing state unchanged", + "region", cfg.region, "name", name, "endpointEnabled", currentEnabled) + return nil + default: + slog.Info("replication exists but is not ready for endpoint reconciliation; leaving it unchanged", + "region", cfg.region, "name", name, "state", state) + return nil + } +} + +// findReplicationNameByLocation returns the name of the replica whose +// Location matches cfg.region, or "" if none exists. Matching by location +// (rather than assuming the replica is named after the region) mirrors the +// replaced shell script, which discovered the replica via `az resource list +// --query "[?location=='$REPLICATION_REGION']"` instead of relying on naming +// convention; only the name is taken from the list response, matching the +// original's use of the list purely for discovery. +func findReplicationNameByLocation(ctx context.Context, client *armcontainerregistry.ReplicationsClient, cfg *config) (string, error) { + pager := client.NewListPager(cfg.resourceGroup, cfg.acrName, nil) + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return "", fmt.Errorf("list replications: %w", err) + } + for _, r := range page.Value { + if r != nil && r.Location != nil && r.Name != nil && strings.EqualFold(*r.Location, cfg.region) { + // Defensive: some ARM list responses return nested-resource + // names as "/" (the reason the + // replaced shell script did `cut -f 2 -d "/"` on its + // `az resource list` output). Take only the last path + // segment so Get/Update/Delete, which expect the bare + // replication name, always receive it correctly. + name := *r.Name + if idx := strings.LastIndex(name, "/"); idx != -1 { + name = name[idx+1:] + } + return name, nil + } + } + } + return "", nil +} + +// createReplication creates a new replica named after cfg.region with the +// requested regional endpoint state. +func createReplication(ctx context.Context, client *armcontainerregistry.ReplicationsClient, cfg *config, desiredEnabled bool) error { + if cfg.dryRun { + slog.Info("[DRY_RUN] would create replication", "region", cfg.region, "endpointEnabled", desiredEnabled) + return nil + } + slog.Info("creating replication", "region", cfg.region, "endpointEnabled", desiredEnabled) + poller, err := client.BeginCreate(ctx, cfg.resourceGroup, cfg.acrName, cfg.region, armcontainerregistry.Replication{ + Location: to.Ptr(cfg.region), + Properties: &armcontainerregistry.ReplicationProperties{ + RegionEndpointEnabled: to.Ptr(desiredEnabled), + }, + }, nil) + if err != nil { + return fmt.Errorf("create replication %q: %w", cfg.region, err) + } + if _, err := poller.PollUntilDone(ctx, nil); err != nil { + return fmt.Errorf("create replication %q: %w", cfg.region, err) + } + slog.Info("successfully created replication", "region", cfg.region) + return nil +} + +// deleteReplication deletes the replica by its actual resource name. +func deleteReplication(ctx context.Context, client *armcontainerregistry.ReplicationsClient, cfg *config, name string) error { + if cfg.dryRun { + slog.Info("[DRY_RUN] would delete replication", "region", cfg.region, "name", name) + return nil + } + poller, err := client.BeginDelete(ctx, cfg.resourceGroup, cfg.acrName, name, nil) + if err != nil { + return fmt.Errorf("delete replication %q: %w", name, err) + } + if _, err := poller.PollUntilDone(ctx, nil); err != nil { + return fmt.Errorf("delete replication %q: %w", name, err) + } + slog.Info("successfully deleted replication", "region", cfg.region, "name", name) + return nil +} + +// reconcileEndpoint updates an existing replica's regional endpoint to the +// desired state. +func reconcileEndpoint(ctx context.Context, client *armcontainerregistry.ReplicationsClient, cfg *config, name string, desiredEnabled bool) error { + if cfg.dryRun { + slog.Info("[DRY_RUN] would reconcile replication regional endpoint", "region", cfg.region, "name", name, "desiredEnabled", desiredEnabled) + return nil + } + slog.Info("reconciling replication regional endpoint", "region", cfg.region, "name", name, "desiredEnabled", desiredEnabled) + poller, err := client.BeginUpdate(ctx, cfg.resourceGroup, cfg.acrName, name, armcontainerregistry.ReplicationUpdateParameters{ + Properties: &armcontainerregistry.ReplicationUpdateParametersProperties{ + RegionEndpointEnabled: to.Ptr(desiredEnabled), + }, + }, nil) + if err != nil { + return fmt.Errorf("update replication %q: %w", name, err) + } + if _, err := poller.PollUntilDone(ctx, nil); err != nil { + return fmt.Errorf("update replication %q: %w", name, err) + } + slog.Info("successfully reconciled replication regional endpoint", "region", cfg.region, "enabled", desiredEnabled) + return nil +} + +// isNotFound reports whether err is an ARM 404 response, i.e. the replica +// does not exist yet. +func isNotFound(err error) bool { + var respErr *azcore.ResponseError + return errors.As(err, &respErr) && respErr.StatusCode == 404 +} diff --git a/dev-infrastructure/scripts/acr-replication/main_test.go b/dev-infrastructure/scripts/acr-replication/main_test.go new file mode 100644 index 00000000000..355b51b9d65 --- /dev/null +++ b/dev-infrastructure/scripts/acr-replication/main_test.go @@ -0,0 +1,107 @@ +// Copyright 2026 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "strings" + "testing" +) + +func TestParseEnvConfig(t *testing.T) { + t.Run("missing required", func(t *testing.T) { + _, err := parseEnvConfig(func(k string) string { + if k == "ACR_NAME" { + return "myacr" + } + return "" + }) + if err == nil { + t.Fatalf("expected error for missing required vars") + } + for _, want := range []string{"SUBSCRIPTION_ID", "RESOURCE_GROUP", "REPLICATION_REGION"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q should mention %q", err.Error(), want) + } + } + }) + + t.Run("complete config with no disabled regions", func(t *testing.T) { + env := map[string]string{ + "SUBSCRIPTION_ID": "sub", + "RESOURCE_GROUP": "rg", + "ACR_NAME": "myacr", + "REPLICATION_REGION": "eastus2", + } + c, err := parseEnvConfig(func(k string) string { return env[k] }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if c.subscriptionID != "sub" || c.resourceGroup != "rg" || c.acrName != "myacr" || c.region != "eastus2" { + t.Fatalf("unexpected config: %+v", c) + } + if !c.desiredEndpointEnabled() { + t.Fatalf("expected endpoint enabled by default") + } + }) + + t.Run("region in disabled list", func(t *testing.T) { + env := map[string]string{ + "SUBSCRIPTION_ID": "sub", + "RESOURCE_GROUP": "rg", + "ACR_NAME": "myacr", + "REPLICATION_REGION": "eastus2euap", + "ENDPOINT_DISABLED_REGIONS": "eastus2euap westus3", + } + c, err := parseEnvConfig(func(k string) string { return env[k] }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if c.desiredEndpointEnabled() { + t.Fatalf("expected endpoint disabled for eastus2euap") + } + }) + + t.Run("region not in disabled list", func(t *testing.T) { + env := map[string]string{ + "SUBSCRIPTION_ID": "sub", + "RESOURCE_GROUP": "rg", + "ACR_NAME": "myacr", + "REPLICATION_REGION": "eastus2", + "ENDPOINT_DISABLED_REGIONS": "eastus2euap westus3", + } + c, err := parseEnvConfig(func(k string) string { return env[k] }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !c.desiredEndpointEnabled() { + t.Fatalf("expected endpoint enabled for eastus2") + } + }) +} + +func TestIsNotFound(t *testing.T) { + if isNotFound(nil) { + t.Fatalf("nil error should not be not-found") + } + if isNotFound(errPlain("boom")) { + t.Fatalf("plain error should not be not-found") + } +} + +// errPlain is a minimal error type for TestIsNotFound, distinct from +// *azcore.ResponseError so errors.As cannot match it. +type errPlain string + +func (e errPlain) Error() string { return string(e) } diff --git a/dev-infrastructure/scripts/manage-acr-replication.sh b/dev-infrastructure/scripts/manage-acr-replication.sh deleted file mode 100755 index 31ed3c96554..00000000000 --- a/dev-infrastructure/scripts/manage-acr-replication.sh +++ /dev/null @@ -1,172 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o nounset -set -o pipefail - -# Function to display usage -usage() { - echo "Usage: Set environment variables and run the script" - echo "" - echo "Required environment variables:" - echo " ACR_NAME: Name of the Azure Container Registry" - echo " REPLICATION_REGION: Azure region name of the replication to check/create" - echo "" - echo "This script will:" - echo " 1. Delete any failed replications in the specified region" - echo " 2. Create a new replication if none exists in that region" - echo " Note: Replica will be named after the region name" - echo "" - echo "Example:" - echo " export ACR_NAME=myacr" - echo " export REPLICATION_REGION=eastus2" - echo " $0" - exit 1 -} - -# Check if required environment variables are set -if [ -z "${ACR_NAME:-}" ]; then - echo "Error: ACR_NAME environment variable is not set" - usage -fi - -if [ -z "${REPLICATION_REGION:-}" ]; then - echo "Error: REPLICATION_REGION environment variable is not set" - usage -fi - -# Check if DRY_RUN mode is enabled -if [ -n "${DRY_RUN:-}" ]; then - echo "DRY_RUN mode enabled - will only show what would be deleted, not actually delete anything" - DRY_RUN_MODE=true -else - DRY_RUN_MODE=false -fi - -# Function to execute or just log a command based on DRY_RUN mode -execute() { - if [ "$DRY_RUN_MODE" = true ]; then - echo "[DRY_RUN] Command: $*" - else - "$@" - fi -} - -# --region-endpoint-enabled was renamed to --global-endpoint-routing in az CLI 2.86.0 -# and removed in 2.87.0. Detect which flag the installed az CLI supports. -if az acr replication create --help 2>&1 | grep -q -- "--global-endpoint-routing"; then - ENDPOINT_ROUTING_FLAG="--global-endpoint-routing" -else - ENDPOINT_ROUTING_FLAG="--region-endpoint-enabled" -fi - -# Determine the desired regional data-endpoint state for this replica. Regions -# listed (space-separated) in ENDPOINT_DISABLED_REGIONS must keep their regional -# endpoint disabled so a co-located canary replica (e.g. eastus2euap) never -# serves ACR global routing for a neighbouring prod region. Defaults to enabled. -DESIRED_ENDPOINT_ENABLED=true -for disabled_region in ${ENDPOINT_DISABLED_REGIONS:-}; do - if [ "$disabled_region" = "$REPLICATION_REGION" ]; then - DESIRED_ENDPOINT_ENABLED=false - break - fi -done -echo "Desired regional endpoint for $REPLICATION_REGION: enabled=$DESIRED_ENDPOINT_ENABLED" - -# Function to create a new replication -create_replication() { - echo "Creating replication $REPLICATION_REGION for ACR $ACR_NAME in region $REPLICATION_REGION (endpoint enabled=$DESIRED_ENDPOINT_ENABLED)..." - execute az acr replication create \ - --registry "$ACR_NAME" \ - --resource-group "$RESOURCE_GROUP" \ - --location "$REPLICATION_REGION" \ - --name "$REPLICATION_REGION" \ - "$ENDPOINT_ROUTING_FLAG" "$DESIRED_ENDPOINT_ENABLED" - - echo "Successfully created replication $REPLICATION_REGION for ACR $ACR_NAME in region $REPLICATION_REGION" -} - -# Function to reconcile an existing replica's regional endpoint to the desired state -reconcile_replication_endpoint() { - local replica_name="$1" - local current_enabled="$2" - if [ "$current_enabled" = "$DESIRED_ENDPOINT_ENABLED" ]; then - echo "Replica $replica_name regional endpoint already at desired state (enabled=$DESIRED_ENDPOINT_ENABLED)" - return 0 - fi - echo "Reconciling replica $replica_name regional endpoint: $current_enabled -> $DESIRED_ENDPOINT_ENABLED" - execute az acr replication update \ - --registry "$ACR_NAME" \ - --resource-group "$RESOURCE_GROUP" \ - --name "$replica_name" \ - "$ENDPOINT_ROUTING_FLAG" "$DESIRED_ENDPOINT_ENABLED" - echo "Successfully reconciled replica $replica_name regional endpoint to enabled=$DESIRED_ENDPOINT_ENABLED" -} - -echo "Managing ACR replication for $ACR_NAME in region $REPLICATION_REGION..." - -# Get the resource group and location for the ACR -echo "Getting ACR information for $ACR_NAME..." -ACR_INFO=$(az acr show --name "$ACR_NAME" --query '{resourceGroup: resourceGroup, location: location}' -o json) -RESOURCE_GROUP=$(echo "$ACR_INFO" | jq -r '.resourceGroup') -ACR_HOME_REGION=$(echo "$ACR_INFO" | jq -r '.location') -echo "ACR $ACR_NAME is in resource group: $RESOURCE_GROUP, home region: $ACR_HOME_REGION" - -# Check if target region is the same as ACR home region -if [ "$REPLICATION_REGION" = "$ACR_HOME_REGION" ]; then - echo "The ACR is homed in the region $REPLICATION_REGION - replication is only needed for different regions" - exit 0 -fi - -# Check if any replication exists in the region -echo "Checking for existing replications in region $REPLICATION_REGION..." -# we need to query the existance of a replica via az resource list instead az acr replication list -# because the list operation is bugged and reports the wrong replication state at times -REPLICATION_INFO=$(az resource list \ - --resource-group "$RESOURCE_GROUP" \ - --resource-type "Microsoft.ContainerRegistry/registries/replications" \ - --query "[?location=='$REPLICATION_REGION' && contains(id, '/registries/$ACR_NAME/')] | [0]" \ - --output json -) - -if [ -n "$REPLICATION_INFO" ] && [ "$REPLICATION_INFO" != "null" ]; then - REPLICATION_RESOURCE_ID=$(echo "$REPLICATION_INFO" | jq -r '.id') - REPLICATION_NAME=$(echo "$REPLICATION_INFO" | jq -r '.name' | cut -f 2 -d "/") - # we need to query the replication state from the replica resource id and not from the list operation or the ACR - # there are bugs flying around that report the wrong replication state on the list operation - REPLICATION_DETAILS=$(az resource show \ - --ids "$REPLICATION_RESOURCE_ID" \ - --query "{provisioningState:properties.provisioningState, regionEndpointEnabled:properties.regionEndpointEnabled}" \ - --output json - ) - REPLICATION_STATE=$(echo "$REPLICATION_DETAILS" | jq -r '.provisioningState') - REPLICATION_ENDPOINT_ENABLED=$(echo "$REPLICATION_DETAILS" | jq -r '.regionEndpointEnabled') - echo "Found existing replication $REPLICATION_NAME ($REPLICATION_RESOURCE_ID) in state $REPLICATION_STATE with endpoint enabled=$REPLICATION_ENDPOINT_ENABLED" - - # Only check for failed replications if one exists - if [ "$REPLICATION_STATE" = "Failed" ]; then - echo "Replication $REPLICATION_RESOURCE_ID is in failed state. Deleting it..." - execute az acr replication delete \ - --registry "$ACR_NAME" \ - --resource-group "$RESOURCE_GROUP" \ - --name "$REPLICATION_NAME" - echo "Successfully deleted failed replication $REPLICATION_NAME" - - # After deleting failed replication, create a new one - create_replication - elif [ "$REPLICATION_STATE" = "Succeeded" ]; then - echo "Replication already exists and is in good state: $REPLICATION_NAME (state: $REPLICATION_STATE)" - if [ "$DESIRED_ENDPOINT_ENABLED" = "false" ]; then - reconcile_replication_endpoint "$REPLICATION_NAME" "$REPLICATION_ENDPOINT_ENABLED" - else - echo "Endpoint reconciliation not requested for $REPLICATION_NAME; leaving existing enabled=$REPLICATION_ENDPOINT_ENABLED state unchanged" - fi - exit 0 - else - echo "Replication already exists but is not ready for endpoint reconciliation: $REPLICATION_NAME (state: $REPLICATION_STATE)" - exit 0 - fi -else - echo "No replication exists in region $REPLICATION_REGION. Creating new replication..." - create_replication -fi diff --git a/go.work b/go.work index 2b99773ea0a..4fa427d1e8c 100644 --- a/go.work +++ b/go.work @@ -4,6 +4,7 @@ use ( ./admin/client ./admin/server ./backend + ./dev-infrastructure/scripts/acr-replication ./dev-infrastructure/scripts/grafana-group-roles ./dev-infrastructure/scripts/postgres-access ./fleet