test(failover): add happy path test for DNS Failover

Dockerfile

@@ -18,6 +18,10 @@ RUN curl -L https://github.com/cloudflare/cfssl/releases/download/v1.6.4/cfssl_1
 RUN curl -sL https://github.com/kube-burner/kube-burner/releases/download/v1.16.4/kube-burner-V1.16.4-linux-x86_64.tar.gz |\
     tar -xz -C /usr/bin --wildcards 'kube-burner' && chmod 0755 /usr/bin/kube-burner
 
+RUN TAG=$(curl -sI https://github.com/Kuadrant/dns-operator/releases/latest | grep -i ^location: | sed 's|.*/||' | tr -d '\r') && \
+    curl -sL "https://github.com/Kuadrant/dns-operator/releases/download/${TAG}/kubectl-kuadrant_dns-${TAG}-linux-amd64.tar.gz" | \
+    tar -xz -C /usr/local/bin kubectl-kuadrant_dns && chmod +x /usr/local/bin/kubectl-kuadrant_dns
+
 RUN python3.11 -m pip --no-cache-dir install poetry
 
 WORKDIR /opt/workdir/kuadrant-testsuite

config/settings.local.yaml.tpl

@@ -1,7 +1,7 @@
 #default:
 #  tester: "someuser"                          # Optional: name of the user, who is running the tests, defaults to whoami/uid
 #  kuadrantctl: kuadrantctl
-#  kubectl-dns: "kubectl-dns"
+#  kubectl-dns: "kubectl-kuadrant_dns"
 #  console:
 #    username: "CONSOLE_USERNAME"              # OpenShift console username for UI test login
 #    password: "CONSOLE_PASSWORD"              # OpenShift console password for UI tests login

config/settings.yaml

@@ -1,7 +1,7 @@
 default:
   dynaconf_merge: true
   kuadrantctl: "kuadrantctl"
-  kubectl-dns: "kubectl-dns"
+  kubectl-dns: "kubectl-kuadrant_dns"
   tools:
     project: "tools"
   cfssl: "cfssl"

testsuite/gateway/gateway_api/hostname.py

@@ -43,15 +43,20 @@ class DNSPolicyExposer(Exposer):
     """Exposing is done as part of DNSPolicy, so no work needs to be done here"""
 
     @cached_property
-    def base_domain(self) -> str:
+    def zone_domain(self) -> str:
+        """Returns the bare zone domain from the DNS provider secret annotation"""
         provider_secret_name = settings["control_plane"]["provider_secret"]
         try:
             secret = selector(f"secret/{provider_secret_name}", static_context=self.cluster.context).object()
         except OpenShiftPythonException as exc:
             raise OpenShiftPythonException(
                 f"Unable to find secret/{provider_secret_name} in namespace {self.cluster.project}"
             ) from exc
-        return f'{generate_tail(5)}.{secret.model["metadata"]["annotations"]["base_domain"]}'
+        return secret.model["metadata"]["annotations"]["base_domain"]
+
+    @cached_property
+    def base_domain(self) -> str:
+        return f"{generate_tail(5)}.{self.zone_domain}"
 
     def expose_hostname(self, name, exposable: Exposable) -> Hostname:
         return StaticHostname(

testsuite/tests/multicluster/conftest.py

@@ -1,12 +1,14 @@
 """Conftest for Multicluster tests"""
 
+import shutil
 from importlib import resources
 
 import pytest
 from dynaconf import ValidationError
 
 from testsuite.backend.mockserver import MockserverBackend, MockserverBackendConfig
 from testsuite.certificates import Certificate
+from testsuite.cli.kubectl_dns import KubectlDNS
 from testsuite.gateway import Exposer, Hostname
 from testsuite.gateway import TLSGatewayListener
 from testsuite.gateway.gateway_api.gateway import KuadrantGateway
@@ -26,6 +28,15 @@ def dns_server(testconfig, skip_or_fail):
         return skip_or_fail(f"DNS servers configuration is missing: {exc}")
 
 
+@pytest.fixture(scope="session")
+def kubectl_dns(testconfig, skip_or_fail):
+    """Return kubectl-kuadrant_dns CLI wrapper"""
+    binary_path = testconfig["kubectl-dns"]
+    if not shutil.which(binary_path):
+        skip_or_fail("kubectl-dns binary not found")
+    return KubectlDNS(binary_path)
+
+
 @pytest.fixture(scope="session")
 def cluster2(testconfig):
     """Kubernetes client for the primary namespace"""

testsuite/tests/multicluster/coredns/two_clusters/kubectl_dns/test_secret_generation.py

@@ -1,25 +1,13 @@
 """Test kubectl-dns add-cluster-secret command with basic coredns setup with 1 primary and 1 secondary clusters"""
 
-import shutil
-
 import dns.resolver
 import pytest
 
-from testsuite.cli.kubectl_dns import KubectlDNS
 from testsuite.tests.multicluster.coredns.conftest import IP1, IP2
 
 pytestmark = [pytest.mark.cli]
 
 
-@pytest.fixture(scope="session")
-def kubectl_dns(testconfig, skip_or_fail):
-    """Return Kuadrantctl wrapper with merged kubeconfig"""
-    binary_path = testconfig["kubectl-dns"]
-    if not shutil.which(binary_path):
-        skip_or_fail("kubectl-dns binary not found")
-    return KubectlDNS(binary_path)
-
-
 @pytest.fixture(scope="module")
 def kubeconfig_secrets(request, system_project, cluster, cluster2, kubectl_dns, blame):
     """Run add-cluster-secret command on merged kubeconfig to generate kubeconfig secret for the secondary cluster"""

testsuite/tests/multicluster/failover/conftest.py

@@ -0,0 +1,117 @@
+"""Conftest for DNS failover tests using groups"""
+
+import pytest
+
+from testsuite.kuadrant.policy.dns import has_record_condition
+from testsuite.utils import generate_tail
+
+MAX_REQUEUE_TIME = 10
+
+
+@pytest.fixture(scope="module")
+def group1():
+    """Random group name for cluster 1"""
+    return f"group1-{generate_tail(5)}"
+
+
+@pytest.fixture(scope="module")
+def group2():
+    """Random group name for cluster 2"""
+    return f"group2-{generate_tail(5)}"
+
+
+@pytest.fixture(scope="module")
+def provider_ref(cluster, dns_provider_secret):
+    """Returns providerRef in namespace/name format for the kubectl-kuadrant_dns CLI"""
+    return f"{cluster.project}/{dns_provider_secret}"
+
+
+@pytest.fixture(scope="module")
+def configure_dns_failover_groups(request, cluster, cluster2, system_project, group1, group2):
+    """Configure DNS Operator GROUP and MAX_REQUEUE_TIME on both clusters"""
+    for cluster_client, group_id in [(cluster, group1), (cluster2, group2)]:
+        sys_ns = cluster_client.change_project(system_project.project)
+
+        dns_deployment = sys_ns.get_deployment("dns-operator-controller-manager")
+
+        # add finalizer to remove the added variables and restart the controller, ORDER MATTERS
+        request.addfinalizer(dns_deployment.restart)
+        remove_patch = '{"data":{"GROUP":null,"MAX_REQUEUE_TIME":null}}'
+        request.addfinalizer(
+            lambda sys=sys_ns, p=remove_patch: sys.do_action(
+                "patch", "configmap", "dns-operator-controller-env", "--type=merge", "-p", p
+            )
+        )
+
+        # Patch the configmap to add/update GROUP and MAX_REQUEUE_TIME variables
+        add_patch = f'{{"data":{{"GROUP":"{group_id}","MAX_REQUEUE_TIME":"{MAX_REQUEUE_TIME}s"}}}}'
+        sys_ns.do_action("patch", "configmap", "dns-operator-controller-env", "--type=merge", "-p", add_patch)
+
+        dns_deployment.restart()
+
+
+@pytest.fixture(scope="module")
+def add_active_groups(
+    request, configure_dns_failover_groups, kubectl_dns, exposer, provider_ref, cluster, group1, group2
+):  # pylint: disable=unused-argument
+    """Create active-groups TXT record with group1 as the initial active group"""
+
+    def _cleanup():
+        for group in [group1, group2]:
+            kubectl_dns.run(
+                "remove-active-group",
+                group,
+                "-y",
+                "--domain",
+                exposer.zone_domain,
+                "--providerRef",
+                provider_ref,
+                env={"KUBECONFIG": cluster.kubeconfig_path},
+            )
+
+    request.addfinalizer(_cleanup)
+    result = kubectl_dns.run(
+        "add-active-group",
+        group1,
+        "-y",
+        "--domain",
+        exposer.zone_domain,
+        "--providerRef",
+        provider_ref,
+        env={"KUBECONFIG": cluster.kubeconfig_path},
+    )
+    assert result.returncode == 0, f"Failed to add active group: {result.stderr}"
+
+
+@pytest.fixture(scope="module", autouse=True)
+def commit(
+    request,
+    routes,
+    gateway,
+    gateway2,
+    dns_policy,
+    dns_policy2,
+    tls_policy,
+    tls_policy2,
+    add_active_groups,
+):  # pylint: disable=unused-argument
+    """Commits gateways and all policies required for the test"""
+    components = [gateway, gateway2, dns_policy, dns_policy2, tls_policy, tls_policy2]
+    for component in components:
+        request.addfinalizer(component.delete)
+        component.commit()
+
+    for component in [gateway, gateway2, tls_policy, tls_policy2]:
+        component.wait_for_ready()
+
+    dns_policy.wait_for_ready()
+
+    dns_policy2.wait_for_accepted()
+    assert dns_policy2.wait_until(
+        has_record_condition("Active", "False", "NotMemberOfActiveGroup", "Group is not included in active groups")
+    ), f"dns_policy2 should report inactive group, got: {dns_policy2.model.status.recordConditions}"
+    assert dns_policy2.wait_until(
+        has_record_condition(
+            "Ready", "False", "MemberOfInactiveGroup", "No further actions to take while in inactive group"
+        )
+    ), f"dns_policy2 should report inactive group not ready, got: {dns_policy2.model.status.recordConditions}"

testsuite/tests/multicluster/failover/test_dns_failover.py

@@ -0,0 +1,87 @@
+"""Test DNS failover from one cluster to another using DNS Operator active groups"""
+
+import time
+
+import dns.resolver
+import pytest
+
+from testsuite.kuadrant.policy.dns import has_record_condition
+from testsuite.tests.multicluster.failover.conftest import MAX_REQUEUE_TIME
+from testsuite.utils import sleep_ttl
+
+pytestmark = [pytest.mark.multicluster, pytest.mark.disruptive, pytest.mark.flaky(reruns=0)]
+
+
+def test_dns_failover(
+    cluster,
+    exposer,
+    client,
+    hostname,
+    provider_ref,
+    gateway,
+    gateway2,
+    dns_policy,
+    dns_policy2,
+    kubectl_dns,
+    group1,
+    group2,
+):  # pylint: disable=too-many-locals
+    """
+    Test DNS failover from group1 (cluster1) to group2 (cluster2):
+    1. Verify initial state with DNS resolving to cluster1
+    2. Switch active group from group1 to group2
+    3. Verify DNS resolves to cluster2 after failover
+    """
+    gw1_ip = gateway.external_ip().split(":")[0]
+    gw2_ip = gateway2.external_ip().split(":")[0]
+
+    response = client.get("/get")
+    assert not response.has_dns_error(), response.error
+    assert response.status_code == 200
+
+    dns_ips = {ip.address for ip in dns.resolver.resolve(hostname.hostname)}
+    assert {gw1_ip} == dns_ips, f"Initially DNS should only resolve to cluster1 IP ({gw1_ip}), got {dns_ips}"
+
+    # make second cluster group active and first cluster group inactive
+    result = kubectl_dns.run(
+        "add-active-group",
+        group2,
+        "-y",
+        "--domain",
+        exposer.zone_domain,
+        "--providerRef",
+        provider_ref,
+        env={"KUBECONFIG": cluster.kubeconfig_path},
+    )
+    assert result.returncode == 0, f"Failed to add group2 to active groups: {result.stderr}"
+    result = kubectl_dns.run(
+        "remove-active-group",
+        group1,
+        "-y",
+        "--domain",
+        exposer.zone_domain,
+        "--providerRef",
+        provider_ref,
+        env={"KUBECONFIG": cluster.kubeconfig_path},
+    )
+    assert result.returncode == 0, f"Failed to remove group1 from active groups: {result.stderr}"
+    time.sleep(MAX_REQUEUE_TIME)  # wait for DNS operator to process the change
+
+    dns_policy2.wait_for_ready()
+    assert dns_policy.wait_until(
+        has_record_condition("Active", "False", "NotMemberOfActiveGroup", "Group is not included in active groups")
+    ), f"dns_policy should report inactive group, got: {dns_policy.model.status.recordConditions}"
+    assert dns_policy.wait_until(
+        has_record_condition(
+            "Ready", "False", "MemberOfInactiveGroup", "No further actions to take while in inactive group"
+        )
+    ), f"dns_policy should report inactive group not ready, got: {dns_policy.model.status.recordConditions}"
+
+    sleep_ttl(hostname.hostname)
+
+    response = client.get("/get")
+    assert not response.has_dns_error(), response.error
+    assert response.status_code == 200
+
+    dns_ips = {ip.address for ip in dns.resolver.resolve(hostname.hostname)}
+    assert {gw2_ip} == dns_ips, f"After failover DNS should only resolve to cluster2 IP ({gw2_ip}), got {dns_ips}"