From 5c21c23de84c9155080d2b417a3b33c3dd28c648 Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Fri, 29 Nov 2024 15:13:10 +0530 Subject: [PATCH] [ci] Add loadbalancer annotations as part of setup During restarts, the k8s services of type loadbalancer can get diferent IPs and the integration tests tries to communicate with older IPs. Add a step after deploying the bundle to add loadbalancer annotations to services of type loadbalancer so that IPs persist during restart of pods. Pin httpx version in tox.ini due to bug [1] in lightkube [1] https://github.com/gtsystem/lightkube/issues/78 Change-Id: I013ec1c1e9dbac3ae86c57abcd9d87a3b99f6e82 --- roles/k8s/tasks/main.yaml | 13 +++ tests/caas/tests.yaml | 1 + tests/ceph/tests.yaml | 1 + tests/core/tests.yaml | 1 + .../zaza/sunbeam/charm_tests/k8s/__init__.py | 0 .../zaza/sunbeam/charm_tests/k8s/setup.py | 95 +++++++++++++++++++ tests/misc/tests.yaml | 1 + tests/tempest/tests.yaml | 1 + tox.ini | 3 + 9 files changed, 116 insertions(+) create mode 100644 tests/local/zaza/sunbeam/charm_tests/k8s/__init__.py create mode 100644 tests/local/zaza/sunbeam/charm_tests/k8s/setup.py diff --git a/roles/k8s/tasks/main.yaml b/roles/k8s/tasks/main.yaml index 4afb3edd..7cf0541a 100644 --- a/roles/k8s/tasks/main.yaml +++ b/roles/k8s/tasks/main.yaml @@ -72,3 +72,16 @@ become: true changed_when: false failed_when: 'res.rc != 0 or "status: ready" not in res.stdout' + +- name: Get k8s config + ansible.builtin.command: + cmd: k8s config + become: true + register: kubeconfig + +- name: Copy kubeconfig to local + ansible.builtin.copy: + content: '{{ kubeconfig.stdout }}' + dest: '{{ ansible_env.HOME }}/kubeconfig' + mode: '0644' + owner: '{{ ansible_user }}' diff --git a/tests/caas/tests.yaml b/tests/caas/tests.yaml index 45270e45..9b90d312 100644 --- a/tests/caas/tests.yaml +++ b/tests/caas/tests.yaml @@ -3,6 +3,7 @@ gate_bundles: smoke_bundles: - smoke configure: + - zaza.sunbeam.charm_tests.k8s.setup.add_loadbalancer_annotations - zaza.openstack.charm_tests.keystone.setup.wait_for_all_endpoints - zaza.openstack.charm_tests.keystone.setup.add_tempest_roles tests: diff --git a/tests/ceph/tests.yaml b/tests/ceph/tests.yaml index a927dfab..89ac801f 100644 --- a/tests/ceph/tests.yaml +++ b/tests/ceph/tests.yaml @@ -4,6 +4,7 @@ smoke_bundles: - smoke # There is no storage provider at the moment so cannot run tests. configure: + - zaza.sunbeam.charm_tests.k8s.setup.add_loadbalancer_annotations - zaza.charm_tests.noop.setup.basic_setup tests: - zaza.charm_tests.noop.tests.NoopTest diff --git a/tests/core/tests.yaml b/tests/core/tests.yaml index 232c3e8f..f3d4c4fa 100644 --- a/tests/core/tests.yaml +++ b/tests/core/tests.yaml @@ -3,6 +3,7 @@ gate_bundles: smoke_bundles: - smoke configure: + - zaza.sunbeam.charm_tests.k8s.setup.add_loadbalancer_annotations - zaza.openstack.charm_tests.keystone.setup.wait_for_all_endpoints - zaza.openstack.charm_tests.keystone.setup.add_tempest_roles - zaza.openstack.charm_tests.nova.setup.create_flavors diff --git a/tests/local/zaza/sunbeam/charm_tests/k8s/__init__.py b/tests/local/zaza/sunbeam/charm_tests/k8s/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/local/zaza/sunbeam/charm_tests/k8s/setup.py b/tests/local/zaza/sunbeam/charm_tests/k8s/setup.py new file mode 100644 index 00000000..9f455479 --- /dev/null +++ b/tests/local/zaza/sunbeam/charm_tests/k8s/setup.py @@ -0,0 +1,95 @@ +# Copyright (c) 2024 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +from pathlib import ( + Path, +) + +import yaml +from lightkube.config.kubeconfig import ( + KubeConfig, +) +from lightkube.core.client import Client as KubeClient +from lightkube.core.exceptions import ( + ApiError, + ConfigError, +) +from lightkube.resources.core_v1 import ( + Service, +) + +METALLB_ANNOTATION = "metallb.universe.tf/loadBalancerIPs" + + +def add_loadbalancer_annotations( + model: str = "openstack", lb_annotation: str = METALLB_ANNOTATION +): + """Add loadbalancer annotations for all services of type LoadBalancer.""" + home: Path = Path(os.environ["HOME"]) + kubeconfig_file = home / "kubeconfig" + + if not kubeconfig_file.exists(): + logging.warning( + "No kubeconfig file present, not adding k8s lb service annotations." + ) + return + + try: + with kubeconfig_file.open() as f: + kubeconfig = KubeConfig.from_dict(yaml.safe_load(f)) + except (AttributeError, KeyError) as e: + logging.warning("Error in kubeconfig content", exc_info=True) + return + + try: + kube = KubeClient(kubeconfig, model, trust_env=False) + for service in kube.list( + Service, namespace=model, fields={"spec.type": "LoadBalancer"} + ): + if not service.metadata: + logging.warning(f"No metadata for service, {service}") + continue + + service_name = str(service.metadata.name) + service_annotations = service.metadata.annotations or {} + if lb_annotation not in service_annotations: + if not service.status: + logging.warning( + f"k8s service {service_name!r} has no status" + ) + continue + if not service.status.loadBalancer: + logging.warning( + f"k8s service {service_name!r} has no loadBalancer status" + ) + continue + if not service.status.loadBalancer.ingress: + logging.warning( + f"k8s service {service_name!r} has no loadBalancer ingress" + ) + continue + loadbalancer_ip = service.status.loadBalancer.ingress[0].ip + service_annotations[lb_annotation] = loadbalancer_ip + service.metadata.annotations = service_annotations + logging.info( + f"Patching {service_name!r} to use IP {loadbalancer_ip!r}" + ) + kube.patch(Service, service_name, obj=service) + except ConfigError: + logging.warning("Error creating k8s client", exc_info=True) + except ApiError: + logging.warning("Error getting services list", exc_info=True) diff --git a/tests/misc/tests.yaml b/tests/misc/tests.yaml index dd37faf5..1e9acdd1 100644 --- a/tests/misc/tests.yaml +++ b/tests/misc/tests.yaml @@ -3,6 +3,7 @@ gate_bundles: smoke_bundles: - smoke configure: + - zaza.sunbeam.charm_tests.k8s.setup.add_loadbalancer_annotations - zaza.charm_tests.noop.setup.basic_setup # https://bugs.launchpad.net/snap-openstack/+bug/2045206 # - zaza.openstack.charm_tests.keystone.setup.wait_for_all_endpoints diff --git a/tests/tempest/tests.yaml b/tests/tempest/tests.yaml index 7dadf221..cb4cc944 100644 --- a/tests/tempest/tests.yaml +++ b/tests/tempest/tests.yaml @@ -6,6 +6,7 @@ gate_bundles: smoke_bundles: - smoke configure: + - zaza.sunbeam.charm_tests.k8s.setup.add_loadbalancer_annotations - zaza.openstack.charm_tests.keystone.setup.wait_for_all_endpoints - zaza.openstack.charm_tests.keystone.setup.add_tempest_roles - zaza.openstack.charm_tests.nova.setup.create_flavors diff --git a/tox.ini b/tox.ini index b3f3d8da..72fd2929 100644 --- a/tox.ini +++ b/tox.ini @@ -89,6 +89,9 @@ deps = git+https://github.com/openstack-charmers/zaza.git#egg=zaza git+https://github.com/openstack-charmers/zaza-openstack-tests.git#egg=zaza.openstack git+https://opendev.org/openstack/tempest.git#egg=tempest + # Pin httpx version due to bug https://github.com/gtsystem/lightkube/issues/78 + httpx>=0.24.0,<0.28.0 + lightkube commands = functest-run-suite --help