Divingbell Integration Tests Update

- cleanup unused and obsolete scripts
    - update zuul job to use airskiff deployment job
    - uplift HTK commit to 80041dfbb5c22aa67dcd8dfa3332a943aa0d366c

Change-Id: I5320214f1c3ec5df5f3b31c2f66587d977b65607
Signed-off-by: Sergiy Markin <smarkin@mirantis.com>

.zuul.yaml

@@ -29,22 +29,14 @@
     check:
       jobs:
         - airship-divingbell-linter
-        - airship-divingbell-ubuntu:
+        - airship-divingbell-tests
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
-              - ^releasenotes/.*$
         - airship-divingbell-chart-build-gate
         - airship-divingbell-chart-build-latest-htk
     gate:
       jobs:
         - airship-divingbell-linter
         - airship-divingbell-chart-build-gate
-        - airship-divingbell-ubuntu:
+        - airship-divingbell-tests
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
-              - ^releasenotes/.*$
 
     post:
       jobs:
@@ -54,6 +46,12 @@
     name: airship-divingbell-linter
     run: tools/gate/playbooks/zuul-linter.yaml
     nodeset: divingbell-single-node
+    irrelevant-files: &irrelevant-files
+      - ^.*\.rst$
+      - ^doc/.*$
+      - ^etc/.*$
+      - ^releasenotes/.*$
+      - ^setup.cfg$
 
 - job:
     name: airship-divingbell-chart-build-gate
@@ -62,7 +60,8 @@
     run: ./tools/gate/playbooks/build-charts.yaml
     nodeset: divingbell-single-node
     vars:
-      HTK_COMMIT: 49c117443391cec75e0bd52bb4a9d033325927ad
+      HTK_COMMIT: 80041dfbb5c22aa67dcd8dfa3332a943aa0d366c
+    irrelevant-files: *irrelevant-files
 
 - job:
     name: airship-divingbell-chart-build-latest-htk
@@ -73,78 +72,34 @@
     nodeset: divingbell-single-node
     vars:
       HTK_COMMIT: master
+    irrelevant-files: *irrelevant-files
 
 - job:
-    name: airship-divingbell
+    name: airship-divingbell-base
+    parent: treasuremap-airskiff-infra-deploy-base
     abstract: true
-    description: Deploys all UC's and executes functional tests
+    description: Base job for deploying divingbell with treasuremap.
-    nodeset: treasuremap-airskiff-1node-ubuntu_jammy
-    roles:
-      - zuul: airship/kubernetes-entrypoint
-      - zuul: openstack/openstack-helm
-      - zuul: zuul/zuul-jobs
     required-projects:
-      - name: openstack/openstack-helm
-      - name: openstack/openstack-helm-plugin
       - name: airship/treasuremap
         override-checkout: v1.9
-    irrelevant-files:
+    roles:
-      - ^.*\.rst$
+      - zuul: airship/treasuremap
-      - ^doc/.*$
+    irrelevant-files: *irrelevant-files
-    timeout: 10800
-    pre-run:
-      - tools/gate/playbooks/prepare-hosts.yaml
-      - tools/gate/playbooks/mount-volumes.yaml
-    run:
-      - tools/gate/playbooks/deploy-env.yaml
-      - tools/gate/playbooks/osh-infra-gate-runner.yaml
-    post-run:
-      - tools/gate/playbooks/osh-infra-collect-logs.yaml
     vars:
-      extra_volume:
+      treasuremap_ref: v1.9
-        size: 80G
+      args:
-        type: Linux
+        chdir: ../../airship/porthole
-        mount_point: /opt/ext_vol
-      docker:
-        root_path: "/opt/ext_vol/docker"
-      containerd:
-        root_path: "/opt/ext_vol/containerd"
-      kubeadm:
-        pod_network_cidr: "10.244.0.0/16"
-        service_cidr: "10.96.0.0/16"
-      osh_plugin_repo: "{{ zuul.project.src_dir }}/../../openstack/openstack-helm-plugin"
-      loopback_setup: true
-      loopback_device: /dev/loop100
-      loopback_image: "/opt/ext_vol/openstack-helm/ceph-loop.img"
-      ceph_osd_data_device: /dev/loop100
-      kube_version_repo: "v1.30"
-      kube_version: "1.30.3-1.1"
-      calico_setup: true
-      calico_version: "v3.27.4"
-      cilium_setup: false
-      cilium_version: "1.16.0"
-      flannel_setup: false
-      flannel_version: v0.25.4
-      helm_version: "v3.15.4"
-      crictl_version: "v1.30.1"
       zuul_osh_relative_path: ../../openstack/openstack-helm
-      zuul_osh_infra_relative_path: ../../openstack/openstack-helm
       zuul_treasuremap_relative_path: ../../airship/treasuremap
-      gate_scripts_relative_path: .
+
-      run_helm_tests: "no"
-      distro_suffix: ubuntu_jammy
-      osh_params:
-        container_distro_name: ubuntu
-        container_distro_version: jammy
-      HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.16.4-linux-amd64.tar.gz
-      HTK_COMMIT: 49c117443391cec75e0bd52bb4a9d033325927ad
-      OSH_COMMIT: 49c117443391cec75e0bd52bb4a9d033325927ad
-      gate_scripts:
-        - ./tools/gate/scripts/run-gates.sh
 
 - job:
-    name: airship-divingbell-ubuntu
+    name: airship-divingbell-tests
-    parent: airship-divingbell
+    parent: airship-divingbell-base
+    description: Deploys divingbell with treasuremap and runs tests.
+    nodeset: treasuremap-airskiff-1node-ubuntu_jammy
+    run:
+      - tools/gate/playbooks/airship-divingbell-tests.yaml
 
 - job:
     name: divingbell-upload-git-mirror

tools/gate/playbooks/airship-divingbell-tests.yaml

@@ -14,10 +14,11 @@
 
 - hosts: primary
   tasks:
-    - name: Build Charts
+    - name: Run tests
       shell: |
-        set -e;
+        set -ex
+
         ./tools/gate/scripts/010-build-charts.sh
-        sudo ./tools/gate/scripts/020-test-divingbell.sh
+        sudo --preserve-env ./tools/gate/scripts/020-test-divingbell.sh
       args:
-        chdir: "{{ zuul_osh_relative_path | default(zuul.project.src_dir) }}"
+        chdir: "{{ zuul.project.src_dir }}"

tools/gate/playbooks/deploy-env.yaml

@@ -1,35 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
----
-- hosts: all
-  become: true
-  gather_facts: true
-
-  roles:
-    - ensure-python
-    - ensure-pip
-    - role: ensure-tox
-      become: false
-    - clear-firewall
-    - deploy-apparmor
-    - deploy-env
-
-  tasks:
-    - name: Install Packaging python module for tools/airship
-      block:
-        - pip:
-            name: packaging
-            version: 23.1
-            executable: pip3
-      become: true
-...

tools/gate/playbooks/mount-volumes.yaml

@@ -1,17 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
----
-- hosts: all
-  roles:
-    - mount-extra-volume
-...

tools/gate/playbooks/osh-infra-build.yaml

@@ -1,36 +0,0 @@
-# Copyright 2017 The Openstack-Helm Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- hosts: primary
-  vars_files:
-    - vars.yaml
-  vars:
-    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
-  gather_facts: True
-  roles:
-    - build-helm-packages
-  tags:
-    - build-helm-packages
-
-- hosts: all
-  vars_files:
-    - vars.yaml
-  vars:
-    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
-  gather_facts: False
-  become: yes
-  roles:
-    - build-images
-  tags:
-    - build-images

tools/gate/playbooks/osh-infra-collect-logs.yaml

@@ -1,30 +0,0 @@
-# Copyright 2017 The Openstack-Helm Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- hosts: all
-  vars:
-    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
-    logs_dir: "/tmp/logs"
-  roles:
-    - gather-host-logs
-    - helm-release-status
-    - describe-kubernetes-objects
-    - gather-pod-logs
-    - gather-prom-metrics
-  tags:
-    - gather-host-logs
-    - helm-release-status
-    - describe-kubernetes-objects
-    - gather-pod-logs
-    - gather-prom-metrics

tools/gate/playbooks/osh-infra-deploy-docker.yaml

@@ -1,43 +0,0 @@
-# Copyright 2017 The Openstack-Helm Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- hosts: all
-  vars_files:
-    - vars.yaml
-  vars:
-    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
-  gather_facts: False
-  become: yes
-  roles:
-    - deploy-python
-  tags:
-    - deploy-python
-
-- hosts: all
-  vars_files:
-    - vars.yaml
-  vars:
-    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
-  gather_facts: True
-  become: yes
-  roles:
-    - setup-firewall
-    - deploy-python-pip
-    - deploy-docker
-    - deploy-jq
-  tags:
-    - setup-firewall
-    - deploy-python-pip
-    - deploy-docker
-    - deploy-jq

tools/gate/playbooks/osh-infra-deploy-k8s.yaml

@@ -1,36 +0,0 @@
-# Copyright 2017 The Openstack-Helm Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- hosts: primary
-  vars_files:
-    - vars.yaml
-  vars:
-    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
-  gather_facts: True
-  roles:
-    - build-helm-packages
-    - deploy-kubeadm-aio-master
-  tags:
-    - build-helm-packages
-    - deploy-kubeadm-aio-master
-
-- hosts: nodes
-  vars_files:
-    - vars.yaml
-  vars:
-    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
-  roles:
-    - deploy-kubeadm-aio-node
-  tags:
-    - deploy-kubeadm-aio-node

tools/gate/playbooks/osh-infra-gate-runner.yaml

@@ -1,32 +0,0 @@
-# Copyright 2017 The Openstack-Helm Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- hosts: primary
-  tasks:
-    - name: "creating directory for run artifacts"
-      file:
-        path: "/tmp/artifacts"
-        state: directory
-    - name: Run gate scripts
-      include_role:
-        name: osh-run-script
-      vars:
-        gate_script_path: "{{ item }}"
-      with_items: "{{ gate_scripts }}"
-    - name: "Downloads artifacts to executor"
-      synchronize:
-        src: "/tmp/artifacts"
-        dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
-        mode: pull
-      ignore_errors: True

tools/gate/playbooks/osh-infra-upgrade-host.yaml

@@ -1,55 +0,0 @@
-# Copyright 2017 The Openstack-Helm Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- hosts: all
-  vars_files:
-    - vars.yaml
-  vars:
-    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
-  gather_facts: False
-  become: yes
-  roles:
-    - deploy-python
-  tags:
-    - deploy-python
-
-- hosts: all
-  vars_files:
-    - vars.yaml
-  vars:
-    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
-  gather_facts: True
-  become: yes
-  roles:
-    - upgrade-host
-    - start-zuul-console
-    - disable-local-nameserver
-    - setup-firewall
-  tags:
-    - upgrade-host
-    - start-zuul-console
-    - disable-local-nameserver
-    - setup-firewall
-
-- hosts: all
-  vars_files:
-    - vars.yaml
-  vars:
-    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
-  gather_facts: False
-  become: yes
-  roles:
-    - deploy-apparmor
-  tags:
-    - deploy-apparmor

tools/gate/playbooks/prepare-hosts.yaml

@@ -1,17 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
----
-- hosts: all
-  roles:
-    - start-zuul-console
-...

tools/gate/playbooks/roles/setup-firewall/tasks/main.yaml

@@ -1,29 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# NOTE(portdirect): This needs refinement but drops the firewall on zuul nodes
----
-- name: deploy iptables packages
-  include_role:
-    name: deploy-package
-    tasks_from: dist
-  vars:
-    packages:
-      deb:
-        - iptables
-      rpm:
-        - iptables
-- command: iptables -S
-- command: iptables -F
-- command: iptables -P INPUT ACCEPT
-- command: iptables -S
-...

tools/gate/playbooks/vars.yaml

@@ -1,5 +1,3 @@
-# Copyright 2017 The Openstack-Helm Authors.
-#
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -12,4 +10,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-null: null
+docker_daemon:
+  group: zuul
+  registry-mirrors:
+    - "http://{{ zuul_site_mirror_fqdn }}:8082/"
+  storage-driver: overlay2
+ansible_python_interpreter: python3
+...

tools/gate/scripts/run-gates.sh

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-# Copyright 2017 The Openstack-Helm Authors.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-set -xe
-
-./tools/gate/scripts/010-build-charts.sh
-sudo --preserve-env ./tools/gate/scripts/020-test-divingbell.sh

tools/helm_install.sh

@@ -17,7 +17,7 @@
 set -x
 
 HELM=$1
-HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.6.3-linux-amd64.tar.gz"}
+HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.18.4-linux-amd64.tar.gz"}
 
 
 function install_helm_binary {