diff --git a/roles/deploy-env/defaults/main.yaml b/roles/deploy-env/defaults/main.yaml
index 524de14d1a..e516642f19 100644
--- a/roles/deploy-env/defaults/main.yaml
+++ b/roles/deploy-env/defaults/main.yaml
@@ -74,4 +74,16 @@ tunnel_cluster_cidr: "172.24.5.1/24"
 
 dnsmasq_image: "quay.io/airshipit/neutron:2024.2-ubuntu_jammy"
 nginx_image: "quay.io/airshipit/nginx:alpine3.18"
+
+overlay_network_setup: true
+overlay_network_prefix: "10.248.0."
+overlay_network_vxlan_iface: vxlan42
+overlay_network_vxlan_id: 42
+# NOTE: This is to avoid conflicts with the vxlan overlay managed by Openstack
+# which uses 4789 by default. Some alternative implementations used to
+# leverage 8472, so let's use it.
+overlay_network_vxlan_port: 8472
+overlay_network_bridge_name: brvxlan
+overlay_network_bridge_ip: "{{ overlay_network_prefix }}{{ (groups['all'] | sort).index(inventory_hostname) + 1 }}"
+overlay_network_underlay_dev: "{{ hostvars[inventory_hostname]['ansible_default_ipv4']['interface'] }}"
 ...
diff --git a/roles/deploy-env/files/calico_patch.yaml b/roles/deploy-env/files/calico_patch.yaml
index bdada7422d..882d4a84f6 100644
--- a/roles/deploy-env/files/calico_patch.yaml
+++ b/roles/deploy-env/files/calico_patch.yaml
@@ -1,20 +1,10 @@
 ---
 spec:
   template:
-    metadata:
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "9091"
     spec:
       containers:
         - name: calico-node
           env:
-            - name: FELIX_PROMETHEUSMETRICSENABLED
-              value: "true"
-            - name: FELIX_PROMETHEUSMETRICSPORT
-              value: "9091"
-            - name: FELIX_IGNORELOOSERPF
-              value: "true"
             # we need Calico to skip this interface while discovering the
             # network changes on the host to prevent announcing unnecessary networks.
             - name: IP_AUTODETECTION_METHOD
diff --git a/roles/deploy-env/files/kubeadm_config.yaml b/roles/deploy-env/files/kubeadm_config.yaml
deleted file mode 100644
index 137e0781a5..0000000000
--- a/roles/deploy-env/files/kubeadm_config.yaml
+++ /dev/null
@@ -1,27 +0,0 @@
----
-apiVersion: kubeproxy.config.k8s.io/v1alpha1
-kind: KubeProxyConfiguration
-mode: ipvs
-ipvs:
-  strictARP: true
-...
----
-apiVersion: kubeadm.k8s.io/v1beta3
-kind: ClusterConfiguration
-networking:
-  serviceSubnet: "{{ kubeadm.service_cidr }}"  # --service-cidr
-  podSubnet: "{{ kubeadm.pod_network_cidr }}"  # --pod-network-cidr
-  dnsDomain: "cluster.local"
-...
----
-apiVersion: kubeadm.k8s.io/v1beta3
-kind: InitConfiguration
-nodeRegistration:
-  taints: []
-...
----
-apiVersion: kubeadm.k8s.io/v1beta3
-kind: JoinConfiguration
-nodeRegistration:
-  taints: []
-...
diff --git a/roles/deploy-env/files/kubeadm_config.yaml.j2 b/roles/deploy-env/files/kubeadm_config.yaml.j2
new file mode 100644
index 0000000000..1b32d264d7
--- /dev/null
+++ b/roles/deploy-env/files/kubeadm_config.yaml.j2
@@ -0,0 +1,38 @@
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+mode: ipvs
+ipvs:
+  strictARP: true
+...
+---
+apiVersion: kubeadm.k8s.io/v1beta4
+kind: ClusterConfiguration
+networking:
+  serviceSubnet: "{{ kubeadm.service_cidr }}"  # --service-cidr
+  podSubnet: "{{ kubeadm.pod_network_cidr }}"  # --pod-network-cidr
+  dnsDomain: "cluster.local"
+...
+---
+apiVersion: kubeadm.k8s.io/v1beta4
+kind: InitConfiguration
+nodeRegistration:
+  criSocket: unix:///run/containerd/containerd.sock
+  taints: []
+  ignorePreflightErrors:
+    - NumCPU
+localAPIEndpoint:
+{% if overlay_network_setup %}
+  advertiseAddress: "{{ overlay_network_prefix }}{{ (groups['all'] | sort).index(groups['k8s_control_plane'][0]) + 1 }}"
+{% endif %}
+  bindPort: 6443
+...
+---
+apiVersion: kubeadm.k8s.io/v1beta4
+kind: JoinConfiguration
+nodeRegistration:
+  criSocket: unix:///run/containerd/containerd.sock
+  taints: []
+  ignorePreflightErrors:
+    - NumCPU
+...
diff --git a/roles/deploy-env/tasks/calico.yaml b/roles/deploy-env/tasks/calico.yaml
index f79d6311c4..9d16c2b19f 100644
--- a/roles/deploy-env/tasks/calico.yaml
+++ b/roles/deploy-env/tasks/calico.yaml
@@ -7,6 +7,7 @@
   shell: |
     curl -LSs {{ calico_manifest_url }} -o /tmp/calico.yaml
     sed -i -e 's#docker.io/calico/#quay.io/calico/#g' /tmp/calico.yaml
+    sed -i '/CALICO_IPV4POOL_IPIP/{n;s/Always/Never/}' /tmp/calico.yaml
     export CONTAINER_RUNTIME_ENDPOINT=unix:///run/containerd/containerd.sock
     export IMAGE_SERVICE_ENDPOINT=unix:///run/containerd/containerd.sock
     awk '/image:/ { print $2 }' /tmp/calico.yaml | xargs -I{} crictl pull {}
@@ -22,6 +23,7 @@
         if [[ ! -f /tmp/calico.yaml ]]; then
           curl -LSs {{ calico_manifest_url }} -o /tmp/calico.yaml
           sed -i -e 's#docker.io/calico/#quay.io/calico/#g' /tmp/calico.yaml
+          sed -i '/CALICO_IPV4POOL_IPIP/{n;s/Always/Never/}' /tmp/calico.yaml
         fi
       args:
         executable: /bin/bash
@@ -47,6 +49,9 @@
     - name: Patch Calico
       command: kubectl -n kube-system patch daemonset calico-node --patch-file /tmp/calico_patch.yaml
 
+    - name: Delete Calico pods (for hard restart)
+      command: kubectl -n kube-system delete pods -l k8s-app=calico-node
+
     - name: Wait for Calico pods ready (after patch)
       command: kubectl -n kube-system wait --timeout=20s --for=condition=Ready pods -l k8s-app=calico-node
       register: calico_pods_wait
diff --git a/roles/deploy-env/tasks/k8s_control_plane.yaml b/roles/deploy-env/tasks/k8s_control_plane.yaml
index 563f09b2d6..600d11f803 100644
--- a/roles/deploy-env/tasks/k8s_control_plane.yaml
+++ b/roles/deploy-env/tasks/k8s_control_plane.yaml
@@ -21,7 +21,7 @@
 
 - name: Prepare kubeadm config
   template:
-    src: files/kubeadm_config.yaml
+    src: files/kubeadm_config.yaml.j2
     dest: /tmp/kubeadm_config.yaml
 
 - name: Initialize the Kubernetes cluster using kubeadm
diff --git a/roles/deploy-env/tasks/main.yaml b/roles/deploy-env/tasks/main.yaml
index d1caef39ae..9705890694 100644
--- a/roles/deploy-env/tasks/main.yaml
+++ b/roles/deploy-env/tasks/main.yaml
@@ -15,6 +15,11 @@
   include_tasks:
     file: prerequisites.yaml
 
+- name: Overlay network
+  include_tasks:
+    file: overlay.yaml
+  when: overlay_network_setup
+
 - name: Configure /etc/hosts
   template:
     src: files/hosts
diff --git a/roles/deploy-env/tasks/overlay.yaml b/roles/deploy-env/tasks/overlay.yaml
new file mode 100644
index 0000000000..51e1205bd6
--- /dev/null
+++ b/roles/deploy-env/tasks/overlay.yaml
@@ -0,0 +1,42 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: Create vxlan bridge
+  shell: |
+    ip link add name {{ overlay_network_bridge_name }} type bridge
+    ip link set dev {{ overlay_network_bridge_name }} up
+    ip addr add {{ overlay_network_bridge_ip }}/24 dev {{ overlay_network_bridge_name }}
+  args:
+    creates: "/sys/class/net/{{ overlay_network_bridge_name }}"
+
+- name: Create vxlan interface
+  shell: |
+    ip link add {{ overlay_network_vxlan_iface }} \
+      type vxlan \
+      id {{ overlay_network_vxlan_id }} \
+      dev {{ overlay_network_underlay_dev }} \
+      dstport {{ overlay_network_vxlan_port }} \
+      local {{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }}
+    ip link set {{ overlay_network_vxlan_iface }} up
+    ip link set {{ overlay_network_vxlan_iface }} master {{ overlay_network_bridge_name }}
+  args:
+    creates: "/sys/class/net/{{ overlay_network_vxlan_iface }}"
+
+- name: Populate FDB
+  shell: |
+    bridge fdb append 00:00:00:00:00:00 \
+      dev {{ overlay_network_vxlan_iface }} \
+      dst {{ hostvars[item]['ansible_host'] }}
+  loop: "{{ groups['all'] | sort }}"
+  when: item != inventory_hostname
+...
diff --git a/tools/deployment/component/compute-kit/compute-kit.sh b/tools/deployment/component/compute-kit/compute-kit.sh
index 571999f016..a845003164 100755
--- a/tools/deployment/component/compute-kit/compute-kit.sh
+++ b/tools/deployment/component/compute-kit/compute-kit.sh
@@ -57,7 +57,9 @@ helm upgrade --install nova ${OSH_HELM_REPO}/nova \
 tee /tmp/neutron.yaml << EOF
 network:
   interface:
-    tunnel: null
+    # the CI env overlay interface is used by default
+    # for internal cluster communication
+    tunnel: brvxlan
 conf:
   neutron:
     DEFAULT:
diff --git a/zuul.d/base.yaml b/zuul.d/base.yaml
index 776bd4598d..7563cf1931 100644
--- a/zuul.d/base.yaml
+++ b/zuul.d/base.yaml
@@ -73,6 +73,7 @@
       - playbooks/deploy-env.yaml
       - playbooks/run-scripts.yaml
     vars:
+      overlay_network_setup: true
       extra_volume:
         size: 80G
         type: Linux
diff --git a/zuul.d/infra_jobs.yaml b/zuul.d/infra_jobs.yaml
index da59c8793c..7fb32cc859 100644
--- a/zuul.d/infra_jobs.yaml
+++ b/zuul.d/infra_jobs.yaml
@@ -82,10 +82,7 @@
 - job:
     name: openstack-helm-mariadb-operator-2024-1-ubuntu_jammy
     parent: openstack-helm-deploy
-    nodeset: openstack-helm-3nodes-ubuntu_jammy
-    pre-run:
-      - playbooks/prepare-hosts.yaml
-      - playbooks/mount-volumes.yaml
+    nodeset: openstack-helm-5nodes-ubuntu_jammy
     vars:
       osh_params:
         openstack_release: "2024.1"
@@ -95,10 +92,10 @@
       gate_scripts:
         - ./tools/deployment/common/prepare-k8s.sh
         - ./tools/deployment/common/prepare-charts.sh
-        - ./tools/deployment/common/namespace-config.sh
-        - ./tools/deployment/ceph/ceph.sh
-        - ./tools/deployment/ceph/ceph-ns-activate.sh
         - ./tools/deployment/common/setup-client.sh
+        - ./tools/deployment/common/namespace-config.sh
+        - ./tools/deployment/ceph/ceph-rook.sh
+        - ./tools/deployment/ceph/ceph-adapter-rook.sh
         - ./tools/deployment/component/common/rabbitmq.sh
         - ./tools/deployment/component/common/memcached.sh
         - ./tools/deployment/db/mariadb-operator-cluster.sh
diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml
index fedf16edbb..188135174e 100644
--- a/zuul.d/project.yaml
+++ b/zuul.d/project.yaml
@@ -25,7 +25,9 @@
         - openstack-helm-tls-2024-1-ubuntu_jammy  # 3 nodes rook
         - openstack-helm-cinder-2024-1-ubuntu_jammy  # 5 nodes rook
         - openstack-helm-compute-kit-2024-1-ubuntu_jammy  # 3 nodes
-        - openstack-helm-compute-kit-cilium-2024-1-ubuntu_jammy  # 1 node + 3 nodes
+        # TODO: Configure Cilium not to setup it's own overlay and
+        # use existing VXLAN overlay interface for internal K8s communication
+        # - openstack-helm-compute-kit-cilium-2024-1-ubuntu_jammy  # 1 node + 3 nodes
         - openstack-helm-horizon-2024-1-ubuntu_jammy  # 1 node
         - openstack-helm-tacker-2024-1-ubuntu_jammy
         - openstack-helm-compute-kit-dpdk-2024-1-ubuntu_jammy  # 32GB node