From 4c5bf9f1546333dc058c467d04dcb11f5f008a0f Mon Sep 17 00:00:00 2001 From: Marcelo de Castro Loebens Date: Wed, 21 May 2025 15:54:57 -0400 Subject: [PATCH] Add haproxy config for kube-apiserver Include puppet class to configure HAproxy to proxy requests for kube-apiserver. Requests coming from OAM to port 6443 will reach HAproxy on SSL Bridge mode, and will be redirected to port 6443 on Cluster Host network after SSL termination in HAproxy. Requests coming from Cluster Host network on port 6443 will reach HAproxy on SSL passtrough mode, and be forwarded to kube-apiserver (Cluster Host network on port 16443). Test plan: PASS: Deploy AIO-SX w/IPv4 Check k8s entries in haproxy config Check curl to https://:6443 PASS: Deploy AIO-SX w/IPv6 Check k8s entries in haproxy config Check curl to https://[]:6443 PASS: Deploy AIO-DX Check k8s entries in haproxy config Check curl to https://:6443 Perform swact Check k8s entries in haproxy config Check curl to https://:6443 PASS: Deploy DC Systemcontroller + SX subcloud Verify subcloud online and in sync Check k8s entries in haproxy config Check curl to https://:6443, in SystemController and subcloud. Story: 2011399 Task: 52241 Change-Id: Idf8b8af0e50b07fdb998d8c5acc974c66a78c056 Signed-off-by: Marcelo de Castro Loebens --- .gitignore | 2 + .../src/modules/platform/manifests/haproxy.pp | 1 + .../modules/platform/manifests/kubernetes.pp | 125 ++++++++++++++++++ 3 files changed, 128 insertions(+) diff --git a/.gitignore b/.gitignore index 81ccda05c..992b30f6d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ .tox .stestr +# IDE +.vscode/* diff --git a/puppet-manifests/src/modules/platform/manifests/haproxy.pp b/puppet-manifests/src/modules/platform/manifests/haproxy.pp index 47ba69f15..357162e91 100644 --- a/puppet-manifests/src/modules/platform/manifests/haproxy.pp +++ b/puppet-manifests/src/modules/platform/manifests/haproxy.pp @@ -265,6 +265,7 @@ class platform::haproxy::runtime { include ::openstack::keystone::haproxy include ::openstack::barbican::haproxy include ::platform::smapi::haproxy + include ::platform::kubernetes::haproxy class {'::platform::haproxy::reload': stage => post diff --git a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp index bbd73ae54..237136868 100644 --- a/puppet-manifests/src/modules/platform/manifests/kubernetes.pp +++ b/puppet-manifests/src/modules/platform/manifests/kubernetes.pp @@ -436,6 +436,129 @@ class platform::kubernetes::set_crt_permissions { } } +class platform::kubernetes::haproxy { + include ::platform::params + include ::platform::network::oam::params + include ::platform::network::oam::ipv4::params + include ::platform::network::oam::ipv6::params + include ::platform::network::cluster_host::params + include ::platform::network::cluster_host::ipv4::params + include ::platform::network::cluster_host::ipv6::params + + # FLOATING + $ipv4_oam_floating_ip = $::platform::network::oam::ipv4::params::controller_address + $ipv6_oam_floating_ip = $::platform::network::oam::ipv6::params::controller_address + $ipv4_cluster_floating_ip = $::platform::network::cluster_host::ipv4::params::controller_address + $ipv6_cluster_floating_ip = $::platform::network::cluster_host::ipv6::params::controller_address + $primary_cluster_floating_ip = $::platform::network::cluster_host::params::controller_address + + # HOST + $controller_0_hostname = $::platform::params::controller_0_hostname + $controller_1_hostname = $::platform::params::controller_1_hostname + + case $::hostname { + $controller_0_hostname: { + $ipv4_oam_host_ip = $::platform::network::oam::ipv4::params::controller0_address + $ipv6_oam_host_ip = $::platform::network::oam::ipv6::params::controller0_address + $ipv4_cluster_host_ip = $::platform::network::cluster_host::ipv4::params::controller0_address + $ipv6_cluster_host_ip = $::platform::network::cluster_host::ipv6::params::controller0_address + if $::platform::params::system_mode == 'simplex' { + $primary_cluster_host_ip = $::platform::network::cluster_host::params::controller_address + } else { + $primary_cluster_host_ip = $::platform::network::cluster_host::params::controller0_address + } + } + $controller_1_hostname: { + $ipv4_oam_host_ip = $::platform::network::oam::ipv4::params::controller1_address + $ipv6_oam_host_ip = $::platform::network::oam::ipv6::params::controller1_address + $ipv4_cluster_host_ip = $::platform::network::cluster_host::ipv4::params::controller1_address + $ipv6_cluster_host_ip = $::platform::network::cluster_host::ipv6::params::controller1_address + $primary_cluster_host_ip = $::platform::network::cluster_host::params::controller1_address + } + default: { + fail("Hostname must be either ${controller_0_hostname} or ${controller_1_hostname}") + } + } + + # SSL Bridge (OAM -> HAproxy) + $haproxy_port = '6443' + $ssl_server_option = 'ssl crt /etc/ssl/private/server-cert.pem' + $ssl_client_option = 'check-ssl ssl verify required ca-file /etc/kubernetes/pki/ca.crt' + + $proto_header = 'X-Forwarded-Proto https' + $hsts_option = 'Strict-Transport-Security max-age=63072000;\ includeSubDomains' + $http_frontend_options = { + 'default_backend' => 'k8s-backend', + 'http-request' => "add-header ${proto_header}", + 'http-response' => "add-header ${hsts_option}", + } + + $oam_ips = [ + $ipv4_oam_floating_ip, + $ipv6_oam_floating_ip, + $ipv4_oam_host_ip, + $ipv6_oam_host_ip, + ].filter |$value| { $value != undef }.unique + + $oam_ips_serving_rest_api_certificate = $oam_ips.reduce( {} ) |Hash $memo, $item| { + $memo + {"${item}:${haproxy_port}" => $ssl_server_option} + } + + haproxy::frontend { 'k8s-frontend': + collect_exported => false, + name => 'k8s-frontend', + bind => $oam_ips_serving_rest_api_certificate, + options => $http_frontend_options, + } + + haproxy::backend { 'k8s-backend': + collect_exported => false, + name => 'k8s-backend', + options => { + 'server' => "s-k8s ${primary_cluster_floating_ip}:${haproxy_port} ${ssl_client_option}", + }, + } + + # SSL Passtrough (Cluster Host -> kube-apiserver) + $kube_apiserver_port = '16443' + + $tcp_ssl_hello = 'content accept if { req_ssl_hello_type 1 }' + $tcp_req_delay = 'inspect-delay 5s' + $tcp_frontend_options = { + 'mode' => 'tcp', + 'option' => 'tcplog', + 'default_backend' => 'k8s-backend-internal', + 'tcp-request' => [$tcp_ssl_hello,$tcp_req_delay], + } + + $cluster_ips = [ + $ipv4_cluster_floating_ip, + $ipv6_cluster_floating_ip, + $ipv4_cluster_host_ip, + $ipv6_cluster_host_ip, + ].filter |$value| { $value != undef }.unique + + $cluster_ips_ssl_passtrough = $cluster_ips.reduce( {} ) |Hash $memo, $item| { + $memo + {"${item}:${haproxy_port}" => ' '} + } + + haproxy::frontend { 'k8s-frontend-internal': + collect_exported => false, + name => 'k8s-frontend-internal', + bind => $cluster_ips_ssl_passtrough, + options => $tcp_frontend_options, + } + + haproxy::backend { 'k8s-backend-internal': + collect_exported => false, + name => 'k8s-backend-internal', + options => { + 'mode' => 'tcp', + 'server' => "s-k8s-internal ${primary_cluster_host_ip}:${kube_apiserver_port} check", + }, + } +} + class platform::kubernetes::master::init inherits ::platform::kubernetes::params { @@ -639,6 +762,7 @@ class platform::kubernetes::master contain ::platform::kubernetes::coredns contain ::platform::kubernetes::firewall contain ::platform::kubernetes::configuration + contain ::platform::kubernetes::haproxy Class['::platform::sysctl::controller::reserve_ports'] -> Class[$name] Class['::platform::k8splatform'] -> Class[$name] @@ -654,6 +778,7 @@ class platform::kubernetes::master -> Class['::platform::kubernetes::master::init'] -> Class['::platform::kubernetes::coredns'] -> Class['::platform::kubernetes::firewall'] + -> Class['::platform::kubernetes::haproxy'] } class platform::kubernetes::worker::init