From 484618eb80ed7a052da12e792952030762a231ef Mon Sep 17 00:00:00 2001
From: Erickson Silva de Oliveira <Erickson.SilvadeOliveira@windriver.com>
Date: Tue, 23 Sep 2025 16:54:51 -0300
Subject: [PATCH] MGR restful config-key improvements

The ceph mgr restful module certificate is set with the key
'mgr/restful/<mgr_name>/crt'. However, a race condition can
occur between the creation of the mgr and the execution of
the 'ceph-mgr-provision' job, causing restful to not running.

To avoid this, instead of using a specific mgr key, the global
key 'mgr/restful/crt' will be used, so there is no risk of
having the mgr without a certificate.

Test Plan:
 - PASS: AIO-SX with rook-ceph backend
 - PASS: AIO-DX with rook-ceph backend

Closes-Bug: 2125605

Depends-On: https://review.opendev.org/c/starlingx/root/+/962216

Change-Id: I5cdbac51d623bae1b0c0aab40bda4c1afb72a750
Signed-off-by: Erickson Silva de Oliveira <Erickson.SilvadeOliveira@windriver.com>
---
 .../templates/clusterrole.yaml                |   3 +
 .../templates/job-stx-ceph-mgr-provision.yaml | 134 ++++++++++++------
 .../rook-ceph-provisioner/values.yaml         |   2 +-
 ...ook-ceph-provisioner-static-overrides.yaml |   2 +-
 4 files changed, 99 insertions(+), 42 deletions(-)

diff --git a/helm-charts/custom/rook-ceph-provisioner-helm/rook-ceph-provisioner-helm/rook-ceph-provisioner/templates/clusterrole.yaml b/helm-charts/custom/rook-ceph-provisioner-helm/rook-ceph-provisioner-helm/rook-ceph-provisioner/templates/clusterrole.yaml
index 0339f43..55c692c 100644
--- a/helm-charts/custom/rook-ceph-provisioner-helm/rook-ceph-provisioner-helm/rook-ceph-provisioner/templates/clusterrole.yaml
+++ b/helm-charts/custom/rook-ceph-provisioner-helm/rook-ceph-provisioner-helm/rook-ceph-provisioner/templates/clusterrole.yaml
@@ -49,4 +49,7 @@ rules:
   - apiGroups: [""]
     resources: ["namespaces"]
     verbs: ["get", "create", "list", "update"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["list"]
 {{- end}}
diff --git a/helm-charts/custom/rook-ceph-provisioner-helm/rook-ceph-provisioner-helm/rook-ceph-provisioner/templates/job-stx-ceph-mgr-provision.yaml b/helm-charts/custom/rook-ceph-provisioner-helm/rook-ceph-provisioner-helm/rook-ceph-provisioner/templates/job-stx-ceph-mgr-provision.yaml
index 9ff99f4..b22f453 100644
--- a/helm-charts/custom/rook-ceph-provisioner-helm/rook-ceph-provisioner-helm/rook-ceph-provisioner/templates/job-stx-ceph-mgr-provision.yaml
+++ b/helm-charts/custom/rook-ceph-provisioner-helm/rook-ceph-provisioner-helm/rook-ceph-provisioner/templates/job-stx-ceph-mgr-provision.yaml
@@ -17,34 +17,77 @@ data:
   provision.sh: |-
     #!/bin/bash
 
-    echo "===================================="
+    set -x
+
+    CRT_CONFIG_KEY="mgr/restful/crt"
+    KEY_CONFIG_KEY="mgr/restful/key"
+    RESTFUL_PORT="7999"
+    REQUIRES_CERT_GENERATION=false
+    REQUIRES_MODULE_RESTART=false
 
     retries=60
     retry_count=1
-
+    # Attempt to access Ceph up to the max number of retries
     while [ $retry_count -le $retries ]; do
-        mgr_metadata=`ceph mgr metadata`
-        [ $? -eq 0 ] && break
+      ceph -s --connect-timeout 30
+      if [ $? -eq 0 ]; then
+        break
+      fi
 
-        echo "Retry #" $retry_count
-
-        sleep 10
-        let retry_count++
+      echo "Retry #$retry_count"
+      sleep 10
+      ((retry_count++))
     done
 
+    # Check if retries exceeded the limit
     if [ $retry_count -gt $retries ]; then
-      echo "Error: Ceph MGR does not correctly initialized."
+      echo "Error: Ceph is not responding."
       exit 1
     fi
 
-    mgr_hostnames=(`echo ${mgr_metadata} | jq -r '.[].hostname'`)
-    [[ $? -ne 0 || -z ${mgr_hostnames} ]] && echo "Error: Ceph MGR does not return the hostnames correctly." && exit 1
+    # Retrieve the hostnames
+    hostnames=$(kubectl get nodes --no-headers -o custom-columns=":metadata.name" | tr '\n' ' ')
+    if [[ $? -ne 0 || -z ${hostnames} ]]; then
+      echo "Error: Unable to get hostnames."
+      exit 1
+    fi
 
-    mgr_names=(`echo ${mgr_metadata} | jq -r '.[].name'`)
-    [[ $? -ne 0 || -z ${mgr_names} ]] && echo "Error: Ceph MGR does not return the names correctly." && exit 1
+    # Check if certificate and key exist
+    if ceph config-key exists "$CRT_CONFIG_KEY" && ceph config-key exists "$KEY_CONFIG_KEY"; then
+      CURRENT_CERTIFICATE="/tmp/current.crt"
+      ceph config-key get "$CRT_CONFIG_KEY" -o "$CURRENT_CERTIFICATE"
 
-    # Creating config file to be used to create the ssl certificate
-    cat > /tmp/all_hosts.conf << EOF
+      # Check the expiration date of the certificate
+      expiry_date=$(openssl x509 -in "$CURRENT_CERTIFICATE" -noout -enddate | cut -d= -f2)
+      expiry_timestamp=$(date -d "$expiry_date" +%s)
+      current_timestamp=$(date +%s)
+
+      # If it has expired, a new certificate needs to be generated.
+      if [[ $expiry_timestamp -lt $current_timestamp ]]; then
+        REQUIRES_CERT_GENERATION=true
+      fi
+
+      # Extract Subject Alternative Names (SANs) from the certificate
+      alts=$(openssl x509 -in "$CURRENT_CERTIFICATE" -noout -text | grep -A 1 "X509v3 Subject Alternative Name" | tail -n +2 | tr -d ' ' | tr ',' '\n')
+
+      # Check if all hostnames are present in the SAN
+      for hostname in $hostnames; do
+        if ! echo "$alts" | grep -q "$hostname"; then
+          echo "$hostname is not present in the certificate."
+          REQUIRES_CERT_GENERATION=true
+          break
+        fi
+      done
+    else
+      REQUIRES_CERT_GENERATION=true
+    fi
+
+    # If certificate needs to be generated
+    if [ "$REQUIRES_CERT_GENERATION" == "true" ]; then
+      echo "Creating SSL certificate..."
+
+      # Generate the OpenSSL config file
+      cat > /tmp/ssl.conf << EOF
     [req]
     req_extensions = v3_ca
     distinguished_name = req_distinguished_name
@@ -60,39 +103,50 @@ data:
     [alt_names]
     EOF
 
-    # Populating the config file including all hosts that there are MGRs
-    counter=1
-    for mgr in "${mgr_hostnames[@]}"; do
-      echo "DNS.${counter} = ${mgr}" >> /tmp/all_hosts.conf
-      counter=$((counter+1))
-    done
+      # Populate the config file with hostnames
+      counter=1
+      for hostname in $hostnames; do
+        echo "DNS.${counter} = ${hostname}" >> /tmp/ssl.conf
+        ((counter++))
+      done
 
-    openssl req -new -nodes -x509 -subj /O=IT/CN=ceph-restful -days 3650 -config /tmp/all_hosts.conf -out /tmp/ceph-restful.crt -keyout /tmp/ceph-restful.key -extensions v3_ca
-    [ $? == 1 ] && echo "Error: The certificate generation failed. Failing pod..." && exit 1
-    echo "Create ssl certificate using the config file created before"
+      # Create the certificate and key
+      openssl req -new -nodes -x509 -subj /O=IT/CN=ceph-restful -days 3650 -config /tmp/ssl.conf -out /tmp/ceph-restful.crt -keyout /tmp/ceph-restful.key -extensions v3_ca
+      if [ $? -ne 0 ]; then
+        echo "Error: The certificate generation failed."
+        exit 1
+      fi
 
-    for mgr in "${mgr_names[@]}"; do
-      ceph config-key set config/mgr/mgr/restful/${mgr}/crt -i /tmp/ceph-restful.crt
-      ceph config-key set config/mgr/mgr/restful/${mgr}/key -i /tmp/ceph-restful.key
+      echo "Setting the certificate and key..."
+      ceph config-key set "$CRT_CONFIG_KEY" -i /tmp/ceph-restful.crt
+      ceph config-key set "$KEY_CONFIG_KEY" -i /tmp/ceph-restful.key
 
-      ceph config-key set mgr/restful/${mgr}/crt -i /tmp/ceph-restful.crt
-      ceph config-key set mgr/restful/${mgr}/key -i /tmp/ceph-restful.key
-    done
-    echo "Set certificate and key for each MGR"
+      REQUIRES_MODULE_RESTART=true
+    else
+      echo "The certificate already exists, there is no need to regenerate it."
+    fi
 
-    ceph config set mgr mgr/restful/server_port 7999
-    echo "Set server port"
+    # Check and configure the restful module port
+    restful_url=$(ceph mgr services --format=json | jq -r '.restful')
+    if [[ "$restful_url" != *":$RESTFUL_PORT"* ]]; then
+      echo "Setting the mgr restful module port..."
+      ceph config set mgr mgr/restful/server_port "$RESTFUL_PORT"
+      REQUIRES_MODULE_RESTART=true
+    fi
 
-    ceph mgr module disable restful
-    echo "Disable restful"
-
-    ceph mgr module enable restful
-    echo "Enable restful"
+    # If the module needs to be restarted
+    if [ "$REQUIRES_MODULE_RESTART" == "true" ]; then
+      echo "Restarting restful module..."
+      ceph mgr module disable restful
+      ceph mgr module enable restful
+    fi
 
+    # Create the admin key. If it already exists, it will be kept.
+    echo "Creating admin key..."
     ceph restful create-key admin
-    echo "Ceph Mgr Provision Complete"
 
-    echo "===================================="
+    echo "Ceph mgr provision completed."
+    exit 0
 ---
 apiVersion: batch/v1
 kind: Job
diff --git a/helm-charts/custom/rook-ceph-provisioner-helm/rook-ceph-provisioner-helm/rook-ceph-provisioner/values.yaml b/helm-charts/custom/rook-ceph-provisioner-helm/rook-ceph-provisioner-helm/rook-ceph-provisioner/values.yaml
index 9f864d9..1b60d6e 100644
--- a/helm-charts/custom/rook-ceph-provisioner-helm/rook-ceph-provisioner-helm/rook-ceph-provisioner/values.yaml
+++ b/helm-charts/custom/rook-ceph-provisioner-helm/rook-ceph-provisioner-helm/rook-ceph-provisioner/values.yaml
@@ -38,7 +38,7 @@ rbac:
 images:
   tags:
     ceph_config_helper: docker.io/openstackhelm/ceph-config-helper:ubuntu_jammy_18.2.2-1-20240312
-    stx_ceph_manager: docker.io/starlingx/stx-ceph-manager:stx.11.0-v18.2.2-0
+    stx_ceph_manager: docker.io/starlingx/stx-ceph-manager:stx.11.0-v18.2.2-1
     kubectl: docker.io/bitnamilegacy/kubectl:1.29
     rook: docker.io/rook/ceph:v1.16.6
 
diff --git a/stx-rook-ceph-helm/stx-rook-ceph-helm/fluxcd-manifests/rook-ceph-provisioner/rook-ceph-provisioner-static-overrides.yaml b/stx-rook-ceph-helm/stx-rook-ceph-helm/fluxcd-manifests/rook-ceph-provisioner/rook-ceph-provisioner-static-overrides.yaml
index 8d25536..775c88a 100644
--- a/stx-rook-ceph-helm/stx-rook-ceph-helm/fluxcd-manifests/rook-ceph-provisioner/rook-ceph-provisioner-static-overrides.yaml
+++ b/stx-rook-ceph-helm/stx-rook-ceph-helm/fluxcd-manifests/rook-ceph-provisioner/rook-ceph-provisioner-static-overrides.yaml
@@ -78,7 +78,7 @@ imagePullSecrets:
 images:
   tags:
     ceph_config_helper: docker.io/openstackhelm/ceph-config-helper:ubuntu_jammy_18.2.2-1-20240312
-    stx_ceph_manager: docker.io/starlingx/stx-ceph-manager:stx.11.0-v18.2.2-0
+    stx_ceph_manager: docker.io/starlingx/stx-ceph-manager:stx.11.0-v18.2.2-1
     kubectl: docker.io/bitnamilegacy/kubectl:1.29
     rook: docker.io/rook/ceph:v1.16.6