Bootstrap mode

Change-Id: I0f6ca7d9054371ebc08392d853e0d855018b5fe0
Signed-off-by: Ruslan Aliev <raliev@mirantis.com>

charts/armada/templates/daemonset-bootstrap.yaml

@@ -0,0 +1,187 @@
+{{/*
+Copyright 2025 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.daemonset_bootstrap }}
+{{- $envAll := . }}
+{{- $labels :=  tuple $envAll "armada" "bootstrap" | include "helm-toolkit.snippets.kubernetes_metadata_labels" -}}
+{{- $mounts_armada_bootstrap := .Values.pod.mounts.bootstrap.armada }}
+{{- $mounts_manager_bootstrap := .Values.pod.mounts.bootstrap.manager }}
+{{- $mounts_monitor_bootstrap := .Values.pod.mounts.bootstrap.monitor }}
+{{- $mounts_apiserver_bootstrap := .Values.pod.mounts.bootstrap.apiserver }}
+
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: armada-bootstrap
+spec:
+  selector:
+    matchLabels:
+{{ $labels | indent 6 }}
+  template:
+    metadata:
+      labels:
+{{ $labels | indent 8 }}
+    spec:
+      automountServiceAccountToken: false
+      containers:
+      - command:
+        - /bin/bash
+        - -c
+        - |-
+          set -x
+
+          if [ ! -e "${ARMADA_LOGFILE}" ]; then
+            touch "${ARMADA_LOGFILE}"
+          fi
+          tail -f "${ARMADA_LOGFILE}" &
+
+          declare -i attempt=1
+
+          while true; do
+              sleep 10
+              if armada \
+                      apply \
+                      --target-manifest cluster-bootstrap \
+                      $([[ $attempt -le $ARMADA_METRICS_MAX_ATTEMPTS ]] && echo --metrics-output "${ARMADA_METRICS_OUTPUT_DIR}/armada-bootstrap-${attempt}.prom") \
+                      /etc/genesis/armada/assets/manifest.yaml &>> "${ARMADA_LOGFILE}"; then
+                  break
+              fi
+              attempt+=1
+          done
+          touch /ipc/armada-done
+          sleep 10000
+{{- if .Values.pod.env.armada_bootstrap }}
+        env:
+{{- range .Values.pod.env.armada_bootstrap }}
+        - name: {{ .name | quote }}
+          value: {{ .value | quote }}
+{{- end }}
+{{- end }}
+{{ tuple $envAll "api" | include "helm-toolkit.snippets.image" | indent 8 }}
+        name: armada
+{{ tuple $envAll $envAll.Values.pod.resources.armada_bootstrap | include "helm-toolkit.snippets.kubernetes_resources" | indent 8 }}
+{{ dict "envAll" $envAll "application" "armada_bootstrap" "container" "armada" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 8 }}
+{{ if $mounts_armada_bootstrap.volumeMounts }}
+        volumeMounts:
+{{ toYaml $mounts_armada_bootstrap.volumeMounts | indent 8 }}
+{{ end }}
+{{- if .Values.conf.armada.DEFAULT.enable_operator }}
+      - command:
+        - /bin/sh
+        - -c
+        - |-
+          set -x
+
+          if [ ! -e "${ARMADA_OPERATOR_LOGFILE}" ]; then
+            touch "${ARMADA_OPERATOR_LOGFILE}"
+          fi
+          tail -f "${ARMADA_OPERATOR_LOGFILE}" &
+
+          /manager \
+            --health-probe-bind-address=:8081 \
+            --metrics-bind-address=127.0.0.1:8080 \
+            --leader-elect \
+            --leader-elect-namespace=ucp 2>&1 | tee -a "${ARMADA_OPERATOR_LOGFILE}"
+{{- if .Values.pod.env.manager_bootstrap }}
+        env:
+{{- range .Values.pod.env.manager_bootstrap }}
+        - name: {{ .name | quote }}
+          value: {{ .value | quote }}
+{{- end }}
+{{- end }}
+{{ tuple $envAll "operator" | include "helm-toolkit.snippets.image" | indent 8 }}
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8081
+          initialDelaySeconds: 15
+          periodSeconds: 20
+        name: manager
+        readinessProbe:
+          httpGet:
+            path: /readyz
+            port: 8081
+          initialDelaySeconds: 5
+          periodSeconds: 10
+{{ tuple $envAll $envAll.Values.pod.resources.manager_bootstrap | include "helm-toolkit.snippets.kubernetes_resources" | indent 8 }}
+{{ dict "envAll" $envAll "application" "armada_bootstrap" "container" "manager" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 8 }}
+{{ if $mounts_manager_bootstrap.volumeMounts }}
+        volumeMounts:
+{{ toYaml $mounts_manager_bootstrap.volumeMounts | indent 8 }}
+{{ end }}
+{{- end }}
+      - command:
+        - /bin/sh
+        - -c
+        - |-
+          set -x
+
+          while ! [ -e /ipc/armada-done ]; do
+            sleep 5
+          done
+
+          rm -f /etc/kubernetes/manifests/bootstrap-armada.yaml
+          sleep 10000
+{{ tuple $envAll "monitor" | include "helm-toolkit.snippets.image" | indent 8 }}
+        name: monitor
+{{ tuple $envAll $envAll.Values.pod.resources.monitor_bootstrap | include "helm-toolkit.snippets.kubernetes_resources" | indent 8 }}
+{{ dict "envAll" $envAll "application" "armada" "container" "manager" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 8 }}
+{{ if $mounts_monitor_bootstrap.volumeMounts }}
+        volumeMounts:
+{{ toYaml $mounts_monitor_bootstrap.volumeMounts | indent 8 }}
+{{ end }}
+      - command:
+        {{- range .Values.bootstrap_apiserver.command_prefix }}
+        - {{ . }}
+        {{- end }}
+        {{- range .Values.bootstrap_apiserver.genesis_arguments }}
+        - {{ . }}
+        {{- end }}
+        {{- range .Values.bootstrap_apiserver.arguments }}
+        - {{ . }}
+        {{- end }}
+{{- if .Values.pod.env.apiserver_bootstrap }}
+        env:
+        {{- range .Values.pod.env.apiserver_bootstrap }}
+        - name: {{ .name | quote }}
+          value: {{ .value | quote }}
+        {{- end }}
+{{- end }}
+{{ tuple $envAll "apiserver" | include "helm-toolkit.snippets.image" | indent 8 }}
+        name: kubectl-apiserver
+{{ tuple $envAll $envAll.Values.pod.resources.apiserver_bootstrap | include "helm-toolkit.snippets.kubernetes_resources" | indent 8 }}
+{{ dict "envAll" $envAll "application" "armada" "container" "manager" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 8 }}
+{{ if $mounts_apiserver_bootstrap.volumeMounts }}
+        volumeMounts:
+{{ toYaml $mounts_apiserver_bootstrap.volumeMounts | indent 8 }}
+{{ end }}
+      dnsPolicy: Default
+      hostNetwork: true
+      nodeSelector:
+        {{ .Values.labels.bootstrap.node_selector_key }}: {{ .Values.labels.bootstrap.node_selector_value }}
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      securityContext: {}
+      terminationGracePeriodSeconds: 30
+      tolerations:
+      - operator: Exists
+{{ if $mounts_armada_bootstrap.volumes }}
+      volumes:
+{{ toYaml $mounts_armada_bootstrap.volumes | indent 8 }}
+{{ end }}
+
+{{- end }}

charts/armada/templates/deployment-api.yaml

@@ -105,9 +105,11 @@ spec:
       nodeSelector:
         {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
       terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.api.timeout | default "30" }}
+{{- if or (not .Values.conf.armada.DEFAULT.disable_keystone) (not (hasKey .Values.conf.armada.DEFAULT "disable_keystone")) }}
       initContainers:
 {{ tuple $envAll "api" $mounts_armada_api_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
 {{ dict "envAll" $envAll "application" "armada" "container" "armada_api_init" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+{{- end }}
       containers:
 {{- if .Values.conf.armada.DEFAULT.enable_operator }}
         - name: manager

charts/armada/templates/network_policy.yaml

@@ -1,16 +1,18 @@
-# Copyright 2017-2018 The Openstack-Helm Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+{{/*
+ Copyright 2017-2018 The Openstack-Helm Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/}}
 
 {{- if .Values.manifests.network_policy -}}
 {{- $netpol_opts := dict "envAll" . "name" "application" "label" "armada" -}}

charts/armada/values.yaml

@@ -23,6 +23,9 @@ labels:
   test:
     node_selector_key: ucp-control-plane
     node_selector_value: enabled
+  bootstrap:
+    node_selector_key: armada-bootstrap
+    node_selector_value: enabled
   node_selector_key: ucp-control-plane
   node_selector_value: enabled
 
@@ -43,6 +46,32 @@ images:
       - dep_check
       - image_repo_sync
 
+bootstrap_apiserver:
+  command_prefix:
+  - kube-apiserver
+  - --advertise-address=$(ADVERTISE_ADDRESS)
+  - --allow-privileged=true
+  - --anonymous-auth=false
+  - --bind-address=0.0.0.0
+  - --client-ca-file=/etc/kubernetes/apiserver/pki/cluster-ca.pem
+  - --etcd-cafile=/etc/kubernetes/apiserver/pki/etcd-client-ca.pem
+  - --etcd-certfile=/etc/kubernetes/apiserver/pki/etcd-client.pem
+  - --etcd-keyfile=/etc/kubernetes/apiserver/pki/etcd-client-key.pem
+  - --kubelet-certificate-authority=/etc/kubernetes/apiserver/pki/kubelet-client-ca.pem
+  - --kubelet-client-certificate=/etc/kubernetes/apiserver/pki/kubelet-client.pem
+  - --kubelet-client-key=/etc/kubernetes/apiserver/pki/kubelet-client-key.pem
+  - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
+  - --service-account-issuer=https://kubernetes.default.svc.cluster.local
+  - --service-account-key-file=/etc/kubernetes/apiserver/pki/service-account.pub
+  - --service-account-signing-key-file=/etc/kubernetes/apiserver/pki/service-account.key
+  - --tls-cert-file=/etc/kubernetes/apiserver/pki/apiserver.pem
+  - --tls-private-key-file=/etc/kubernetes/apiserver/pki/apiserver-key.pem
+  genesis_arguments: []
+  arguments:
+  - --etcd-servers=https://localhost:12379
+  - --secure-port=6444
+  - --endpoint-reconciler-type=none
+
 network:
   api:
     ingress:
@@ -229,6 +258,12 @@ pod:
             initialDelaySeconds: 15
             periodSeconds: 10
   security_context:
+    armada_bootstrap:
+      container:
+        armada:
+          runAsUser: 0
+        manager:
+          runAsUser: 0
     armada:
       pod:
         runAsUser: 1000
@@ -247,6 +282,23 @@ pod:
           readOnlyRootFilesystem: true
           allowPrivilegeEscalation: false
   env:
+    armada_bootstrap:
+      - name: ARMADA_LOGFILE
+        value: /tmp/log/bootstrap-armada.log
+      - name: ARMADA_METRICS_OUTPUT_DIR
+        value: /tmp/metrics
+      - name: ARMADA_METRICS_MAX_ATTEMPTS
+        value: '10'
+      - name: KUBECONFIG
+        value: /root/.kube/config
+    manager_bootstrap:
+      - name: ARMADA_OPERATOR_LOGFILE
+        value: /tmp/log/bootstrap-armada-operator.log
+      - name: KUBECONFIG
+        value: /root/.kube/config
+    apiserver_bootstrap:
+      - name: KUBECONFIG
+        value: /etc/kubernetes/admin/config
     # NOTE(@drewwalters96): These configuration values change the Armada API's
     #    uWSGI configuration.
     armada_api:
@@ -273,6 +325,59 @@ pod:
 #      - name: NO_PROXY
 #        value: 10.96.0.1
   mounts:
+    bootstrap:
+      armada:
+        volumeMounts:
+        - name: assets
+          mountPath: /etc/genesis/armada/assets
+        - name: auth
+          mountPath: /root/.kube
+        - name: ipc
+          mountPath: /ipc
+        - name: log
+          mountPath: /tmp/log
+        - name: metrics
+          mountPath: /tmp/metrics
+        volumes:
+        - name: assets
+          hostPath:
+            path: /etc/genesis/armada/assets
+        - name: auth
+          hostPath:
+            path: /etc/genesis/armada/auth
+        - name: manifest
+          hostPath:
+            path: /etc/kubernetes/manifests
+        - name: ipc
+          emptyDir: { }
+        - name: log
+          hostPath:
+            path: /var/log/armada
+        - name: metrics
+          hostPath:
+            path: '/var/log/node-exporter-textfiles'
+        - name: config
+          hostPath:
+            path: /etc/genesis/apiserver
+      manager:
+        volumeMounts:
+        - name: auth
+          mountPath: /root/.kube
+        - name: log
+          mountPath: /tmp/log
+      monitor:
+        volumeMounts:
+        - mountPath: /ipc
+          name: ipc
+        - mountPath: /etc/kubernetes/manifests
+          name: manifest
+      apiserver:
+        volumeMounts:
+        - name: auth
+          mountPath: /etc/kubernetes/admin
+        - name: config
+          mountPath: /etc/kubernetes/apiserver
+          readOnly: true
     armada_api:
       init_container: null
       armada_api:
@@ -303,6 +408,34 @@ pod:
     prestop_sleep: 20
   resources:
     enabled: false
+    armada_bootstrap:
+      limits:
+        cpu: "8"
+        memory: "8Gi"
+      requests:
+        cpu: "100m"
+        memory: "128Mi"
+    manager_bootstrap:
+      limits:
+        cpu: "8"
+        memory: "8Gi"
+      requests:
+        cpu: "100m"
+        memory: "128Mi"
+    monitor_bootstrap:
+      limits:
+        cpu: "8"
+        memory: "8Gi"
+      requests:
+        cpu: "100m"
+        memory: "128Mi"
+    apiserver_bootstrap:
+      limits:
+        cpu: "8"
+        memory: "8Gi"
+      requests:
+        cpu: "100m"
+        memory: "128Mi"
     api:
       limits:
         memory: "128Mi"
@@ -360,6 +493,7 @@ manifests:
   configmap_etc: true
   configmap_gnp: false
   deployment_api: true
+  daemonset_bootstrap: false
   ingress_api: true
   job_image_repo_sync: true
   job_ks_endpoints: true