Use a dependencies file (#55)

* First pass at reading dependencies from file

* Write to a file first

* Add manifest fetch

* Check manifest download

* See if job outputs support fromJSON

* Try again

* Try download from manifest

* Use explicit skip steps

* Use explicit image output name for curl

* See if complex matrix values work

* Re-enable tests in PR workflow

* Use deps file in tests

* Debug outputs

* Use toJSON

* Extract matrix outputs properly

* Reinstate tests

* Remove debug statements

* [skip ci] don't run CI on edited PRs

* [skip ci] Remove edited trigger from PR workflow

.github/actions/ensure-image/action.yml

@@ -12,13 +12,12 @@ inputs:
     description: The name of the cloud within the OpenStack clouds file
     required: true
     default: openstack
-  images-base-url:
-    description: The base URL to use for downloading images
-    required: true
-    default: https://object.arcus.openstack.hpc.cam.ac.uk/swift/v1/AUTH_f0dc9cb312144d0aa44037c9149d2513/azimuth-images-prerelease/
   image-name:
     description: The name of the image to use
     required: true
+  image-url:
+    description: The URL of the image
+    required: true
 
 outputs:
   image-id:
@@ -34,5 +33,5 @@ runs:
       env:
         OS_CLIENT_CONFIG_FILE: ${{ inputs.os-client-config-file }}
         OS_CLOUD: ${{ inputs.os-cloud }}
-        IMAGES_BASE_URL: ${{ inputs.images-base-url }}
         IMAGE_NAME: ${{ inputs.image-name }}
+        IMAGE_URL: ${{ inputs.image-url }}

.github/workflows/ensure-capi-images.yaml

@@ -3,64 +3,104 @@ on:
   workflow_call:
     outputs:
       kube-1-25-image:
-        value: ${{ jobs.ensure-capi-images.outputs.kube-1-25-image }}
+        value: ${{ jobs.produce_outputs.outputs.kube-1-25-image }}
       kube-1-25-version:
-        value: ${{ jobs.ensure-capi-images.outputs.kube-1-25-version }}
+        value: ${{ jobs.produce_outputs.outputs.kube-1-25-version }}
       kube-1-26-image:
-        value: ${{ jobs.ensure-capi-images.outputs.kube-1-26-image }}
+        value: ${{ jobs.produce_outputs.outputs.kube-1-26-image }}
       kube-1-26-version:
-        value: ${{ jobs.ensure-capi-images.outputs.kube-1-26-version }}
+        value: ${{ jobs.produce_outputs.outputs.kube-1-26-version }}
       kube-1-27-image:
-        value: ${{ jobs.ensure-capi-images.outputs.kube-1-27-image }}
+        value: ${{ jobs.produce_outputs.outputs.kube-1-27-image }}
       kube-1-27-version:
-        value: ${{ jobs.ensure-capi-images.outputs.kube-1-27-version }}
+        value: ${{ jobs.produce_outputs.outputs.kube-1-27-version }}
 
 jobs:
-  ensure-capi-images:
+  image_manifest:
     runs-on: ubuntu-latest
-    # We output the image IDs for each image
     outputs:
-      kube-1-25-image: ${{ steps.kube-1-25.outputs.image-id }}
+      manifest: ${{ steps.images.outputs.manifest }}
-      kube-1-25-version: 1.25.10
-      kube-1-26-image: ${{ steps.kube-1-26.outputs.image-id }}
-      kube-1-26-version: 1.26.5
-      kube-1-27-image: ${{ steps.kube-1-27.outputs.image-id }}
-      kube-1-27-version: 1.27.2
     steps:
       - name: Checkout
         uses: actions/checkout@v3
 
+      - name: Fetch image details
+        id: images
+        run: |
+          VN="$(jq -r '.["azimuth-images"]' ./dependencies.json)"
+          MANIFEST="$(curl -fsSL "https://github.com/stackhpc/azimuth-images/releases/download/${VN}/manifest.json")"
+          echo "manifest=$(jq -c . <<< "$MANIFEST")" >> $GITHUB_OUTPUT
+
+  ensure_image:
+    runs-on: ubuntu-latest
+    needs: [image_manifest]
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: kube-1-25
+            image: ${{ fromJSON(needs.image_manifest.outputs.manifest).kubernetes-1-25-jammy }}
+            skip: ${{ github.event.pull_request.draft }}
+          - name: kube-1-26
+            image: ${{ fromJSON(needs.image_manifest.outputs.manifest).kubernetes-1-26-jammy }}
+            skip: ${{ github.event.pull_request.draft }}
+          - name: kube-1-27
+            image: ${{ fromJSON(needs.image_manifest.outputs.manifest).kubernetes-1-27-jammy }}
+            skip: false
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        if: ${{ !matrix.skip }}
+
       - name: Write cloud credential
         run: >
           echo "$CLOUD" > clouds.yml
         shell: bash
         env:
           CLOUD: ${{ secrets.CLOUD }}
+        if: ${{ !matrix.skip }}
 
       - uses: actions/setup-python@v4
         with:
           python-version: '3.9'
           check-latest: true
+        if: ${{ !matrix.skip }}
 
       - name: Install OpenStack CLI
         run: pip install python-openstackclient
+        if: ${{ !matrix.skip }}
 
-      - name: Ensure Kubernetes 1.25 image
+      - name: Ensure Kubernetes image
-        id: kube-1-25
+        id: ensure-image
         uses: ./.github/actions/ensure-image
         with:
-          image-name: ubuntu-focal-kube-v1.25.10-230602-1732
+          image-name: ${{ matrix.image.name }}
-        if: ${{ !github.event.pull_request.draft }}
+          image-url: ${{ matrix.image.url }}
+        if: ${{ !matrix.skip }}
 
-      - name: Ensure Kubernetes 1.26 image
+      - name: Write matrix outputs
-        id: kube-1-26
+        uses: cloudposse/github-action-matrix-outputs-write@main
-        uses: ./.github/actions/ensure-image
         with:
-          image-name: ubuntu-focal-kube-v1.26.5-230602-1757
+          matrix-step-name: ${{ github.job }}
-        if: ${{ !github.event.pull_request.draft }}
+          matrix-key: ${{ matrix.name }}
+          outputs: |-
+            image-id: ${{ steps.ensure-image.outputs.image-id }}
+            kube-version: ${{ matrix.image.kubernetes_version }}
+        if: ${{ !matrix.skip }}
 
-      - name: Ensure Kubernetes 1.27 image
+  produce_outputs:
-        id: kube-1-27
+    runs-on: ubuntu-latest
-        uses: ./.github/actions/ensure-image
+    needs: [ensure_image]
+    outputs:
+      kube-1-25-image: ${{ fromJSON(steps.matrix-outputs.outputs.result).image-id.kube-1-25 }}
+      kube-1-25-version: ${{ fromJSON(steps.matrix-outputs.outputs.result).kube-version.kube-1-25 }}
+      kube-1-26-image: ${{ fromJSON(steps.matrix-outputs.outputs.result).image-id.kube-1-26 }}
+      kube-1-26-version: ${{ fromJSON(steps.matrix-outputs.outputs.result).kube-version.kube-1-26 }}
+      kube-1-27-image: ${{ fromJSON(steps.matrix-outputs.outputs.result).image-id.kube-1-27 }}
+      kube-1-27-version: ${{ fromJSON(steps.matrix-outputs.outputs.result).kube-version.kube-1-27 }}
+    steps:
+      - name: Read matrix outputs
+        id: matrix-outputs
+        uses: cloudposse/github-action-matrix-outputs-read@main
         with:
-          image-name: ubuntu-focal-kube-v1.27.2-230602-1824
+          matrix-step-name: ensure_image

.github/workflows/pr.yaml

@@ -1,7 +1,11 @@
 name: test pr
 on:
   pull_request:
-    types: [opened,synchronize,ready_for_review,edited,reopened]
+    types:
+      - opened
+      - synchronize
+      - ready_for_review
+      - reopened
     branches:
       - main
 

.github/workflows/test.yaml

@@ -12,58 +12,62 @@ on:
         required: true
         default: false
 
-env:
-  HELM_VERSION: v3.11.3
-  CAPI_VERSION: v1.4.3
-  CAPO_VERSION: v0.7.3
-  ADDON_PROVIDER_VERSION: 0.1.0-dev.0.main.26
-  SONOBUOY_VERSION: 0.56.16
-  CERTMANAGER_VERSION: v1.12.1
-
 jobs:
   test-chart:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      
+
+      - name: Read dependencies
+        id: deps
+        run: |
+          echo "addon-provider=$(jq -r '.["addon-provider"]' ./dependencies.json)" >> $GITHUB_OUTPUT
+          echo "cluster-api=$(jq -r '.["cluster-api"]' ./dependencies.json)" >> $GITHUB_OUTPUT
+          echo "cluster-api-provider-openstack=$(jq -r '.["cluster-api-provider-openstack"]' ./dependencies.json)" >> $GITHUB_OUTPUT
+          echo "cert-manager=$(jq -r '.["cert-manager"]' ./dependencies.json)" >> $GITHUB_OUTPUT
+          echo "helm=$(jq -r '.["helm"]' ./dependencies.json)" >> $GITHUB_OUTPUT
+          echo "sonobuoy=$(jq -r '.["sonobuoy"]' ./dependencies.json)" >> $GITHUB_OUTPUT
+
       - name: Install tools
         run: sudo apt install -y zip unzip
-      
+
       - name: Install sonobuoy
         run: >
           wget https://github.com/vmware-tanzu/sonobuoy/releases/download/v${SONOBUOY_VERSION}/sonobuoy_${SONOBUOY_VERSION}_linux_amd64.tar.gz &&
           tar -xf sonobuoy_${SONOBUOY_VERSION}_linux_amd64.tar.gz &&
           sudo mv -n sonobuoy /usr/bin/
+        env:
+          SONOBUOY_VERSION: ${{ steps.deps.outputs.sonobuoy }}
 
       - uses: actions/setup-python@v4
         with:
           python-version: '3.9'
           check-latest: true
-      
+
       - name: Set up Helm
         uses: azure/setup-helm@v3
         with:
-          version: ${{ env.HELM_VERSION }}
+          version: ${{ steps.deps.outputs.helm }}
-      
+
       - name: Create k8s Kind Cluster
         uses: helm/kind-action@v1.5.0
-      
+
       - name: Install cert-manager
         run: |-
           helm upgrade cert-manager cert-manager \
             --repo https://charts.jetstack.io \
-            --version ${CERTMANAGER_VERSION} \
+            --version ${{ steps.deps.outputs.cert-manager }} \
             --namespace cert-manager \
             --create-namespace  \
             --install \
             --set installCRDs=true \
             --wait \
             --timeout 10m
-      
+
       - name: Ensure Cluster API kustomization directory exists
         run: mkdir -p clusterapi
-      
+
       # From here: https://github.com/stackhpc/ansible-collection-azimuth-ops/blob/main/roles/clusterapi/defaults/main.yml
       - name: Write Cluster API kustomization file
         uses: DamianReeves/write-file-action@master
@@ -72,8 +76,8 @@ jobs:
           write-mode: overwrite
           contents: |
             resources:
-              - https://github.com/kubernetes-sigs/cluster-api/releases/download/${{ env.CAPI_VERSION }}/cluster-api-components.yaml
+              - https://github.com/kubernetes-sigs/cluster-api/releases/download/${{ steps.deps.outputs.cluster-api }}/cluster-api-components.yaml
-              - https://github.com/kubernetes-sigs/cluster-api-provider-openstack/releases/download/${{ env.CAPO_VERSION }}/infrastructure-components.yaml
+              - https://github.com/kubernetes-sigs/cluster-api-provider-openstack/releases/download/${{ steps.deps.outputs.cluster-api-provider-openstack }}/infrastructure-components.yaml
             patches:
               - patch: |-
                   - op: replace
@@ -105,7 +109,7 @@ jobs:
                   kind: Deployment
                   namespace: capi-kubeadm-control-plane-system
                   name: capi-kubeadm-control-plane-controller-manager
-      
+
       - name: Install Cluster API resources
         run: kubectl apply -k clusterapi/
 
@@ -131,13 +135,13 @@ jobs:
         run: |-
           helm upgrade cluster-api-addon-provider cluster-api-addon-provider \
             --repo https://stackhpc.github.io/cluster-api-addon-provider \
-            --version ${ADDON_PROVIDER_VERSION} \
+            --version ${{ steps.deps.outputs.addon-provider }} \
             --namespace capi-addon-system \
             --create-namespace \
             --install \
             --wait \
             --timeout 10m
-      
+
       - name: Write cloud credential
         run: >
           echo "$CLOUD" > clouds.yml

dependencies.json

@@ -0,0 +1,9 @@
+{
+    "addon-provider": "0.1.0-dev.0.main.26",
+    "azimuth-images": "0.1.2",
+    "cluster-api": "v1.4.4",
+    "cluster-api-provider-openstack": "v0.7.3",
+    "cert-manager": "v1.12.2",
+    "helm": "v3.12.2",
+    "sonobuoy": "0.56.16"
+}

scripts/ensure-image.sh

@@ -2,8 +2,13 @@
 
 set -ex
 
-[ -z "$IMAGES_BASE_URL" ] && echo "IMAGES_BASE_URL is required" >&2 && exit 1
+#####
+# This script uploads an image to OpenStack unless already present
+#####
+
+
 [ -z "$IMAGE_NAME" ] && echo "IMAGE_NAME is required" >&2 && exit 1
+[ -z "$IMAGE_URL" ] && echo "IMAGE_URL is required" >&2 && exit 1
 
 # Default the GITHUB_OUTPUT to stdout
 GITHUB_OUTPUT="${GITHUB_OUTPUT:-/dev/stdout}"
@@ -20,8 +25,7 @@ fi
 
 # If not, download the image and upload it to Glance
 IMAGE_FNAME="${IMAGE_NAME}.${IMAGE_DISK_FORMAT:-qcow2}"
-IMAGE_URL="${IMAGES_BASE_URL}${IMAGE_FNAME}"
+curl -Lo "$IMAGE_FNAME" --progress-bar "$IMAGE_URL"
-curl -LO --progress-bar "$IMAGE_URL"
 IMAGE_ID="$(
   openstack image create \
     --progress \