From 39f967781e699a7f70922ab589758356abb1409d Mon Sep 17 00:00:00 2001 From: Andrii Ostapenko Date: Sat, 1 Aug 2020 20:30:43 -0500 Subject: [PATCH] Brings back post-review buildset registry based image testing This commit brings back post-review image testing implemented using buildset registry to remove necessity of pushing not tested images to dockerhub. Flow on post-review: create buildset registy -> build images and push to buildset registry -> pause each upload job -> run deployment against built images -> upload images to dockerhub -> promote images. Current implementation does not use opendev-* parent jobs for both pre and post-review to avoid pushing images to intermediate registry to save up to 15 minutes on publishing. Change-Id: I272cdd1bfc77f4b76998f96cd16606bb9e04b8b8 Signed-off-by: Andrii Ostapenko --- zuul.d/base.yaml | 85 ++++++++++++++++++- zuul.d/playbooks/_return-image.yml | 14 +++ .../_write-buildset-registry-cert.yml | 10 +++ zuul.d/playbooks/build-loci.yml | 19 +++++ zuul.d/playbooks/build.yml | 21 +++++ 5 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 zuul.d/playbooks/_return-image.yml create mode 100644 zuul.d/playbooks/_write-buildset-registry-cert.yml create mode 100644 zuul.d/playbooks/build.yml diff --git a/zuul.d/base.yaml b/zuul.d/base.yaml index 65868c7f..269bb7ef 100644 --- a/zuul.d/base.yaml +++ b/zuul.d/base.yaml @@ -24,6 +24,16 @@ - openstack-helm-images-cinder-stein-ubuntu_bionic - openstack-helm-images-cinder-train-ubuntu_bionic - openstack-helm-images-horizon-stein-ubuntu_bionic + gate: + jobs: + - opendev-buildset-registry + - openstack-helm-images-aio-monitoring + - openstack-helm-images-aio-logging + - openstack-helm-images-compute-kit-stein-ubuntu_bionic + - openstack-helm-images-compute-kit-train-ubuntu_bionic + - openstack-helm-images-cinder-stein-ubuntu_bionic + - openstack-helm-images-cinder-train-ubuntu_bionic + - openstack-helm-images-horizon-stein-ubuntu_bionic templates: - publish-openstack-docs-pti - release-notes-jobs-python3 @@ -64,7 +74,9 @@ # `dockerfile`. Explicit. - job: name: openstack-helm-images-build - parent: opendev-build-docker-image + parent: build-docker-image + run: zuul.d/playbooks/build.yml + match-on-config-updates: false dependencies: - name: opendev-buildset-registry abstract: true @@ -92,6 +104,10 @@ - job: name: openstack-helm-images-upload parent: upload-docker-image + run: zuul.d/playbooks/build.yml + match-on-config-updates: false + dependencies: + - name: opendev-buildset-registry abstract: true ansible-version: 2.8 secrets: @@ -102,7 +118,7 @@ - job: name: openstack-helm-images-promote - parent: opendev-promote-docker-image + parent: promote-docker-image abstract: true ansible-version: 2.8 secrets: @@ -116,6 +132,7 @@ - job: name: openstack-helm-images-aio-monitoring parent: openstack-helm-infra-aio-monitoring + match-on-config-updates: false dependencies: - name: openstack-helm-images-build-prometheus-openstack-exporter soft: true @@ -131,6 +148,20 @@ soft: true - name: openstack-helm-images-build-alerta soft: true + - name: openstack-helm-images-upload-prometheus-openstack-exporter + soft: true + - name: openstack-helm-images-upload-patroni + soft: true + - name: openstack-helm-images-upload-osh-selenium + soft: true + - name: openstack-helm-images-upload-node-problem-detector + soft: true + - name: openstack-helm-images-upload-nagios + soft: true + - name: openstack-helm-images-upload-minikube-aio + soft: true + - name: openstack-helm-images-upload-alerta + soft: true files: - prometheus-openstack-exporter/.* - zuul.d/prometheus-openstack-exporter.yaml @@ -150,6 +181,7 @@ - job: name: openstack-helm-images-aio-logging parent: openstack-helm-infra-aio-logging + match-on-config-updates: false dependencies: - name: openstack-helm-images-build-fluentd soft: true @@ -159,6 +191,14 @@ soft: true - name: openstack-helm-images-build-minikube-aio soft: true + - name: openstack-helm-images-upload-fluentd + soft: true + - name: openstack-helm-images-upload-elasticsearch-s3 + soft: true + - name: openstack-helm-images-upload-osh-selenium + soft: true + - name: openstack-helm-images-upload-minikube-aio + soft: true files: - fluentd/.* - zuul.d/fluentd.yaml @@ -172,6 +212,7 @@ - job: name: openstack-helm-images-compute-kit-stein-ubuntu_bionic parent: openstack-helm-compute-kit-stein-ubuntu_bionic + match-on-config-updates: false dependencies: - name: openstack-helm-images-build-ospurge soft: true @@ -185,6 +226,18 @@ soft: true - name: openstack-helm-images-build-minikube-aio soft: true + - name: openstack-helm-images-upload-ospurge + soft: true + - name: openstack-helm-images-upload-openvswitch + soft: true + - name: openstack-helm-images-upload-openstack-loci-stein-ubuntu_bionic + soft: true + - name: openstack-helm-images-upload-mariadb + soft: true + - name: openstack-helm-images-upload-libvirt + soft: true + - name: openstack-helm-images-upload-minikube-aio + soft: true files: - ospurge/.* - zuul.d/ospurge.yaml @@ -202,6 +255,7 @@ - job: name: openstack-helm-images-compute-kit-train-ubuntu_bionic parent: openstack-helm-compute-kit-train-ubuntu_bionic + match-on-config-updates: false dependencies: - name: openstack-helm-images-build-openvswitch soft: true @@ -213,6 +267,16 @@ soft: true - name: openstack-helm-images-build-minikube-aio soft: true + - name: openstack-helm-images-upload-openvswitch + soft: true + - name: openstack-helm-images-upload-openstack-loci-train-ubuntu_bionic + soft: true + - name: openstack-helm-images-upload-mariadb + soft: true + - name: openstack-helm-images-upload-libvirt + soft: true + - name: openstack-helm-images-upload-minikube-aio + soft: true files: - openvswitch/.* - zuul.d/openvswitch.yaml @@ -228,11 +292,16 @@ - job: name: openstack-helm-images-cinder-stein-ubuntu_bionic parent: openstack-helm-cinder-stein-ubuntu_bionic + match-on-config-updates: false dependencies: - name: openstack-helm-images-build-openstack-loci-stein-ubuntu_bionic soft: true - name: openstack-helm-images-build-minikube-aio soft: true + - name: openstack-helm-images-upload-openstack-loci-stein-ubuntu_bionic + soft: true + - name: openstack-helm-images-upload-minikube-aio + soft: true files: - ^openstack/loci/.* - zuul.d/openstack-loci.yaml @@ -242,11 +311,16 @@ - job: name: openstack-helm-images-cinder-train-ubuntu_bionic parent: openstack-helm-cinder-train-ubuntu_bionic + match-on-config-updates: false dependencies: - name: openstack-helm-images-build-openstack-loci-train-ubuntu_bionic soft: true - name: openstack-helm-images-build-minikube-aio soft: true + - name: openstack-helm-images-upload-openstack-loci-train-ubuntu_bionic + soft: true + - name: openstack-helm-images-upload-minikube-aio + soft: true files: - ^openstack/loci/.* - zuul.d/openstack-loci.yaml @@ -258,6 +332,7 @@ - job: name: openstack-helm-images-horizon-stein-ubuntu_bionic parent: openstack-helm-horizon-stein-ubuntu_bionic + match-on-config-updates: false dependencies: - name: openstack-helm-images-build-osh-selenium soft: true @@ -265,6 +340,12 @@ soft: true - name: openstack-helm-images-build-minikube-aio soft: true + - name: openstack-helm-images-upload-osh-selenium + soft: true + - name: openstack-helm-images-upload-openstack-loci-stein-ubuntu_bionic + soft: true + - name: openstack-helm-images-upload-minikube-aio + soft: true files: - osh-selenium/.* - zuul.d/osh-selenium.yaml diff --git a/zuul.d/playbooks/_return-image.yml b/zuul.d/playbooks/_return-image.yml new file mode 100644 index 00000000..f6b6a6a5 --- /dev/null +++ b/zuul.d/playbooks/_return-image.yml @@ -0,0 +1,14 @@ +- name: Return artifact to Zuul + zuul_return: + data: + zuul: + artifacts: + - name: "{{ zj_image.repository }}:{{ zj_image_tag }}" + url: "docker://zuul-jobs.buildset-registry:{{ buildset_registry.port }}/{{ zj_image.repository }}:{{ zj_image_tag }}" + metadata: + type: container_image + repository: "{{ zj_image.repository }}" + tag: "{{ zj_image_tag }}" + loop: "{{ zj_image.tags | default(['latest']) }}" + loop_control: + loop_var: zj_image_tag diff --git a/zuul.d/playbooks/_write-buildset-registry-cert.yml b/zuul.d/playbooks/_write-buildset-registry-cert.yml new file mode 100644 index 00000000..d18321d4 --- /dev/null +++ b/zuul.d/playbooks/_write-buildset-registry-cert.yml @@ -0,0 +1,10 @@ +- name: Write buildset registry TLS certificate + become: true + copy: + content: "{{ buildset_registry.cert }}" + dest: "/usr/local/share/ca-certificates/zuul-jobs.buildset-registry.crt" + register: _tls_ca +- name: Update CA certs + command: update-ca-certificates + become: true + when: _tls_ca is changed diff --git a/zuul.d/playbooks/build-loci.yml b/zuul.d/playbooks/build-loci.yml index c62e7b1c..91597230 100644 --- a/zuul.d/playbooks/build-loci.yml +++ b/zuul.d/playbooks/build-loci.yml @@ -15,6 +15,12 @@ - hosts: all[0] gather_facts: true + pre_tasks: + - name: Use buildset registry + include_role: + name: use-buildset-registry + - name: Write buildset registry TLS certificate + include_tasks: _write-buildset-registry-cert.yml tasks: - name: Export the right vars shell: | @@ -70,3 +76,16 @@ name: build-docker-image vars: zuul_work_dir: "src/opendev.org/openstack/loci" + + - name: Return images to zuul + include_tasks: _return-image.yml + loop: "{{ docker_images }}" + loop_control: + loop_var: zj_image + + - name: Pause the job + when: buildset_registry is defined and zuul.pipeline != 'check' + zuul_return: + data: + zuul: + pause: true diff --git a/zuul.d/playbooks/build.yml b/zuul.d/playbooks/build.yml new file mode 100644 index 00000000..4cc93ac9 --- /dev/null +++ b/zuul.d/playbooks/build.yml @@ -0,0 +1,21 @@ +- hosts: all + pre_tasks: + - name: Use buildset registry + include_role: + name: use-buildset-registry + - name: Write buildset registry TLS certificate + include_tasks: _write-buildset-registry-cert.yml + roles: + - build-docker-image + tasks: + - name: Return images to zuul + include_tasks: _return-image.yml + loop: "{{ docker_images }}" + loop_control: + loop_var: zj_image + - name: Pause the job + when: buildset_registry is defined and zuul.pipeline != 'check' + zuul_return: + data: + zuul: + pause: true