From 15828c7a8595fe615f05308e7f53b8e4a07ef7d5 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Thu, 28 Aug 2025 13:33:05 -0700 Subject: [PATCH] Simplify testing of some upload roles These two roles no longer need the delegation variable used in testing, so it is removed. Role execution now happens on the executor. Since the remote servers are still on the worker node, we open the firewall on the remote node in both cases. In the artifactory role, the ensure-docker role is moved to a "roles" section for simplicity. Additionally, one of the artifactory API calls is wrapped in retries as a precaution since I saw that fail in local testing. Change-Id: Ia4409edc217e1935775a5aece2e7d9bcfd935762 --- roles/upload-artifactory/tasks/upload.yaml | 15 ++++----- roles/upload-logs-s3/tasks/main.yaml | 4 +-- test-playbooks/artifactory/run.yaml | 37 +++++++++++----------- test-playbooks/upload-logs-s3.yaml | 25 +++++---------- 4 files changed, 35 insertions(+), 46 deletions(-) diff --git a/roles/upload-artifactory/tasks/upload.yaml b/roles/upload-artifactory/tasks/upload.yaml index ed8ce5269..301a5e2d6 100644 --- a/roles/upload-artifactory/tasks/upload.yaml +++ b/roles/upload-artifactory/tasks/upload.yaml @@ -5,18 +5,16 @@ - name: Make sure artifact exists on the executor stat: - path: "{{ _undocumented_test_work_dir_ | default(zuul.executor.work_root) }}\ - /artifacts/{{ zj_artifact.src }}" + path: "{{ zuul.executor.work_root }}/artifacts/{{ zj_artifact.src }}" register: artifact - delegate_to: "{{ _undocumented_test_worker_node_ | default('localhost') }}" + delegate_to: "localhost" failed_when: not artifact.stat.exists - name: Get sha256 checksum stat: - path: "{{ _undocumented_test_work_dir_ | default(zuul.executor.work_root) }}\ - /artifacts/{{ zj_artifact.src }}" + path: "{{ zuul.executor.work_root }}/artifacts/{{ zj_artifact.src }}" checksum_algorithm: sha256 - delegate_to: "{{ _undocumented_test_worker_node_ | default('localhost') }}" + delegate_to: "localhost" register: artifact_sha256_checksum - name: Set request header fact @@ -82,15 +80,14 @@ user: "{{ upload_artifactory_instances[zj_artifact.instance].user }}" password: "{{ _artifactory_password | default(omit) }}" url: "{{ _artifactory_upload_url }}" - src: "{{ _undocumented_test_work_dir_ | default(zuul.executor.work_root) }}/\ - artifacts/{{ zj_artifact.src }}" + src: "{{ zuul.executor.work_root }}/artifacts/{{ zj_artifact.src }}" headers: "{{ request_header }}" status_code: 201 method: PUT force_basic_auth: "{{ upload_artifactory_instances[zj_artifact.instance].force_basic_auth | default(false) | bool }}" remote_src: yes # To not unecessarily copy artifact to a tempfile - delegate_to: "{{ _undocumented_test_worker_node_ | default('localhost') }}" + delegate_to: "localhost" - name: Add artifact link to build page zuul_return: diff --git a/roles/upload-logs-s3/tasks/main.yaml b/roles/upload-logs-s3/tasks/main.yaml index 0de925c15..7f82ffb5a 100644 --- a/roles/upload-logs-s3/tasks/main.yaml +++ b/roles/upload-logs-s3/tasks/main.yaml @@ -15,14 +15,14 @@ msg: "log_root {{ zuul.executor.log_root }}" - name: Ensure logs are readable before uploading - delegate_to: "{{ _undocumented_test_worker_node_ | default('localhost') }}" + delegate_to: "localhost" command: "chmod -R u=rwX,g=rX,o=rX {{ zuul.executor.log_root }}/" # ANSIBLE0007 chmod used in place of argument mode to file tags: - skip_ansible_lint - name: Upload logs to S3 - delegate_to: "{{ _undocumented_test_worker_node_ | default('localhost') }}" + delegate_to: "localhost" no_log: true zuul_s3_upload: endpoint: "{{ upload_logs_s3_endpoint | default(omit) }}" diff --git a/test-playbooks/artifactory/run.yaml b/test-playbooks/artifactory/run.yaml index 15139b866..b5f393cc5 100644 --- a/test-playbooks/artifactory/run.yaml +++ b/test-playbooks/artifactory/run.yaml @@ -1,9 +1,8 @@ - hosts: all + roles: + - clear-firewall + - ensure-docker tasks: - - name: Ensure docker is installed - include_role: - name: ensure-docker - - name: Start artifactory in a container command: >- docker run -d @@ -33,6 +32,9 @@ type: generic headers: Content-Type: application/yaml + until: artifactory_status.status == 200 + retries: 12 + delay: 20 - name: Create an api key for the admin user uri: @@ -49,36 +51,35 @@ cacheable: true upload_artifactory_instances: localhost_password: - fqdn: localhost:8081 + fqdn: "{{ ansible_host }}:8081" transport: http user: admin password: password force_basic_auth: true localhost_api_key: - fqdn: localhost:8081 + fqdn: "{{ ansible_host }}:8081" transport: http user: admin api_key: "{{ (artifactory_api_key.content | from_json)['apiKey'] }}" localhost_properties: - fqdn: localhost:8081 + fqdn: "{{ ansible_host }}:8081" transport: http user: admin api_key: "{{ (artifactory_api_key.content | from_json)['apiKey'] }}" - hosts: all - vars: - # Since we're testing a role that normally requires a - # trusted context flip the delegate_to so we execute on the - # remote instead. Also set the working directory to something - # that is known to exist on the remote. - _undocumented_test_worker_node_: "{{ inventory_hostname }}" - _undocumented_test_work_dir_: "{{ ansible_user_dir }}/zuul-output" pre_tasks: + - name: Create artifacts directory + delegate_to: "localhost" + file: + path: "{{ zuul.executor.work_root }}/artifacts" + state: directory - name: Write a file with some content to artifacts directory + delegate_to: "localhost" copy: content: | First file - dest: "{{ ansible_user_dir }}/zuul-output/artifacts/test-file.txt" + dest: "{{ zuul.executor.work_root }}/artifacts/test-file.txt" - name: Set upload_artifactory_manifest_basic fact set_fact: upload_artifactory_manifest: @@ -111,11 +112,11 @@ set_fact: expected_artifacts: - name: test-properties.txt - url: http://localhost:8081/artifactory/generic-repository/path/to/dest/test-properties.txt + url: "http://{{ ansible_host }}:8081/artifactory/generic-repository/path/to/dest/test-properties.txt" - name: test-api-key.txt - url: http://localhost:8081/artifactory/generic-repository/path/to/dest/test-api-key.txt + url: "http://{{ ansible_host }}:8081/artifactory/generic-repository/path/to/dest/test-api-key.txt" - name: test-password.txt - url: http://localhost:8081/artifactory/generic-repository/path/to/dest/test-password.txt + url: "http://{{ ansible_host }}:8081/artifactory/generic-repository/path/to/dest/test-password.txt" - name: Assert artifact assert: that: diff --git a/test-playbooks/upload-logs-s3.yaml b/test-playbooks/upload-logs-s3.yaml index 2374e3221..64580077f 100644 --- a/test-playbooks/upload-logs-s3.yaml +++ b/test-playbooks/upload-logs-s3.yaml @@ -4,8 +4,8 @@ zuul_log_aws_secret_key: minioadmin zuul_log_bucket: zuul test_content: "123abc" - _undocumented_test_worker_node_: "{{ inventory_hostname }}" roles: + - clear-firewall - ensure-docker - ensure-pip tasks: @@ -31,38 +31,29 @@ aws_access_key: "{{ zuul_log_aws_access_key }}" aws_secret_key: "{{ zuul_log_aws_secret_key }}" - - name: Create Logdir - become: true - file: - state: directory - path: "{{ zuul.executor.log_root }}" - mode: "0755" - group: "zuul" - owner: "zuul" - register: fake_zuul_logdir - - name: Add content to tempfile + delegate_to: localhost copy: content: "{{ test_content }}" - dest: "{{ fake_zuul_logdir.path }}/testfile" + dest: "{{ zuul.executor.log_root }}/testfile" - name: Upload file to s3 include_role: name: upload-logs-s3 vars: - upload_logs_s3_endpoint: 'http://localhost:9000' + upload_logs_s3_endpoint: 'http://{{ ansible_host }}:9000' - name: Download mc get_url: url: https://dl.min.io/client/mc/release/linux-amd64/mc - dest: "{{ fake_zuul_logdir.path }}/mc" + dest: "{{ ansible_user_dir }}/mc" mode: 0755 - name: Add localhost minio host - command: "{{ fake_zuul_logdir.path }}/mc alias set local http://localhost:9000 {{ zuul_log_aws_access_key }} {{ zuul_log_aws_secret_key }}" + command: "{{ ansible_user_dir }}/mc alias set local http://localhost:9000 {{ zuul_log_aws_access_key }} {{ zuul_log_aws_secret_key }}" - name: List files in minio bucket - command: "{{ fake_zuul_logdir.path }}/mc find local/zuul" + command: "{{ ansible_user_dir }}/mc find local/zuul" - name: Check for testfile in minio bucket - command: "{{ fake_zuul_logdir.path }}/mc find local/zuul/{{ zuul_log_path }}/testfile" + command: "{{ ansible_user_dir }}/mc find local/zuul/{{ zuul_log_path }}/testfile"