Add OIDC support to upload-logs-s3

Add support for the upload-logs-s3 role to obtain a short-term
token from the AWS sts service using a federated OIDC provider
(which may be Zuul itself).

Change-Id: Ic69fb1f61f53b3b8dd08f776b96e9d5db57dbf5a

roles/upload-logs-base/library/zuul_s3_upload.py

@@ -60,9 +60,41 @@ except ImportError:
 MAX_UPLOAD_THREADS = 24
 
 
+def get_creds_from_assumed_role(role_arn, session_name, token, duration):
+    client = boto3.client('sts')
+    if session_name is None:
+        session_name = 'zuul'
+    if duration is None:
+        duration = 3600
+    resp = client.assume_role_with_web_identity(
+        RoleArn=role_arn,
+        RoleSessionName=session_name,
+        WebIdentityToken=token,
+        DurationSeconds=duration,
+    )
+    return dict(
+        aws_access_key_id=resp['Credentials']['AccessKeyId'],
+        aws_secret_access_key=resp['Credentials']['SecretAccessKey'],
+        aws_session_token=resp['Credentials']['SessionToken'],
+    )
+
+
 class Uploader():
     def __init__(self, bucket, public, endpoint=None, prefix=None,
-                 dry_run=False, aws_access_key=None, aws_secret_key=None):
+                 dry_run=False, aws_access_key=None, aws_secret_key=None,
+                 aws_oidc_role_arn=None, aws_oidc_session_name=None,
+                 aws_oidc_token=None, aws_oidc_token_duration=None):
+
+        if aws_oidc_token:
+            credential_args = get_creds_from_assumed_role(
+                aws_oidc_role_arn, aws_oidc_session_name, aws_oidc_token,
+                aws_oidc_token_duration)
+        else:
+            credential_args = dict(
+                aws_access_key_id=aws_access_key,
+                aws_secret_access_key=aws_secret_key,
+            )
+
         self.dry_run = dry_run
         self.public = public
         if dry_run:
@@ -83,8 +115,7 @@ class Uploader():
 
         self.s3 = boto3.resource('s3',
                                  endpoint_url=self.endpoint,
-                                 aws_access_key_id=aws_access_key,
+                                 **credential_args)
-                                 aws_secret_access_key=aws_secret_key)
         self.bucket = self.s3.Bucket(bucket)
 
         cors = {
@@ -95,8 +126,7 @@ class Uploader():
         }
         client = boto3.client('s3',
                               endpoint_url=self.endpoint,
-                              aws_access_key_id=aws_access_key,
+                              **credential_args)
-                              aws_secret_access_key=aws_secret_key)
         try:
             current_cors = None
             try:
@@ -224,7 +254,9 @@ class Uploader():
 def run(bucket, public, files, endpoint=None,
         indexes=True, parent_links=True, topdir_parent_link=False,
         partition=False, footer='index_footer.html',
-        prefix=None, aws_access_key=None, aws_secret_key=None):
+        prefix=None, aws_access_key=None, aws_secret_key=None,
+        aws_oidc_role_arn=None, aws_oidc_session_name=None,
+        aws_oidc_token=None, aws_oidc_token_duration=None):
 
     if prefix:
         prefix = prefix.lstrip('/')
@@ -258,7 +290,12 @@ def run(bucket, public, files, endpoint=None,
                             endpoint,
                             prefix,
                             aws_access_key=aws_access_key,
-                            aws_secret_key=aws_secret_key)
+                            aws_secret_key=aws_secret_key,
+                            aws_oidc_role_arn=aws_oidc_role_arn,
+                            aws_oidc_session_name=aws_oidc_session_name,
+                            aws_oidc_token=aws_oidc_token,
+                            aws_oidc_token_duration=aws_oidc_token_duration)
+
         upload_failures = uploader.upload(file_list)
 
         return uploader.url, upload_failures
@@ -279,6 +316,10 @@ def ansible_main():
             endpoint=dict(type='str'),
             aws_access_key=dict(type='str'),
             aws_secret_key=dict(type='str', no_log=True),
+            aws_oidc_role_arn=dict(type='str'),
+            aws_oidc_session_name=dict(type='str'),
+            aws_oidc_token=dict(type='str', no_log=True),
+            aws_oidc_token_duration=dict(type='int'),
         )
     )
 
@@ -294,7 +335,13 @@ def ansible_main():
                         footer=p.get('footer'),
                         prefix=p.get('prefix'),
                         aws_access_key=p.get('aws_access_key'),
-                        aws_secret_key=p.get('aws_secret_key'))
+                        aws_secret_key=p.get('aws_secret_key'),
+                        aws_oidc_role_arn=p.get('aws_oidc_role_arn'),
+                        aws_oidc_session_name=p.get('aws_oidc_session_name'),
+                        aws_oidc_token=p.get('aws_oidc_token'),
+                        aws_oidc_token_duration=p.get(
+                            'aws_oidc_token_duration'),
+                        )
     if failures:
         failure_msg = pprint.pformat(failures)
         module.fail_json(msg=f"Failure(s) during log upload:\n{failure_msg}",

roles/upload-logs-s3/README.rst

@@ -53,6 +53,17 @@ installed in the Ansible environment on the Zuul executor.
 
    Whether to create `index.html` files with directory indexes.
 
+.. zuul:rolevar:: upload_logs_s3_endpoint
+
+   The endpoint to use when uploading logs to an s3 compatible
+   service.  By default this will be automatically constructed by boto
+   but should be set when working with non-aws hosted s3 service.
+
+Conventional authentication
+
+To authenticate with a conventional AWS access key and secret, supply
+the following two variables:
+
 .. zuul:rolevar:: zuul_log_aws_access_key
 
    AWS access key to use.
@@ -61,7 +72,29 @@ installed in the Ansible environment on the Zuul executor.
 
    AWS secret key for the AWS access key.
 
-.. zuul:rolevar:: upload_logs_s3_endpoint
+OIDC federated authentication
 
-   The endpoint to use when uploading logs to an s3 compatible service.
+It is also possible to authenticate usinc OIDC, including using Zuul
-   By default this will be automatically constructed by boto but should be set when working with non-aws hosted s3 service.
+as an ID provider with Zuul's OIDC token secrets feature.  Use the
+following variables to do so:
+
+.. zuul:rolevar:: zuul_log_aws_idc_role_arn
+
+   The ARN of the AWS role to assume when authenticating.
+
+.. zuul:rolevar:: zuul_log_aws_oidc_token
+
+   The token issued by the federated IDP.  If the IDP is Zuul, this
+   should be the token secret.
+
+.. zuul:rolevar:: zuul_log_aws_oidc_session_name
+   :default: zuul
+
+   The AWS session name.  Defaults to "zuul".
+
+.. zuul:rolevar:: zuul_log_aws_oidc_token_duration
+   :default: 3600
+
+   This value is used when requeting the temporary token from AWS and
+   indicates the requested lifetime of that token.  Defaults to one
+   hour.

roles/upload-logs-s3/tasks/main.yaml

@@ -31,8 +31,12 @@
         public: "{{ zuul_log_bucket_public }}"
         prefix: "{{ zuul_log_path }}"
         indexes: "{{ zuul_log_create_indexes }}"
-        aws_access_key: "{{ zuul_log_aws_access_key }}"
+        aws_access_key: "{{ zuul_log_aws_access_key | default(omit) }}"
-        aws_secret_key: "{{ zuul_log_aws_secret_key }}"
+        aws_secret_key: "{{ zuul_log_aws_secret_key | default(omit) }}"
+        aws_oidc_role_arn: "{{ zuul_log_aws_oidc_role_arn | default(omit) }}"
+        aws_oidc_session_name: "{{ zuul_log_aws_oidc_session_name | default(omit) }}"
+        aws_oidc_token: "{{ zuul_log_aws_oidc_token | default(omit) }}"
+        aws_oidc_token_duration: "{{ zuul_log_aws_oidc_token_duration | default(omit) }}"
         files:
           - "{{ zuul.executor.log_root }}/"
       register: upload_results