Merge "Add limit-log-files role"

doc/source/general-roles.rst

@@ -26,6 +26,7 @@ General Purpose Roles
 .. zuul:autorole:: fetch-markdownlint
 .. zuul:autorole:: git-prepare-nodecache
 .. zuul:autorole:: intercept-job
+.. zuul:autorole:: limit-log-files
 .. zuul:autorole:: log-inventory
 .. zuul:autorole:: markdownlint
 .. zuul:autorole:: multi-node-bridge

roles/limit-log-files/README.rst

@@ -0,0 +1,33 @@
+Limits log upload files based on number and size
+
+This role checks that the log directory doesn't exceed a set number of files
+or size limits per file pattern. If limits are violated, it registers the error
+and reports a warning to Zuul.
+
+**Role Variables**
+
+.. zuul:rolevar:: limit_log_files_fail:
+   :type: bool
+   :default: false
+
+   If set to true, the role will fail instead of reporting a warning to zuul.
+
+.. zuul:rolevar:: limit_log_files_file_limit:
+   :type: int
+   :default: 100
+
+   Maximum number of files allowed in the logs directory.
+
+.. zuul:rolevar:: limit_log_files_file_rules:
+   :type: string
+   :default: see Example
+
+   A list of file patterns and their size limits.
+   The role calls the find command, which has specific file size constants, see `man find`.
+
+   Example
+   .. code-block:: yaml
+
+      limit_log_files_file_rules:
+         - ".*job-output.json$ 500k"
+         - ".*\\zuul-manifest.json$ 1k"

roles/limit-log-files/defaults/main.yaml

@@ -0,0 +1,10 @@
+# this var just for testing
+limit_log_files_log_dir: "{{ zuul.executor.log_root }}"
+
+limit_log_files_fail: false
+
+limit_log_files_file_limit: 100
+
+limit_log_files_file_rules:
+  - ".*job-output.json$ 500k"
+  - ".*\\zuul-manifest.json$ 1k"

roles/limit-log-files/tasks/main.yaml

@@ -0,0 +1,64 @@
+- name: Run limit-log-files script
+  shell: |-
+    set -o pipefail
+
+    # File count limit
+    FILE_LIMIT={{ limit_log_files_file_limit }}
+
+    # Size rules
+    RULES=(
+        "{{ limit_log_files_file_rules | join('" "') }}"
+    )
+
+    TARGET_DIR="{{ limit_log_files_log_dir }}"
+    WORK_DIR="{{ zuul.executor.work_root }}"
+
+    if [[ ! -d "$TARGET_DIR" ]]; then
+        echo "Error: Directory '$TARGET_DIR' not found."
+        exit 1
+    fi
+
+    # Count files in target directory
+    TOTAL_FILES=$(find "$TARGET_DIR" -type f | wc -l)
+
+    if (( TOTAL_FILES > FILE_LIMIT )); then
+        echo "Total number of files in '$TARGET_DIR' exceed limit: $TOTAL_FILES > $FILE_LIMIT"
+        exit 2
+    fi
+
+    ERROR=0
+
+    # check for violations off each rule
+    for rule in "${RULES[@]}"; do
+        REGEX=$(echo "$rule" | awk '{print $1}')
+        LIMIT=$(echo "$rule" | awk '{print $2}')
+
+        echo "Rule: '$REGEX' (limit: $LIMIT):"
+        # find files and remove the working dir prefix
+        FILES=$(find "$TARGET_DIR" -type f -size +"$LIMIT" -regex "$REGEX" -exec ls -lh {} \; | \
+                awk '{ print $9 ": " $5 }' | \
+                sed "s|$WORK_DIR||")
+        echo "$FILES"
+        if [[ -n "$FILES" ]]; then
+            ERROR=1
+        fi
+
+    done
+
+    if [[ "$ERROR" -eq 1 ]]; then
+        echo "One or more files exceeded their size limit rule"
+        exit 3
+    fi
+  args:
+    executable: /bin/bash
+  delegate_to: localhost
+  register: limit_log_file_violations
+  failed_when: limit_log_files_fail and limit_log_file_violations.rc != 0
+
+- name: Report warnings to Zuul
+  zuul_return:
+    data:
+      zuul:
+        warnings:
+          - "{{ limit_log_file_violations.stdout }}"
+  when: limit_log_file_violations.rc != 0

test-playbooks/general/limit-log-files.yaml

@@ -0,0 +1,64 @@
+- hosts: all
+  vars:
+    test_folder: "{{ zuul.executor.work_root }}/test_folder"
+  tasks:
+    # create files for test runs
+    - name: Create folder for test files
+      file:
+        path: "{{ test_folder }}"
+        state: directory
+        mode: '0755'
+      delegate_to: localhost
+    - name: Create large file 501
+      community.general.filesize:
+        path: "{{ test_folder }}/file_501"
+        size: 501B
+      delegate_to: localhost
+    - name: Create large file 350
+      community.general.filesize:
+        path: "{{ test_folder }}/file_350"
+        size: 350B
+      delegate_to: localhost
+    - name: Create large file 100
+      community.general.filesize:
+        path: "{{ test_folder }}/file_100"
+        size: 100B
+      delegate_to: localhost
+    # execute role without issue, should not warn and not fail
+    - name: Execute role without issue
+      include_role:
+        name: limit-log-files
+      vars:
+        limit_log_files_fail: false
+        limit_log_files_log_dir: "{{ test_folder }}"
+        limit_log_files_file_rules:
+          - ".* 1M"
+          - ".* 2M"
+    # execute role with warning, should warn but not fail
+    - name: Execute role in warning mode
+      include_role:
+        name: limit-log-files
+      vars:
+        limit_log_files_fail: false
+        limit_log_files_log_dir: "{{ test_folder }}"
+        limit_log_files_file_rules:
+          - ".* 250c"
+          - ".* 150c"
+    # execute role with fail, should fail
+    - name: Block to rescue failing of role
+      block:
+        - name: Execute role in fail mode
+          include_role:
+            name: limit-log-files
+          vars:
+            limit_log_files_fail: true
+            limit_log_files_log_dir: "{{ test_folder }}"
+            limit_log_files_file_rules:
+              - ".* 500c"
+      rescue:
+        - name: Set fact for assertion
+          set_fact:
+            has_failed: true
+    - name: Assert that second run failed
+      assert:
+        that: has_failed is defined

zuul-tests.d/general-roles-jobs.yaml

@@ -957,6 +957,14 @@
       zuul_tenant_conf_check_tenant_config_path: '{{ zuul.executor.work_root }}/{{
         zuul.project.src_dir }}/test-playbooks/zuul-tenant-conf-check/main.yaml'
 
+- job:
+    name: zuul-jobs-test-limit-log-files
+    description: Test the limit-log-files role
+    parent: zuul-tenant-conf-check
+    files:
+      - roles/limit-log-files/.*
+    run: test-playbooks/general/limit-log-files.yaml
+
 # -* AUTOGENERATED *-
 #  The following project section is autogenerated by
 #    tox -e update-test-platforms
@@ -1034,6 +1042,7 @@
         - zuul-jobs-test-fetch-translation-output
         - zuul-jobs-test-fetch-translation-output-synchronize
         - zuul-jobs-test-zuul-tenant-conf-check
+        - zuul-jobs-test-limit-log-files
     gate:
       jobs: *id001
     periodic-weekly: