docker: Set ulimit nofile on EL9

Some latest RPM-based distributions, such as RHEL 9 and CentOS Stream 9, ship
a recent version of systemd that sets the default open file handle limit is
set to 1073741816.
That effects in various issues (like rabbitmq not working), including
ceph/other services in containers not working properly.
kolla-ansible has set that as a default ulimit for Kolla container images [1],
this patch sets that as a Docker Engine default (for container images
that don't have this overridden).

[1]: https://review.opendev.org/c/openstack/kolla-ansible/+/839715

Closes-Bug: #2008761

Change-Id: I48a8f18287aa3b017ce04d4b076d88989f927a56

releasenotes/notes/bug-2008761-d23c8b12763a6d7f.yaml

@@ -0,0 +1,9 @@
+---
+fixes:
+  - |
+    Fixes a problem where containers that do not set a reasonable nofiles
+    ulimit (e.g. ceph currently) fail to work properly due to the new 1b
+    systemd nofiles ulimit for RHEL9 and clones.
+    The default nofiles ulimit for Docker containers is now set to 1048576
+    on EL9 hosts.
+    `LP#2008761 <https://launchpad.net/bugs/[2008761]>`__

roles/docker/defaults/main.yml

@@ -43,6 +43,11 @@ docker_disable_ip_forward: "{{ docker_disable_default_iptables_rules }}"
 
 docker_runtime_directory: ""
 
+# NOTE(mnasiadka): Lower 1073741816 nofile limit on EL9 (RHEL9/CentOS Stream 9/Rocky Linux 9)
+docker_ulimit_nofile: "{{ ansible_facts.os_family == 'RedHat' and ansible_facts.distribution_major_version == '9' }}"
+docker_ulimit_nofile_hard: 1048576
+docker_ulimit_nofile_soft: 1048576
+
 # URL of docker registry
 docker_registry:
 docker_registry_insecure: false

roles/docker/tasks/config.yml

@@ -21,6 +21,12 @@
       bridge: "none"
     docker_config_ip_forward:
       ip-forward: false
+    docker_config_ulimit_nofile:
+      default-ulimits:
+        nofile:
+          name: nofile
+          hard: "{{ docker_ulimit_nofile_hard }}"
+          soft: "{{ docker_ulimit_nofile_soft }}"
     docker_config: >-
       {{ {}
          | combine(docker_zun_config if docker_configure_for_zun | bool and 'zun-compute' in group_names else {})
@@ -30,6 +36,7 @@
          | combine(docker_config_iptables if docker_disable_default_iptables_rules | bool else {})
          | combine(docker_config_bridge if docker_disable_default_network | bool else {})
          | combine(docker_config_ip_forward if docker_disable_ip_forward | bool else {})
+         | combine(docker_config_ulimit_nofile if docker_ulimit_nofile | bool else {})
          | combine(docker_custom_config) }}
   copy:
     content: "{{ docker_config | to_nice_json }}"