From 96acb25b3c875d3ebc9c6f8767e17f8168036fa9 Mon Sep 17 00:00:00 2001 From: Asma Syed Hameed Date: Thu, 2 Dec 2021 11:50:01 +0530 Subject: [PATCH] Add support for SQLAlchemy collectd plugin The purpose of sqlalchemy-collectd is to monitor database usage from the perspective of the applications that are actually attaching to it, which is mostly monitoring of connection pool use. Change-Id: I59e7e4bf930a05689c0a62d943f40b2199b5a5e5 --- ansible/install/group_vars/all.yml | 5 + ansible/install/roles/collectd/tasks/main.yml | 43 +++++ .../templates/controller.collectd.conf.j2 | 25 +++ ansible/install/roles/collectd/vars/16.yml | 19 ++ ...enstack_general_system_performance.yaml.j2 | 5 + .../templates/partials/sqlalchemy.yaml | 165 ++++++++++++++++++ .../collectd-openstack/Dockerfile | 2 +- 7 files changed, 263 insertions(+), 1 deletion(-) create mode 100644 ansible/install/roles/grafana-dashboards/templates/partials/sqlalchemy.yaml diff --git a/ansible/install/group_vars/all.yml b/ansible/install/group_vars/all.yml index c2a0cc590..286113b7c 100644 --- a/ansible/install/group_vars/all.yml +++ b/ansible/install/group_vars/all.yml @@ -231,6 +231,11 @@ ceph_controller_collectd_pool_interval: 10 ceph_controller_collectd_plugin: false ceph_storage_collectd_plugin: false +############################ +# sqlalchemy plugin +########################### +sqlalchemy_collectd_plugin: false + ######################## # Gnocchi Status plugin ######################## diff --git a/ansible/install/roles/collectd/tasks/main.yml b/ansible/install/roles/collectd/tasks/main.yml index 3fd0e04b8..dfcc5a67d 100644 --- a/ansible/install/roles/collectd/tasks/main.yml +++ b/ansible/install/roles/collectd/tasks/main.yml @@ -60,6 +60,49 @@ version: master when: not (directory_exists.stat.isdir is defined and directory_exists.stat.isdir) +- name: Get mysql string + shell: "crudini --get {{item.value.path}} database connection" + become: true + register: db_connections + with_dict: "{{mysql_svcs}}" + when: ('Controller' in group_names and sqlalchemy_collectd_plugin) + ignore_errors: true + +- name: Append to mysql string + ini_file: + backup: true + dest: "{{ item.item.value.path }}" + section: database + option: connection + value: "{{ item.stdout }}&plugin=collectd&collectd_program_name={{ item.item.key }}" + when: (('Controller' in group_names) and ('plugin=collectd' not in item.stdout) and sqlalchemy_collectd_plugin) + with_items: "{{ db_connections.results }}" + become: true + +- name: restart containers + shell: podman restart {{ item }} + become: true + with_items: + - cinder_api + - cinder_api_cron + - cinder_scheduler + - nova_conductor + - nova_api_cron + - nova_scheduler + - nova_vnc_proxy + - nova_api + - nova_metadata + - neutron_api + - gnocchi_api + - keystone + - heat_api + - heat_engine + - heat_api_cron + - heat_api_cfn + ignore_errors: yes + when: ('Controller' in group_names and sqlalchemy_collectd_plugin) + + # CephStorage OSD monitoring # Only Monitors a single OSD on each CephStorage Node - name: Get 1st OSD socket diff --git a/ansible/install/roles/collectd/templates/controller.collectd.conf.j2 b/ansible/install/roles/collectd/templates/controller.collectd.conf.j2 index 2d401d5a3..bef0caa76 100644 --- a/ansible/install/roles/collectd/templates/controller.collectd.conf.j2 +++ b/ansible/install/roles/collectd/templates/controller.collectd.conf.j2 @@ -539,6 +539,31 @@ PreCacheChain "PreCache" {% endif %} +{% if sqlalchemy_collectd_plugin %} +LoadPlugin python + + LogTraces true + Import "sqlalchemy_collectd.server.plugin" + + + # ipv4 only for the moment + listen "localhost" 25827 + + # set to "debug" to show messages received + loglevel "info" + + + + Import "sqlalchemy_collectd.connmon.plugin" + + monitor "localhost" 25828 + + # set to "debug" to show messaging + #loglevel "debug" + + +{% endif %} + {%if pacemaker_monitoring %} ModulePath "/usr/local/bin/" diff --git a/ansible/install/roles/collectd/vars/16.yml b/ansible/install/roles/collectd/vars/16.yml index 6249f408d..47f030b54 100644 --- a/ansible/install/roles/collectd/vars/16.yml +++ b/ansible/install/roles/collectd/vars/16.yml @@ -161,3 +161,22 @@ collectd_logs: instance: nova-compute - log_path: /var/log/containers/openvswitch/ovn-controller.log instance: ovn-controller +mysql_svcs: + cinder: + path: "/var/lib/config-data/puppet-generated/cinder/etc/cinder/cinder.conf" + glance: + path: "/var/lib/config-data/puppet-generated/glance_api/etc/glance/glance-api.conf" + heat: + path: "/var/lib/config-data/puppet-generated/heat/etc/heat/heat.conf" + heat_api: + path: "/var/lib/config-data/puppet-generated/heat_api/etc/heat/heat.conf" + heat_api_cnf: + path: "/var/lib/config-data/puppet-generated/heat_api_cfn/etc/heat/heat.conf" + keystone: + path: "/var/lib/config-data/puppet-generated/keystone/etc/keystone/keystone.conf" + neutron: + path: "/var/lib/config-data/puppet-generated/neutron/etc/neutron/neutron.conf" + nova: + path: "/var/lib/config-data/puppet-generated/nova/etc/nova/nova.conf" + nova_metadata: + path: "/var/lib/config-data/puppet-generated/nova_metadata/etc/nova/nova.conf" diff --git a/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.yaml.j2 b/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.yaml.j2 index 52d246d28..e64ed683b 100644 --- a/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.yaml.j2 +++ b/ansible/install/roles/grafana-dashboards/templates/openstack_general_system_performance.yaml.j2 @@ -9,6 +9,7 @@ {% set ovsagent_groups = ['controller', 'compute', '*'] %} {% set rabbitmq_groups = ['undercloud', 'controller', '*'] %} {% set swift_stat_groups = ['controller', '*'] %} +{% set sqlalchemy_groups = ['controller','*'] %} {% set controller_groups = ['controller', '*'] %} --- dashboard: @@ -173,3 +174,7 @@ dashboard: {% include 'partials/ovn_metrics.yaml' %} {% include 'partials/tail.yaml' %} + +{% if item.template_node_type in sqlalchemy_groups %} + {% include 'partials/sqlalchemy.yaml' %} +{% endif %} diff --git a/ansible/install/roles/grafana-dashboards/templates/partials/sqlalchemy.yaml b/ansible/install/roles/grafana-dashboards/templates/partials/sqlalchemy.yaml new file mode 100644 index 000000000..cb7f2ea3a --- /dev/null +++ b/ansible/install/roles/grafana-dashboards/templates/partials/sqlalchemy.yaml @@ -0,0 +1,165 @@ + - title: SQLAlchemy Metrics + collapse: true + height: 200px + showTitle: true + panels: + - title: SQLAlchemy Checkedin + type: graph + fill: 0 + legend: + alignAsTable: true + avg: false + current: true + max: true + min: true + rightSide: true + show: true + total: false + values: true + nullPointMode: 'null' + targets: + - target: aliasByNode($Cloud.$Node.sqlalchemy-*.count-checkedin, 2, 3) + - title: SQLAlchemy Count Checkouts + type: graph + fill: 0 + legend: + alignAsTable: true + avg: false + current: true + max: true + min: true + rightSide: true + show: true + total: false + values: true + nullPointMode: 'null' + targets: + - target: aliasByNode($Cloud.$Node.sqlalchemy-*.count-checkedout, 2, 3) + - title: SQLAlchemy Connections + type: graph + fill: 0 + legend: + alignAsTable: true + avg: false + current: true + max: true + min: true + rightSide: true + show: true + total: false + values: true + nullPointMode: 'null' + targets: + - target: aliasByNode($Cloud.$Node.sqlalchemy-*.count-connections, 2, 3) + - title: SQLAlchemy Count Detached + type: graph + fill: 0 + legend: + alignAsTable: true + avg: false + current: true + max: true + min: true + rightSide: true + show: true + total: false + values: true + nullPointMode: 'null' + targets: + - target: aliasByNode($Cloud.$Node.sqlalchemy-*.count-detached, 2, 3) + - title: SQLAlchemy Count NumPools + type: graph + fill: 0 + legend: + alignAsTable: true + avg: false + current: true + max: true + min: true + rightSide: true + show: true + total: false + values: true + nullPointMode: 'null' + targets: + - target: aliasByNode($Cloud.$Node.sqlalchemy-*.count-numpools, 2, 3) + - title: SQLAlchemy Count NumProcs + type: graph + fill: 0 + legend: + alignAsTable: true + avg: false + current: true + max: true + min: true + rightSide: true + show: true + total: false + values: true + nullPointMode: 'null' + targets: + - target: aliasByNode($Cloud.$Node.sqlalchemy-*.count-numprocs, 2, 3) + - title: SQLAlchemy Derive Checkouts + type: graph + fill: 0 + legend: + alignAsTable: true + avg: false + current: true + max: true + min: true + rightSide: true + show: true + total: false + values: true + nullPointMode: 'null' + targets: + - target: aliasByNode($Cloud.$Node.sqlalchemy-*.derive-checkouts, 2, 3) + - title: SQLAlchemy Derive Connects + type: graph + fill: 0 + legend: + alignAsTable: true + avg: false + current: true + max: true + min: true + rightSide: true + show: true + total: false + values: true + nullPointMode: 'null' + targets: + - target: aliasByNode($Cloud.$Node.sqlalchemy-*.derive-connects, 2, 3) + - title: SQLAlchemy Derive Disconnects + type: graph + fill: 0 + legend: + alignAsTable: true + avg: false + current: true + max: true + min: true + rightSide: true + show: true + total: false + values: true + nullPointMode: 'null' + targets: + - target: aliasByNode($Cloud.$Node.sqlalchemy-*.derive-disconnects, 2, 3) + - title: SQLAlchemy Derive Invalidated + type: graph + fill: 0 + legend: + alignAsTable: true + avg: false + current: true + max: true + min: true + rightSide: true + show: true + total: false + values: true + nullPointMode: 'null' + targets: + - target: aliasByNode($Cloud.$Node.sqlalchemy-*.derive-invalidated, 2, 3) diff --git a/browbeat-containers/collectd-openstack/Dockerfile b/browbeat-containers/collectd-openstack/Dockerfile index 20f1667ea..3c0b5d8ac 100644 --- a/browbeat-containers/collectd-openstack/Dockerfile +++ b/browbeat-containers/collectd-openstack/Dockerfile @@ -5,7 +5,7 @@ RUN dnf clean all && \ dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \ dnf install -y centos-release-opstools && \ dnf install -y collectd collectd-turbostat collectd-disk collectd-apache collectd-ceph \ - collectd-mysql collectd-python collectd-ping && \ + collectd-mysql collectd-python collectd-ping python3-sqlalchemy-collectd && \ dnf install -y sysstat && \ dnf install -y python3-pip python3-devel && \ pip3 install --upgrade pip && \