diff --git a/elk_metrics_6x/README.rst b/elk_metrics_6x/README.rst index ecba8352..cc0e40f1 100644 --- a/elk_metrics_6x/README.rst +++ b/elk_metrics_6x/README.rst @@ -570,6 +570,22 @@ state variable, `elk_package_state`, to latest. ansible-playbook site.yml $USER_VARS -e 'elk_package_state="latest"' +Forcing the Elasticsearch cluster retention policy to refresh +------------------------------------------------------------- + +To force the cluster retention policy to refresh set `elastic_retention_refresh`, to +"yes". When setting `elastic_retention_refresh` to "yes" the retention policy will forcibly +be refresh across all hosts. This option should only be used when the Elasticsearch storage +array is modified on an existing cluster. Should the Elasticseach cluster size change +(nodes added or removed) the retention policy will automatically be refreshed on playbook +execution. + +.. code-block:: bash + + cd /opt/openstack-ansible-ops/elk_metrics_6x + ansible-playbook site.yml $USER_VARS -e 'elastic_retention_refresh="yes"' + + Trouble shooting ---------------- diff --git a/elk_metrics_6x/roles/elastic_curator/tasks/main.yml b/elk_metrics_6x/roles/elastic_curator/tasks/main.yml index c40c492d..9aa6681c 100644 --- a/elk_metrics_6x/roles/elastic_curator/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_curator/tasks/main.yml @@ -30,6 +30,13 @@ when: ansible_service_mgr != 'systemd' +- name: Refresh local facts + setup: + filter: ansible_local + gather_subset: "!all" + tags: + - always + - name: Ensure virtualenv is installed package: name: "{{ curator_distro_packages }}" diff --git a/elk_metrics_6x/roles/elastic_curator/templates/curator-actions-age.yml.j2 b/elk_metrics_6x/roles/elastic_curator/templates/curator-actions-age.yml.j2 index 2c7b337b..9f2ed25e 100644 --- a/elk_metrics_6x/roles/elastic_curator/templates/curator-actions-age.yml.j2 +++ b/elk_metrics_6x/roles/elastic_curator/templates/curator-actions-age.yml.j2 @@ -15,10 +15,10 @@ {% set action_items = [] -%} {# Delete index loop #} -{% for key in elastic_beat_retention_policy_keys -%} +{% for key in (ansible_local['elastic']['retention']['elastic_beat_retention_policy_keys'] | from_yaml) -%} {% set delete_indices = {} -%} {# Total retention size in days #} -{% set _index_retention = hostvars[inventory_hostname]['elastic_' + key + '_retention'] -%} +{% set _index_retention = ansible_local['elastic']['retention']['elastic_' + key + '_retention'] -%} {% set index_retention = ((_index_retention | int) > 0) | ternary(_index_retention, 1) | int %} {% set _ = delete_indices.update( { diff --git a/elk_metrics_6x/roles/elastic_curator/templates/curator-actions-size.yml.j2 b/elk_metrics_6x/roles/elastic_curator/templates/curator-actions-size.yml.j2 index b0f5f488..88e1c3c4 100644 --- a/elk_metrics_6x/roles/elastic_curator/templates/curator-actions-size.yml.j2 +++ b/elk_metrics_6x/roles/elastic_curator/templates/curator-actions-size.yml.j2 @@ -15,10 +15,10 @@ {% set action_items = [] -%} {# Delete index loop #} -{% for key in elastic_beat_retention_policy_keys -%} +{% for key in (ansible_local['elastic']['retention']['elastic_beat_retention_policy_keys'] | from_yaml) -%} {% set delete_indices = {} -%} {# Total retention size in gigabytes #} -{% set _index_size = ((hostvars[inventory_hostname]['elastic_' + key + '_size'] | int) // 1024) -%} +{% set _index_size = ((ansible_local['elastic']['retention']['elastic_' + key + '_size'] | int) // 1024) -%} {% set index_size = ((_index_size | int) > 0) | ternary(_index_size, 1) | int %} {% set _ = delete_indices.update( { diff --git a/elk_metrics_6x/roles/elastic_retention/defaults/main.yml b/elk_metrics_6x/roles/elastic_retention/defaults/main.yml index 0019cfab..1bb40529 100644 --- a/elk_metrics_6x/roles/elastic_retention/defaults/main.yml +++ b/elk_metrics_6x/roles/elastic_retention/defaults/main.yml @@ -94,3 +94,6 @@ elastic_beat_retention_policy_hosts: packetbeat: weight: 1 hosts: "{{ groups['hosts'] | default([]) }}" + +# Refresh the elasticsearch retention policy local facts. +elastic_retention_refresh: false \ No newline at end of file diff --git a/elk_metrics_6x/roles/elastic_retention/tasks/main.yml b/elk_metrics_6x/roles/elastic_retention/tasks/main.yml index 1a147563..9e096773 100644 --- a/elk_metrics_6x/roles/elastic_retention/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_retention/tasks/main.yml @@ -13,40 +13,90 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Query es storage - uri: - url: "http://{{ coordination_nodes[0] }}/_nodes/{{ (data_nodes | map('extract', hostvars, 'ansible_host') | list) | join(',') }}/stats/fs" - method: GET - register: elk_data - environment: - no_proxy: "{{ coordination_nodes[0].split(':')[0] }}" - until: - - elk_data is success and elk_data['json'] is defined - retries: 5 - delay: 30 - run_once: true +- name: Ensure local facts directory exists + file: + dest: "/etc/ansible/facts.d" + state: directory + group: "root" + owner: "root" + mode: "0755" + recurse: no -- name: Set retention keys fact - set_fact: - es_storage_json: "{{ elk_data['json'] }}" +- name: Initialize local facts + ini_file: + dest: "/etc/ansible/facts.d/elastic.fact" + section: "retention" + option: cacheable + value: true -- name: Load retention algo variables - include_vars: "calculate_index_retention_{{ elastic_index_retention_algorithm }}.yml" +- name: Refresh local facts + setup: + filter: ansible_local + gather_subset: "!all" tags: - always -- name: Set retention facts (mb size) - set_fact: "elastic_{{ item.key }}_size={{ item.value }}" - when: - - hostvars[inventory_hostname]["elastic_" ~ item.key ~ "_size"] is undefined - with_dict: "{{ es_storage_per_index }}" +- name: Retention storage block + block: + - name: Query es storage + uri: + url: "http://{{ coordination_nodes[0] }}/_nodes/{{ (data_nodes | map('extract', hostvars, 'ansible_host') | list) | join(',') }}/stats/fs" + method: GET + register: elk_data + environment: + no_proxy: "{{ coordination_nodes[0].split(':')[0] }}" + until: + - elk_data is success and elk_data['json'] is defined + retries: 5 + delay: 30 + run_once: true -- name: Set retention facts (days) - set_fact: "elastic_{{ item.key }}_retention={{ item.value }}" - when: - - hostvars[inventory_hostname]["elastic_" ~ item.key ~ "_retention"] is undefined - with_dict: "{{ es_days_per_index }}" + - name: Set retention keys fact + set_fact: + es_storage_json: "{{ elk_data['json'] }}" -- name: Set retention keys fact - set_fact: - elastic_beat_retention_policy_keys: "{{ elastic_beat_retention_policy_hosts.keys() | list }}" + - name: Load retention algo variables + include_vars: "calculate_index_retention_{{ elastic_index_retention_algorithm }}.yml" + tags: + - always + + - name: Set storage fact + ini_file: + dest: "/etc/ansible/facts.d/elastic.fact" + section: "retention" + option: "cluster_nodes" + value: "{{ groups['elastic-logstash'] | length }}" + + - name: Set retention policy fact + ini_file: + dest: "/etc/ansible/facts.d/elastic.fact" + section: "retention" + option: "elastic_beat_retention_policy_keys" + value: "{{ elastic_beat_retention_policy_hosts.keys() | list }}" + + - name: Set size fact + ini_file: + dest: "/etc/ansible/facts.d/elastic.fact" + section: "retention" + option: "elastic_{{ item.key }}_size" + value: "{{ item.value }}" + with_dict: "{{ es_storage_per_index }}" + + - name: Set retention fact + ini_file: + dest: "/etc/ansible/facts.d/elastic.fact" + section: "retention" + option: "elastic_{{ item.key }}_retention" + value: "{{ item.value }}" + with_dict: "{{ es_days_per_index }}" + + - name: Refresh local facts + setup: + filter: ansible_local + gather_subset: "!all" + tags: + - always + when: + - (ansible_local['elastic']['retention']['cluster_nodes'] is undefined) or + ((groups['elastic-logstash'] | length) != (ansible_local['elastic']['retention']['cluster_nodes'] | int)) or + (elastic_retention_refresh | bool) diff --git a/elk_metrics_6x/roles/elastic_rollup/tasks/main.yml b/elk_metrics_6x/roles/elastic_rollup/tasks/main.yml index 77cdf542..96caefdd 100644 --- a/elk_metrics_6x/roles/elastic_rollup/tasks/main.yml +++ b/elk_metrics_6x/roles/elastic_rollup/tasks/main.yml @@ -33,6 +33,7 @@ OK with the old rollup job being purged, add the following option `elastic_allow_rollup_purge=yes` to the command line and rerun the playbook. + run_once: true when: - check_rollup['json']['jobs'] | length > 0 - not elastic_allow_rollup_purge | bool @@ -44,7 +45,7 @@ min_days_until_rollup: |- {% set index_retention = [] %} {% for item in ansible_play_hosts %} - {% set _ = index_retention.append(hostvars[item]['elastic_' + index_name + '_retention'] | int) %} + {% set _ = index_retention.append(ansible_local['elastic']['retention']['elastic_' + index_name + '_retention'] | int) %} {% endfor %} {{ index_retention | min }} run_once: true