From 5e5a2dca0900e5c167e1ab55ef6dc14198ce2903 Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Wed, 7 Dec 2022 14:50:21 +0000 Subject: [PATCH] Set a log retention policy for OpenSearch We previously used ElasticSearch Curator for managing log retention. Now that we have moved to OpenSearch, we can use the Index State Management (ISM) plugin which is bundled with OpenSearch. This change adds support for automating the configuration of the ISM plugin via the OpenSearch API. By default, it has similar behaviour to the previous ElasticSearch Curator default policy. Closes-Bug: #2047037 Change-Id: I5c6d938f2bc380f1575ee4f16fe17c6dca37dcba --- ansible/roles/opensearch/defaults/main.yml | 48 ++++++++++++++ ansible/roles/opensearch/tasks/deploy.yml | 3 + .../roles/opensearch/tasks/post-config.yml | 65 +++++++++++++++++++ ansible/roles/opensearch/tasks/upgrade.yml | 3 + .../central-logging-guide.rst | 44 +++++++++++++ ...search-log-retention-598c3389456a67e6.yaml | 20 ++++++ 6 files changed, 183 insertions(+) create mode 100644 ansible/roles/opensearch/tasks/post-config.yml create mode 100644 releasenotes/notes/opensearch-log-retention-598c3389456a67e6.yaml diff --git a/ansible/roles/opensearch/defaults/main.yml b/ansible/roles/opensearch/defaults/main.yml index 13bc4248d5..da66a9d312 100644 --- a/ansible/roles/opensearch/defaults/main.yml +++ b/ansible/roles/opensearch/defaults/main.yml @@ -58,6 +58,54 @@ opensearch_cluster_name: "kolla_logging" opensearch_heap_size: "1g" opensearch_java_opts: "{% if opensearch_heap_size %}-Xms{{ opensearch_heap_size }} -Xmx{{ opensearch_heap_size }}{% endif %} -Dlog4j2.formatMsgNoLookups=true" +opensearch_apply_log_retention_policy: true + +# Duration after which an index is staged for deletion. This is implemented +# by closing the index. Whilst in this state the index contributes negligible +# load on the cluster and may be manually re-opened if required. +# NOTE: We carry over legacy settings from ElasticSearch Curator if they +# are set. This may be removed in a later release. +opensearch_soft_retention_period_days: "{{ elasticsearch_curator_soft_retention_period_days | default(30) }}" + +# Duration after which an index is permanently erased from the cluster. +opensearch_hard_retention_period_days: "{{ elasticsearch_curator_hard_retention_period_days | default(60) }}" + +opensearch_retention_policy: | + policy: + description: Retention policy for OpenStack logs + error_notification: + default_state: open + states: + - name: open + actions: [] + transitions: + - state_name: close + conditions: + min_index_age: "{{ opensearch_soft_retention_period_days }}d" + - name: close + actions: + - retry: + count: 3 + backoff: exponential + delay: 1m + close: {} + transitions: + - state_name: delete + conditions: + min_index_age: "{{ opensearch_hard_retention_period_days }}d" + - name: delete + actions: + - retry: + count: 3 + backoff: exponential + delay: 1m + delete: {} + transitions: [] + ism_template: + - index_patterns: + - "{{ opensearch_log_index_prefix }}-*" + priority: 1 + #################### # Keystone #################### diff --git a/ansible/roles/opensearch/tasks/deploy.yml b/ansible/roles/opensearch/tasks/deploy.yml index ee17effc62..a0ebfaf7d7 100644 --- a/ansible/roles/opensearch/tasks/deploy.yml +++ b/ansible/roles/opensearch/tasks/deploy.yml @@ -10,3 +10,6 @@ - name: Flush handlers meta: flush_handlers + +- include_tasks: post-config.yml + when: opensearch_apply_log_retention_policy | bool diff --git a/ansible/roles/opensearch/tasks/post-config.yml b/ansible/roles/opensearch/tasks/post-config.yml new file mode 100644 index 0000000000..ac26052449 --- /dev/null +++ b/ansible/roles/opensearch/tasks/post-config.yml @@ -0,0 +1,65 @@ +--- +- name: Wait for OpenSearch to become ready + become: true + kolla_toolbox: + container_engine: "{{ kolla_container_engine }}" + module_name: uri + module_args: + url: "{{ opensearch_internal_endpoint }}/_cluster/stats" + status_code: 200 + register: result + until: result.get('status') == 200 + retries: 30 + delay: 2 + run_once: true + +- name: Check if a log retention policy exists + become: true + kolla_toolbox: + container_engine: "{{ kolla_container_engine }}" + module_name: uri + module_args: + url: "{{ opensearch_internal_endpoint }}/_plugins/_ism/policies/retention" + method: GET + status_code: 200, 404 + return_content: yes + register: opensearch_retention_policy_check + delegate_to: "{{ groups['opensearch'][0] }}" + run_once: true + +- name: Create new log retention policy + become: true + kolla_toolbox: + container_engine: "{{ kolla_container_engine }}" + module_name: uri + module_args: + url: "{{ opensearch_internal_endpoint }}/_plugins/_ism/policies/retention" + method: PUT + status_code: 201 + return_content: yes + body: "{{ opensearch_retention_policy | from_yaml | to_json }}" + body_format: json + register: opensearch_retention_policy_create + delegate_to: "{{ groups['opensearch'][0] }}" + run_once: true + changed_when: opensearch_retention_policy_create.status == 201 + when: opensearch_retention_policy_check.status == 404 + +- name: Apply retention policy to existing indicies + become: true + vars: + opensearch_set_policy_body: {"policy_id": "retention"} + kolla_toolbox: + container_engine: "{{ kolla_container_engine }}" + module_name: uri + module_args: + url: "{{ opensearch_internal_endpoint }}/_plugins/_ism/add/{{ opensearch_log_index_prefix }}-*" + method: POST + status_code: 200 + return_content: yes + body: "{{ opensearch_set_policy_body | to_json }}" + body_format: json + delegate_to: "{{ groups['opensearch'][0] }}" + run_once: true + changed_when: opensearch_retention_policy_create.status == 201 + when: opensearch_retention_policy_check.status == 404 diff --git a/ansible/roles/opensearch/tasks/upgrade.yml b/ansible/roles/opensearch/tasks/upgrade.yml index 2891b64e08..da343e8b75 100644 --- a/ansible/roles/opensearch/tasks/upgrade.yml +++ b/ansible/roles/opensearch/tasks/upgrade.yml @@ -46,3 +46,6 @@ - name: Flush handlers meta: flush_handlers + +- include_tasks: post-config.yml + when: opensearch_apply_log_retention_policy | bool diff --git a/doc/source/reference/logging-and-monitoring/central-logging-guide.rst b/doc/source/reference/logging-and-monitoring/central-logging-guide.rst index 34b265a40d..0ccb5f7545 100644 --- a/doc/source/reference/logging-and-monitoring/central-logging-guide.rst +++ b/doc/source/reference/logging-and-monitoring/central-logging-guide.rst @@ -34,6 +34,50 @@ By default OpenSearch is deployed on port ``9200``. ``opensearch`` to store the data of OpenSearch. The path can be set via the variable ``opensearch_datadir_volume``. +Applying log retention policies +------------------------------- + +To stop your disks filling up, the Index State Management plugin for +OpenSearch can be used to define log retention policies. A default +retention policy is applied to all indicies which match the +``opensearch_log_index_prefix``. This policy first closes old indicies, +and then eventually deletes them. It can be customised via the following +variables: + +- ``opensearch_apply_log_retention_policy`` +- ``opensearch_soft_retention_period_days`` +- ``opensearch_hard_retention_period_days`` + +By default the soft and hard retention periods are 30 and 60 days +respectively. If you are upgrading from ElasticSearch, and have previously +configured ``elasticsearch_curator_soft_retention_period_days`` or +``elasticsearch_curator_hard_retention_period_days``, those variables will +be used instead of the defaults. You should migrate your configuration to +use the new variable names before the Caracal release. + +Advanced users may wish to customise the retention policy, which +is possible by overriding ``opensearch_retention_policy`` with +a valid policy. See the `Index Management plugin documentation `__ +for further details. + +Updating log retention policies +------------------------------- + +By design, Kolla Ansible will NOT update an existing retention +policy in OpenSearch. This is to prevent policy changes that may have +been made via the OpenSearch Dashboards UI, or external tooling, +from being wiped out. + +There are three options for modifying an existing policy: + +1. Via the OpenSearch Dashboards UI. See the `Index Management plugin documentation `__ +for further details. + +2. Via the OpenSearch API using external tooling. + +3. By manually removing the existing policy via the OpenSearch Dashboards + UI (or API), before re-applying the updated policy with Kolla Ansible. + OpenSearch Dashboards ~~~~~~~~~~~~~~~~~~~~~ diff --git a/releasenotes/notes/opensearch-log-retention-598c3389456a67e6.yaml b/releasenotes/notes/opensearch-log-retention-598c3389456a67e6.yaml new file mode 100644 index 0000000000..1df34b0b32 --- /dev/null +++ b/releasenotes/notes/opensearch-log-retention-598c3389456a67e6.yaml @@ -0,0 +1,20 @@ +--- +features: + - | + Set a log retention policy for OpenSearch via Index State Management (ISM). + `Documentation + `__. +fixes: + - | + Added log retention in OpenSearch, previously handled by Elasticsearch + Curator, now using Index State Management (ISM) OpenSearch bundled plugin. + `LP#2047037 `__. +upgrade: + - | + Added log retention in OpenSearch, previously handled by Elasticsearch + Curator. By default the soft and hard retention periods are 30 and 60 days + respectively. If you are upgrading from Elasticsearch, and have previously + configured ``elasticsearch_curator_soft_retention_period_days`` or + ``elasticsearch_curator_hard_retention_period_days``, those variables will + be used instead of the defaults. You should migrate your configuration + to use the new variable names before the Caracal release.