diff --git a/ansible/roles/opensearch/defaults/main.yml b/ansible/roles/opensearch/defaults/main.yml index 13bc4248d5..da66a9d312 100644 --- a/ansible/roles/opensearch/defaults/main.yml +++ b/ansible/roles/opensearch/defaults/main.yml @@ -58,6 +58,54 @@ opensearch_cluster_name: "kolla_logging" opensearch_heap_size: "1g" opensearch_java_opts: "{% if opensearch_heap_size %}-Xms{{ opensearch_heap_size }} -Xmx{{ opensearch_heap_size }}{% endif %} -Dlog4j2.formatMsgNoLookups=true" +opensearch_apply_log_retention_policy: true + +# Duration after which an index is staged for deletion. This is implemented +# by closing the index. Whilst in this state the index contributes negligible +# load on the cluster and may be manually re-opened if required. +# NOTE: We carry over legacy settings from ElasticSearch Curator if they +# are set. This may be removed in a later release. +opensearch_soft_retention_period_days: "{{ elasticsearch_curator_soft_retention_period_days | default(30) }}" + +# Duration after which an index is permanently erased from the cluster. +opensearch_hard_retention_period_days: "{{ elasticsearch_curator_hard_retention_period_days | default(60) }}" + +opensearch_retention_policy: | + policy: + description: Retention policy for OpenStack logs + error_notification: + default_state: open + states: + - name: open + actions: [] + transitions: + - state_name: close + conditions: + min_index_age: "{{ opensearch_soft_retention_period_days }}d" + - name: close + actions: + - retry: + count: 3 + backoff: exponential + delay: 1m + close: {} + transitions: + - state_name: delete + conditions: + min_index_age: "{{ opensearch_hard_retention_period_days }}d" + - name: delete + actions: + - retry: + count: 3 + backoff: exponential + delay: 1m + delete: {} + transitions: [] + ism_template: + - index_patterns: + - "{{ opensearch_log_index_prefix }}-*" + priority: 1 + #################### # Keystone #################### diff --git a/ansible/roles/opensearch/tasks/deploy.yml b/ansible/roles/opensearch/tasks/deploy.yml index ee17effc62..a0ebfaf7d7 100644 --- a/ansible/roles/opensearch/tasks/deploy.yml +++ b/ansible/roles/opensearch/tasks/deploy.yml @@ -10,3 +10,6 @@ - name: Flush handlers meta: flush_handlers + +- include_tasks: post-config.yml + when: opensearch_apply_log_retention_policy | bool diff --git a/ansible/roles/opensearch/tasks/post-config.yml b/ansible/roles/opensearch/tasks/post-config.yml new file mode 100644 index 0000000000..ac26052449 --- /dev/null +++ b/ansible/roles/opensearch/tasks/post-config.yml @@ -0,0 +1,65 @@ +--- +- name: Wait for OpenSearch to become ready + become: true + kolla_toolbox: + container_engine: "{{ kolla_container_engine }}" + module_name: uri + module_args: + url: "{{ opensearch_internal_endpoint }}/_cluster/stats" + status_code: 200 + register: result + until: result.get('status') == 200 + retries: 30 + delay: 2 + run_once: true + +- name: Check if a log retention policy exists + become: true + kolla_toolbox: + container_engine: "{{ kolla_container_engine }}" + module_name: uri + module_args: + url: "{{ opensearch_internal_endpoint }}/_plugins/_ism/policies/retention" + method: GET + status_code: 200, 404 + return_content: yes + register: opensearch_retention_policy_check + delegate_to: "{{ groups['opensearch'][0] }}" + run_once: true + +- name: Create new log retention policy + become: true + kolla_toolbox: + container_engine: "{{ kolla_container_engine }}" + module_name: uri + module_args: + url: "{{ opensearch_internal_endpoint }}/_plugins/_ism/policies/retention" + method: PUT + status_code: 201 + return_content: yes + body: "{{ opensearch_retention_policy | from_yaml | to_json }}" + body_format: json + register: opensearch_retention_policy_create + delegate_to: "{{ groups['opensearch'][0] }}" + run_once: true + changed_when: opensearch_retention_policy_create.status == 201 + when: opensearch_retention_policy_check.status == 404 + +- name: Apply retention policy to existing indicies + become: true + vars: + opensearch_set_policy_body: {"policy_id": "retention"} + kolla_toolbox: + container_engine: "{{ kolla_container_engine }}" + module_name: uri + module_args: + url: "{{ opensearch_internal_endpoint }}/_plugins/_ism/add/{{ opensearch_log_index_prefix }}-*" + method: POST + status_code: 200 + return_content: yes + body: "{{ opensearch_set_policy_body | to_json }}" + body_format: json + delegate_to: "{{ groups['opensearch'][0] }}" + run_once: true + changed_when: opensearch_retention_policy_create.status == 201 + when: opensearch_retention_policy_check.status == 404 diff --git a/ansible/roles/opensearch/tasks/upgrade.yml b/ansible/roles/opensearch/tasks/upgrade.yml index 2891b64e08..da343e8b75 100644 --- a/ansible/roles/opensearch/tasks/upgrade.yml +++ b/ansible/roles/opensearch/tasks/upgrade.yml @@ -46,3 +46,6 @@ - name: Flush handlers meta: flush_handlers + +- include_tasks: post-config.yml + when: opensearch_apply_log_retention_policy | bool diff --git a/doc/source/reference/logging-and-monitoring/central-logging-guide.rst b/doc/source/reference/logging-and-monitoring/central-logging-guide.rst index 34b265a40d..0ccb5f7545 100644 --- a/doc/source/reference/logging-and-monitoring/central-logging-guide.rst +++ b/doc/source/reference/logging-and-monitoring/central-logging-guide.rst @@ -34,6 +34,50 @@ By default OpenSearch is deployed on port ``9200``. ``opensearch`` to store the data of OpenSearch. The path can be set via the variable ``opensearch_datadir_volume``. +Applying log retention policies +------------------------------- + +To stop your disks filling up, the Index State Management plugin for +OpenSearch can be used to define log retention policies. A default +retention policy is applied to all indicies which match the +``opensearch_log_index_prefix``. This policy first closes old indicies, +and then eventually deletes them. It can be customised via the following +variables: + +- ``opensearch_apply_log_retention_policy`` +- ``opensearch_soft_retention_period_days`` +- ``opensearch_hard_retention_period_days`` + +By default the soft and hard retention periods are 30 and 60 days +respectively. If you are upgrading from ElasticSearch, and have previously +configured ``elasticsearch_curator_soft_retention_period_days`` or +``elasticsearch_curator_hard_retention_period_days``, those variables will +be used instead of the defaults. You should migrate your configuration to +use the new variable names before the Caracal release. + +Advanced users may wish to customise the retention policy, which +is possible by overriding ``opensearch_retention_policy`` with +a valid policy. See the `Index Management plugin documentation `__ +for further details. + +Updating log retention policies +------------------------------- + +By design, Kolla Ansible will NOT update an existing retention +policy in OpenSearch. This is to prevent policy changes that may have +been made via the OpenSearch Dashboards UI, or external tooling, +from being wiped out. + +There are three options for modifying an existing policy: + +1. Via the OpenSearch Dashboards UI. See the `Index Management plugin documentation `__ +for further details. + +2. Via the OpenSearch API using external tooling. + +3. By manually removing the existing policy via the OpenSearch Dashboards + UI (or API), before re-applying the updated policy with Kolla Ansible. + OpenSearch Dashboards ~~~~~~~~~~~~~~~~~~~~~ diff --git a/releasenotes/notes/opensearch-log-retention-598c3389456a67e6.yaml b/releasenotes/notes/opensearch-log-retention-598c3389456a67e6.yaml new file mode 100644 index 0000000000..1df34b0b32 --- /dev/null +++ b/releasenotes/notes/opensearch-log-retention-598c3389456a67e6.yaml @@ -0,0 +1,20 @@ +--- +features: + - | + Set a log retention policy for OpenSearch via Index State Management (ISM). + `Documentation + `__. +fixes: + - | + Added log retention in OpenSearch, previously handled by Elasticsearch + Curator, now using Index State Management (ISM) OpenSearch bundled plugin. + `LP#2047037 `__. +upgrade: + - | + Added log retention in OpenSearch, previously handled by Elasticsearch + Curator. By default the soft and hard retention periods are 30 and 60 days + respectively. If you are upgrading from Elasticsearch, and have previously + configured ``elasticsearch_curator_soft_retention_period_days`` or + ``elasticsearch_curator_hard_retention_period_days``, those variables will + be used instead of the defaults. You should migrate your configuration + to use the new variable names before the Caracal release.