Merge "Set a log retention policy for OpenSearch"

ansible/roles/opensearch/defaults/main.yml

@@ -58,6 +58,54 @@ opensearch_cluster_name: "kolla_logging"
 opensearch_heap_size: "1g"
 opensearch_java_opts: "{% if opensearch_heap_size %}-Xms{{ opensearch_heap_size }} -Xmx{{ opensearch_heap_size }}{% endif %} -Dlog4j2.formatMsgNoLookups=true"
 
+opensearch_apply_log_retention_policy: true
+
+# Duration after which an index is staged for deletion. This is implemented
+# by closing the index. Whilst in this state the index contributes negligible
+# load on the cluster and may be manually re-opened if required.
+# NOTE: We carry over legacy settings from ElasticSearch Curator if they
+# are set. This may be removed in a later release.
+opensearch_soft_retention_period_days: "{{ elasticsearch_curator_soft_retention_period_days | default(30) }}"
+
+# Duration after which an index is permanently erased from the cluster.
+opensearch_hard_retention_period_days: "{{ elasticsearch_curator_hard_retention_period_days | default(60) }}"
+
+opensearch_retention_policy: |
+  policy:
+    description: Retention policy for OpenStack logs
+    error_notification:
+    default_state: open
+    states:
+    - name: open
+      actions: []
+      transitions:
+      - state_name: close
+        conditions:
+          min_index_age: "{{ opensearch_soft_retention_period_days }}d"
+    - name: close
+      actions:
+      - retry:
+          count: 3
+          backoff: exponential
+          delay: 1m
+        close: {}
+      transitions:
+      - state_name: delete
+        conditions:
+          min_index_age: "{{ opensearch_hard_retention_period_days }}d"
+    - name: delete
+      actions:
+      - retry:
+          count: 3
+          backoff: exponential
+          delay: 1m
+        delete: {}
+      transitions: []
+    ism_template:
+    - index_patterns:
+      - "{{ opensearch_log_index_prefix }}-*"
+      priority: 1
+
 ####################
 # Keystone
 ####################

ansible/roles/opensearch/tasks/deploy.yml

@@ -10,3 +10,6 @@
 
 - name: Flush handlers
   meta: flush_handlers
+
+- include_tasks: post-config.yml
+  when: opensearch_apply_log_retention_policy | bool

ansible/roles/opensearch/tasks/post-config.yml

@@ -0,0 +1,65 @@
+---
+- name: Wait for OpenSearch to become ready
+  become: true
+  kolla_toolbox:
+    container_engine: "{{ kolla_container_engine }}"
+    module_name: uri
+    module_args:
+      url: "{{ opensearch_internal_endpoint }}/_cluster/stats"
+      status_code: 200
+  register: result
+  until: result.get('status') == 200
+  retries: 30
+  delay: 2
+  run_once: true
+
+- name: Check if a log retention policy exists
+  become: true
+  kolla_toolbox:
+    container_engine: "{{ kolla_container_engine }}"
+    module_name: uri
+    module_args:
+      url: "{{ opensearch_internal_endpoint }}/_plugins/_ism/policies/retention"
+      method: GET
+      status_code: 200, 404
+      return_content: yes
+  register: opensearch_retention_policy_check
+  delegate_to: "{{ groups['opensearch'][0] }}"
+  run_once: true
+
+- name: Create new log retention policy
+  become: true
+  kolla_toolbox:
+    container_engine: "{{ kolla_container_engine }}"
+    module_name: uri
+    module_args:
+      url: "{{ opensearch_internal_endpoint }}/_plugins/_ism/policies/retention"
+      method: PUT
+      status_code: 201
+      return_content: yes
+      body: "{{ opensearch_retention_policy | from_yaml | to_json }}"
+      body_format: json
+  register: opensearch_retention_policy_create
+  delegate_to: "{{ groups['opensearch'][0] }}"
+  run_once: true
+  changed_when: opensearch_retention_policy_create.status == 201
+  when: opensearch_retention_policy_check.status == 404
+
+- name: Apply retention policy to existing indicies
+  become: true
+  vars:
+    opensearch_set_policy_body: {"policy_id": "retention"}
+  kolla_toolbox:
+    container_engine: "{{ kolla_container_engine }}"
+    module_name: uri
+    module_args:
+      url: "{{ opensearch_internal_endpoint }}/_plugins/_ism/add/{{ opensearch_log_index_prefix }}-*"
+      method: POST
+      status_code: 200
+      return_content: yes
+      body: "{{ opensearch_set_policy_body | to_json }}"
+      body_format: json
+  delegate_to: "{{ groups['opensearch'][0] }}"
+  run_once: true
+  changed_when: opensearch_retention_policy_create.status == 201
+  when: opensearch_retention_policy_check.status == 404

ansible/roles/opensearch/tasks/upgrade.yml

@@ -46,3 +46,6 @@
 
 - name: Flush handlers
   meta: flush_handlers
+
+- include_tasks: post-config.yml
+  when: opensearch_apply_log_retention_policy | bool

doc/source/reference/logging-and-monitoring/central-logging-guide.rst

@@ -34,6 +34,50 @@ By default OpenSearch is deployed on port ``9200``.
    ``opensearch`` to store the data of OpenSearch. The path can be set via
    the variable ``opensearch_datadir_volume``.
 
+Applying log retention policies
+-------------------------------
+
+To stop your disks filling up, the Index State Management plugin for
+OpenSearch can be used to define log retention policies. A default
+retention policy is applied to all indicies which match the
+``opensearch_log_index_prefix``. This policy first closes old indicies,
+and then eventually deletes them. It can be customised via the following
+variables:
+
+- ``opensearch_apply_log_retention_policy``
+- ``opensearch_soft_retention_period_days``
+- ``opensearch_hard_retention_period_days``
+
+By default the soft and hard retention periods are 30 and 60 days
+respectively. If you are upgrading from ElasticSearch, and have previously
+configured ``elasticsearch_curator_soft_retention_period_days`` or
+``elasticsearch_curator_hard_retention_period_days``, those variables will
+be used instead of the defaults. You should migrate your configuration to
+use the new variable names before the Caracal release.
+
+Advanced users may wish to customise the retention policy, which
+is possible by overriding ``opensearch_retention_policy`` with
+a valid policy. See the `Index Management plugin documentation <https://opensearch.org/docs/latest/im-plugin/index/>`__
+for further details.
+
+Updating log retention policies
+-------------------------------
+
+By design, Kolla Ansible will NOT update an existing retention
+policy in OpenSearch. This is to prevent policy changes that may have
+been made via the OpenSearch Dashboards UI, or external tooling,
+from being wiped out.
+
+There are three options for modifying an existing policy:
+
+1. Via the OpenSearch Dashboards UI. See the `Index Management plugin documentation <https://opensearch.org/docs/latest/im-plugin/index/>`__
+for further details.
+
+2. Via the OpenSearch API using external tooling.
+
+3. By manually removing the existing policy via the OpenSearch Dashboards
+   UI (or API), before re-applying the updated policy with Kolla Ansible.
+
 OpenSearch Dashboards
 ~~~~~~~~~~~~~~~~~~~~~
 

releasenotes/notes/opensearch-log-retention-598c3389456a67e6.yaml

@@ -0,0 +1,20 @@
+---
+features:
+  - |
+    Set a log retention policy for OpenSearch via Index State Management (ISM).
+    `Documentation
+    <https://docs.openstack.org/kolla-ansible/latest/reference/logging-and-monitoring/central-logging-guide.html#applying-log-retention-policies>`__.
+fixes:
+  - |
+    Added log retention in OpenSearch, previously handled by Elasticsearch
+    Curator, now using Index State Management (ISM) OpenSearch bundled plugin.
+    `LP#2047037 <https://bugs.launchpad.net/kolla-ansible/+bug/2047037>`__.
+upgrade:
+  - |
+    Added log retention in OpenSearch, previously handled by Elasticsearch
+    Curator. By default the soft and hard retention periods are 30 and 60 days
+    respectively. If you are upgrading from Elasticsearch, and have previously
+    configured ``elasticsearch_curator_soft_retention_period_days`` or
+    ``elasticsearch_curator_hard_retention_period_days``, those variables will
+    be used instead of the defaults. You should migrate your configuration
+    to use the new variable names before the Caracal release.