diff --git a/elasticsearch/Chart.yaml b/elasticsearch/Chart.yaml index 363c7f466..c9fde3c42 100644 --- a/elasticsearch/Chart.yaml +++ b/elasticsearch/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v7.1.0 description: OpenStack-Helm ElasticSearch name: elasticsearch -version: 0.1.4 +version: 0.1.5 home: https://www.elastic.co/ sources: - https://github.com/elastic/elasticsearch diff --git a/elasticsearch/templates/bin/_create_template.sh.tpl b/elasticsearch/templates/bin/_create_template.sh.tpl index 2ed3effb8..45954dd89 100644 --- a/elasticsearch/templates/bin/_create_template.sh.tpl +++ b/elasticsearch/templates/bin/_create_template.sh.tpl @@ -2,44 +2,9 @@ set -ex -{{ range $template, $fields := .Values.conf.templates }} - -result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ --XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/{{$template}}" \ --H 'Content-Type: application/json' -d @/tmp/{{$template}}.json \ -| python -c "import sys, json; print(json.load(sys.stdin)['acknowledged'])") -if [ "$result" == "True" ]; then - echo "{{$template}} template created!" -else - echo "{{$template}} template not created!" -fi - +{{ range $object := .Values.conf.api_objects }} +curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ + -X{{ $object.method | default "PUT" | upper }} \ + "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/{{ $object.endpoint }}" \ + -H 'Content-Type: application/json' -d '{{ $object.body | toJson }}' {{ end }} - -{{ range $policy_name, $fields := .Values.conf.snapshot_policies }} - -result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ --XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_slm/policy/{{$policy_name}}" \ --H 'Content-Type: application/json' -d @/tmp/{{$policy_name}}.json \ -| python -c "import sys, json; print(json.load(sys.stdin)['acknowledged'])") -if [ "$result" == "True" ]; then - echo "Policy {{$policy_name}} created!" -else - echo "Policy {{$policy_name}} not created!" -fi - -{{ end }} - -{{ range $policy_name, $fields := .Values.conf.index_policies }} - -result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ --XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ilm/policy/{{$policy_name}}" \ --H 'Content-Type: application/json' -d @/tmp/{{$policy_name}}.json \ -| python -c "import sys, json; print(json.load(sys.stdin)['acknowledged'])") -if [ "$result" == "True" ]; then - echo "Policy {{$policy_name}} created!" -else - echo "Policy {{$policy_name}} not created!" -fi - -{{ end }} \ No newline at end of file diff --git a/elasticsearch/templates/bin/_helm-tests.sh.tpl b/elasticsearch/templates/bin/_helm-tests.sh.tpl index 987d0c9d5..2df216a96 100644 --- a/elasticsearch/templates/bin/_helm-tests.sh.tpl +++ b/elasticsearch/templates/bin/_helm-tests.sh.tpl @@ -50,21 +50,18 @@ function check_snapshot_repositories () { {{ end }} } -{{ if and (.Values.manifests.job_elasticsearch_templates) (not (empty .Values.conf.templates)) }} +{{ if .Values.manifests.job_elasticsearch_templates }} # Tests whether elasticsearch has successfully generated the elasticsearch index mapping # templates defined by values.yaml function check_templates () { - {{ range $template, $fields := .Values.conf.templates }} - {{$template}}_total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ - -XGET "${ELASTICSEARCH_ENDPOINT}/_template/{{$template}}" -H 'Content-Type: application/json' \ - | python -c "import sys, json; print(len(json.load(sys.stdin)))") - if [ "${{$template}}_total_hits" -gt 0 ]; then - echo "PASS: Successful hits on {{$template}} template!" + total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ + -XGET "${ELASTICSEARCH_ENDPOINT}/_template" | jq length) + if [ "$total_hits" -gt 0 ]; then + echo "PASS: Successful hits on templates!" else - echo "FAIL: No hits on query for {{$template}} template! Exiting"; + echo "FAIL: No hits on query for templates! Exiting"; exit 1; fi - {{ end }} } {{ end }} diff --git a/elasticsearch/templates/configmap-etc-templates.yaml b/elasticsearch/templates/configmap-etc-templates.yaml deleted file mode 100644 index b8cf7c21c..000000000 --- a/elasticsearch/templates/configmap-etc-templates.yaml +++ /dev/null @@ -1,35 +0,0 @@ -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.manifests.configmap_etc_templates }} -{{- $envAll := . }} - ---- -apiVersion: v1 -kind: Secret -metadata: - name: elasticsearch-templates-etc -type: Opaque -data: -{{ range $template, $fields := .Values.conf.templates }} - {{ $template }}.json: {{ toJson $fields | b64enc }} -{{ end }} -{{ range $policy_name, $fields := .Values.conf.snapshot_policies }} - {{ $policy_name }}.json: {{ toJson $fields | b64enc }} -{{ end }} -{{ range $policy_name, $fields := .Values.conf.index_policies }} - {{ $policy_name }}.json: {{ toJson $fields | b64enc }} -{{ end }} - -{{- end }} diff --git a/elasticsearch/templates/job-elasticsearch-template.yaml b/elasticsearch/templates/job-elasticsearch-template.yaml index ef05f8515..b7c031929 100644 --- a/elasticsearch/templates/job-elasticsearch-template.yaml +++ b/elasticsearch/templates/job-elasticsearch-template.yaml @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if and (.Values.manifests.job_elasticsearch_templates) (not (empty .Values.conf.templates)) }} +{{- if .Values.manifests.job_elasticsearch_templates }} {{- $envAll := . }} {{- $esUserSecret := .Values.secrets.elasticsearch.user }} {{- $mounts_elasticsearch_templates := .Values.pod.mounts.elasticsearch_templates.elasticsearch_templates }} @@ -72,24 +72,6 @@ spec: mountPath: /tmp/create_template.sh subPath: create_template.sh readOnly: true - {{ range $template, $fields := .Values.conf.templates }} - - name: elasticsearch-templates-etc - mountPath: /tmp/{{$template}}.json - subPath: {{$template}}.json - readOnly: true - {{ end }} - {{ range $policy_name, $fields := .Values.conf.snapshot_policies }} - - name: elasticsearch-templates-etc - mountPath: /tmp/{{$policy_name}}.json - subPath: {{$policy_name}}.json - readOnly: true - {{ end }} - {{ range $policy_name, $fields := .Values.conf.index_policies }} - - name: elasticsearch-templates-etc - mountPath: /tmp/{{$policy_name}}.json - subPath: {{$policy_name}}.json - readOnly: true - {{ end }} {{ if $mounts_elasticsearch_templates.volumeMounts }}{{ toYaml $mounts_elasticsearch_templates.volumeMounts | indent 12 }}{{ end }} volumes: - name: pod-tmp @@ -98,9 +80,5 @@ spec: configMap: name: elasticsearch-bin defaultMode: 0555 - - name: elasticsearch-templates-etc - secret: - secretName: elasticsearch-templates-etc - defaultMode: 0444 {{ if $mounts_elasticsearch_templates.volumes }}{{ toYaml $mounts_elasticsearch_templates.volumes | indent 8 }}{{ end }} {{- end }} diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index c2aa13132..83ca90ae3 100644 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -24,11 +24,11 @@ images: ceph_key_placement: docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216 s3_bucket: docker.io/openstackhelm/ceph-daemon:ubuntu_bionic-20191216 s3_user: docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216 - helm_tests: docker.io/openstackhelm/heat:newton-ubuntu_xenial + helm_tests: docker.io/openstackhelm/elasticsearch-s3:latest-7_6_2 prometheus_elasticsearch_exporter: docker.io/justwatch/elasticsearch_exporter:1.1.0 dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 snapshot_repository: docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216 - elasticsearch_templates: docker.io/openstackhelm/heat:newton + elasticsearch_templates: docker.io/openstackhelm/elasticsearch-s3:latest-7_6_2 image_repo_sync: docker.io/docker:17.07.0 pull_policy: "IfNotPresent" local_registry: @@ -746,75 +746,77 @@ conf: ca: null client_private_key: null client_cert: null - templates: - fluent: - index_patterns: "logstash-*" - settings: - index: - number_of_shards: 1 - mappings: - properties: - kubernetes: - properties: - container_name: - type: keyword - index: false - docker_id: - type: keyword - index: false - host: - type: keyword - index: false - namespace_name: - type: keyword - index: false - pod_id: - type: keyword - index: false - pod_name: - type: keyword - index: false - snapshot_policies: - non-security-snapshots: - schedule: "0 30 1 * * ?" - name: "" - repository: logstash_snapshots - config: - indices: ["^(.*calico-|.*ceph-|.*jenkins-|.*journal-|.*kernel_syslog-|.*kubernetes-|.*libvirt-|.*logstash-|.*openvswitch-|.*utility_access-).*$"] - ignore_unavailable: true - include_global_state: false - wait_for_completion: true - max_wait: 64800 - wait_interval: 30 - ignore_empty_list: true - continue_if_exception: true - disable_action: false - retention: - expire_after: 29d - security-snapshots: - schedule: "0 30 1 * * ?" - name: "" - repository: logstash_snapshots - config: - indices: ["^(.*airship-|.*audit_tsee-|.*auth-|.*flows-|.*lma-|.*openstack-).*$"] - ignore_unavailable: true - include_global_state: false - wait_for_completion: true - max_wait: 18000 - wait_interval: 30 - ignore_empty_list: true - continue_if_exception: true - disable_action: false - retention: - expire_after: 179d - index_policies: - delete_all_indexes: - policy: - phases: - delete: - min_age: 14d - actions: - delete: {} + api_objects: + - endpoint: _template/fluent + body: + index_patterns: "logstash-*" + settings: + index: + number_of_shards: 1 + mappings: + properties: + kubernetes: + properties: + container_name: + type: keyword + index: false + docker_id: + type: keyword + index: false + host: + type: keyword + index: false + namespace_name: + type: keyword + index: false + pod_id: + type: keyword + index: false + pod_name: + type: keyword + index: false + - endpoint: _ilm/policy/delete_all_indexes + body: + policy: + phases: + delete: + min_age: 14d + actions: + delete: {} + - endpoint: _slm/policy/non-security-snapshots + body: + schedule: "0 30 1 * * ?" + name: "" + repository: logstash_snapshots + config: + indices: ["^(.*calico-|.*ceph-|.*jenkins-|.*journal-|.*kernel_syslog-|.*kubernetes-|.*libvirt-|.*logstash-|.*openvswitch-|.*utility_access-).*$"] + ignore_unavailable: true + include_global_state: false + wait_for_completion: true + max_wait: 64800 + wait_interval: 30 + ignore_empty_list: true + continue_if_exception: true + disable_action: false + retention: + expire_after: 29d + - endpoint: _slm/policy/security-snapshots + body: + schedule: "0 30 1 * * ?" + name: "" + repository: logstash_snapshots + config: + indices: ["^(.*airship-|.*audit_tsee-|.*auth-|.*flows-|.*lma-|.*openstack-).*$"] + ignore_unavailable: true + include_global_state: false + wait_for_completion: true + max_wait: 18000 + wait_interval: 30 + ignore_empty_list: true + continue_if_exception: true + disable_action: false + retention: + expire_after: 179d endpoints: cluster_domain_suffix: cluster.local local_image_registry: