Elasticsearch: Make templates job more generic
This change updates the logic in our create-elasticsearch-templates job to support creation of a variety of different API objects. Change-Id: I380a55b93e7aabb606e713c21d71a383fef78b3f
This commit is contained in:
Steven Fitzpatrick
parent
1884f2c957
commit
0ab71ae35c
|
|
@ -15,7 +15,7 @@ apiVersion: v1
|
|||
appVersion: v7.1.0
|
||||
description: OpenStack-Helm ElasticSearch
|
||||
name: elasticsearch
|
||||
version: 0.1.4
|
||||
version: 0.1.5
|
||||
home: https://www.elastic.co/
|
||||
sources:
|
||||
- https://github.com/elastic/elasticsearch
|
||||
|
|
|
|||
|
|
@ -2,44 +2,9 @@
|
|||
|
||||
set -ex
|
||||
|
||||
{{ range $template, $fields := .Values.conf.templates }}
|
||||
|
||||
result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
|
||||
-XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_template/{{$template}}" \
|
||||
-H 'Content-Type: application/json' -d @/tmp/{{$template}}.json \
|
||||
| python -c "import sys, json; print(json.load(sys.stdin)['acknowledged'])")
|
||||
if [ "$result" == "True" ]; then
|
||||
echo "{{$template}} template created!"
|
||||
else
|
||||
echo "{{$template}} template not created!"
|
||||
fi
|
||||
|
||||
{{ range $object := .Values.conf.api_objects }}
|
||||
curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
|
||||
-X{{ $object.method | default "PUT" | upper }} \
|
||||
"${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/{{ $object.endpoint }}" \
|
||||
-H 'Content-Type: application/json' -d '{{ $object.body | toJson }}'
|
||||
{{ end }}
|
||||
|
||||
{{ range $policy_name, $fields := .Values.conf.snapshot_policies }}
|
||||
|
||||
result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
|
||||
-XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_slm/policy/{{$policy_name}}" \
|
||||
-H 'Content-Type: application/json' -d @/tmp/{{$policy_name}}.json \
|
||||
| python -c "import sys, json; print(json.load(sys.stdin)['acknowledged'])")
|
||||
if [ "$result" == "True" ]; then
|
||||
echo "Policy {{$policy_name}} created!"
|
||||
else
|
||||
echo "Policy {{$policy_name}} not created!"
|
||||
fi
|
||||
|
||||
{{ end }}
|
||||
|
||||
{{ range $policy_name, $fields := .Values.conf.index_policies }}
|
||||
|
||||
result=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
|
||||
-XPUT "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/_ilm/policy/{{$policy_name}}" \
|
||||
-H 'Content-Type: application/json' -d @/tmp/{{$policy_name}}.json \
|
||||
| python -c "import sys, json; print(json.load(sys.stdin)['acknowledged'])")
|
||||
if [ "$result" == "True" ]; then
|
||||
echo "Policy {{$policy_name}} created!"
|
||||
else
|
||||
echo "Policy {{$policy_name}} not created!"
|
||||
fi
|
||||
|
||||
{{ end }}
|
||||
|
|
@ -50,21 +50,18 @@ function check_snapshot_repositories () {
|
|||
{{ end }}
|
||||
}
|
||||
|
||||
{{ if and (.Values.manifests.job_elasticsearch_templates) (not (empty .Values.conf.templates)) }}
|
||||
{{ if .Values.manifests.job_elasticsearch_templates }}
|
||||
# Tests whether elasticsearch has successfully generated the elasticsearch index mapping
|
||||
# templates defined by values.yaml
|
||||
function check_templates () {
|
||||
{{ range $template, $fields := .Values.conf.templates }}
|
||||
{{$template}}_total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
|
||||
-XGET "${ELASTICSEARCH_ENDPOINT}/_template/{{$template}}" -H 'Content-Type: application/json' \
|
||||
| python -c "import sys, json; print(len(json.load(sys.stdin)))")
|
||||
if [ "${{$template}}_total_hits" -gt 0 ]; then
|
||||
echo "PASS: Successful hits on {{$template}} template!"
|
||||
total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \
|
||||
-XGET "${ELASTICSEARCH_ENDPOINT}/_template" | jq length)
|
||||
if [ "$total_hits" -gt 0 ]; then
|
||||
echo "PASS: Successful hits on templates!"
|
||||
else
|
||||
echo "FAIL: No hits on query for {{$template}} template! Exiting";
|
||||
echo "FAIL: No hits on query for templates! Exiting";
|
||||
exit 1;
|
||||
fi
|
||||
{{ end }}
|
||||
}
|
||||
{{ end }}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,35 +0,0 @@
|
|||
{{/*
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/}}
|
||||
|
||||
{{- if .Values.manifests.configmap_etc_templates }}
|
||||
{{- $envAll := . }}
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: elasticsearch-templates-etc
|
||||
type: Opaque
|
||||
data:
|
||||
{{ range $template, $fields := .Values.conf.templates }}
|
||||
{{ $template }}.json: {{ toJson $fields | b64enc }}
|
||||
{{ end }}
|
||||
{{ range $policy_name, $fields := .Values.conf.snapshot_policies }}
|
||||
{{ $policy_name }}.json: {{ toJson $fields | b64enc }}
|
||||
{{ end }}
|
||||
{{ range $policy_name, $fields := .Values.conf.index_policies }}
|
||||
{{ $policy_name }}.json: {{ toJson $fields | b64enc }}
|
||||
{{ end }}
|
||||
|
||||
{{- end }}
|
||||
|
|
@ -12,7 +12,7 @@ See the License for the specific language governing permissions and
|
|||
limitations under the License.
|
||||
*/}}
|
||||
|
||||
{{- if and (.Values.manifests.job_elasticsearch_templates) (not (empty .Values.conf.templates)) }}
|
||||
{{- if .Values.manifests.job_elasticsearch_templates }}
|
||||
{{- $envAll := . }}
|
||||
{{- $esUserSecret := .Values.secrets.elasticsearch.user }}
|
||||
{{- $mounts_elasticsearch_templates := .Values.pod.mounts.elasticsearch_templates.elasticsearch_templates }}
|
||||
|
|
@ -72,24 +72,6 @@ spec:
|
|||
mountPath: /tmp/create_template.sh
|
||||
subPath: create_template.sh
|
||||
readOnly: true
|
||||
{{ range $template, $fields := .Values.conf.templates }}
|
||||
- name: elasticsearch-templates-etc
|
||||
mountPath: /tmp/{{$template}}.json
|
||||
subPath: {{$template}}.json
|
||||
readOnly: true
|
||||
{{ end }}
|
||||
{{ range $policy_name, $fields := .Values.conf.snapshot_policies }}
|
||||
- name: elasticsearch-templates-etc
|
||||
mountPath: /tmp/{{$policy_name}}.json
|
||||
subPath: {{$policy_name}}.json
|
||||
readOnly: true
|
||||
{{ end }}
|
||||
{{ range $policy_name, $fields := .Values.conf.index_policies }}
|
||||
- name: elasticsearch-templates-etc
|
||||
mountPath: /tmp/{{$policy_name}}.json
|
||||
subPath: {{$policy_name}}.json
|
||||
readOnly: true
|
||||
{{ end }}
|
||||
{{ if $mounts_elasticsearch_templates.volumeMounts }}{{ toYaml $mounts_elasticsearch_templates.volumeMounts | indent 12 }}{{ end }}
|
||||
volumes:
|
||||
- name: pod-tmp
|
||||
|
|
@ -98,9 +80,5 @@ spec:
|
|||
configMap:
|
||||
name: elasticsearch-bin
|
||||
defaultMode: 0555
|
||||
- name: elasticsearch-templates-etc
|
||||
secret:
|
||||
secretName: elasticsearch-templates-etc
|
||||
defaultMode: 0444
|
||||
{{ if $mounts_elasticsearch_templates.volumes }}{{ toYaml $mounts_elasticsearch_templates.volumes | indent 8 }}{{ end }}
|
||||
{{- end }}
|
||||
|
|
|
|||
|
|
@ -24,11 +24,11 @@ images:
|
|||
ceph_key_placement: docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216
|
||||
s3_bucket: docker.io/openstackhelm/ceph-daemon:ubuntu_bionic-20191216
|
||||
s3_user: docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216
|
||||
helm_tests: docker.io/openstackhelm/heat:newton-ubuntu_xenial
|
||||
helm_tests: docker.io/openstackhelm/elasticsearch-s3:latest-7_6_2
|
||||
prometheus_elasticsearch_exporter: docker.io/justwatch/elasticsearch_exporter:1.1.0
|
||||
dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
|
||||
snapshot_repository: docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216
|
||||
elasticsearch_templates: docker.io/openstackhelm/heat:newton
|
||||
elasticsearch_templates: docker.io/openstackhelm/elasticsearch-s3:latest-7_6_2
|
||||
image_repo_sync: docker.io/docker:17.07.0
|
||||
pull_policy: "IfNotPresent"
|
||||
local_registry:
|
||||
|
|
@ -746,75 +746,77 @@ conf:
|
|||
ca: null
|
||||
client_private_key: null
|
||||
client_cert: null
|
||||
templates:
|
||||
fluent:
|
||||
index_patterns: "logstash-*"
|
||||
settings:
|
||||
index:
|
||||
number_of_shards: 1
|
||||
mappings:
|
||||
properties:
|
||||
kubernetes:
|
||||
properties:
|
||||
container_name:
|
||||
type: keyword
|
||||
index: false
|
||||
docker_id:
|
||||
type: keyword
|
||||
index: false
|
||||
host:
|
||||
type: keyword
|
||||
index: false
|
||||
namespace_name:
|
||||
type: keyword
|
||||
index: false
|
||||
pod_id:
|
||||
type: keyword
|
||||
index: false
|
||||
pod_name:
|
||||
type: keyword
|
||||
index: false
|
||||
snapshot_policies:
|
||||
non-security-snapshots:
|
||||
schedule: "0 30 1 * * ?"
|
||||
name: "<non-security-logs-snapshot-{now/d}>"
|
||||
repository: logstash_snapshots
|
||||
config:
|
||||
indices: ["^(.*calico-|.*ceph-|.*jenkins-|.*journal-|.*kernel_syslog-|.*kubernetes-|.*libvirt-|.*logstash-|.*openvswitch-|.*utility_access-).*$"]
|
||||
ignore_unavailable: true
|
||||
include_global_state: false
|
||||
wait_for_completion: true
|
||||
max_wait: 64800
|
||||
wait_interval: 30
|
||||
ignore_empty_list: true
|
||||
continue_if_exception: true
|
||||
disable_action: false
|
||||
retention:
|
||||
expire_after: 29d
|
||||
security-snapshots:
|
||||
schedule: "0 30 1 * * ?"
|
||||
name: "<security-logs-snapshot-{now/d}>"
|
||||
repository: logstash_snapshots
|
||||
config:
|
||||
indices: ["^(.*airship-|.*audit_tsee-|.*auth-|.*flows-|.*lma-|.*openstack-).*$"]
|
||||
ignore_unavailable: true
|
||||
include_global_state: false
|
||||
wait_for_completion: true
|
||||
max_wait: 18000
|
||||
wait_interval: 30
|
||||
ignore_empty_list: true
|
||||
continue_if_exception: true
|
||||
disable_action: false
|
||||
retention:
|
||||
expire_after: 179d
|
||||
index_policies:
|
||||
delete_all_indexes:
|
||||
policy:
|
||||
phases:
|
||||
delete:
|
||||
min_age: 14d
|
||||
actions:
|
||||
delete: {}
|
||||
api_objects:
|
||||
- endpoint: _template/fluent
|
||||
body:
|
||||
index_patterns: "logstash-*"
|
||||
settings:
|
||||
index:
|
||||
number_of_shards: 1
|
||||
mappings:
|
||||
properties:
|
||||
kubernetes:
|
||||
properties:
|
||||
container_name:
|
||||
type: keyword
|
||||
index: false
|
||||
docker_id:
|
||||
type: keyword
|
||||
index: false
|
||||
host:
|
||||
type: keyword
|
||||
index: false
|
||||
namespace_name:
|
||||
type: keyword
|
||||
index: false
|
||||
pod_id:
|
||||
type: keyword
|
||||
index: false
|
||||
pod_name:
|
||||
type: keyword
|
||||
index: false
|
||||
- endpoint: _ilm/policy/delete_all_indexes
|
||||
body:
|
||||
policy:
|
||||
phases:
|
||||
delete:
|
||||
min_age: 14d
|
||||
actions:
|
||||
delete: {}
|
||||
- endpoint: _slm/policy/non-security-snapshots
|
||||
body:
|
||||
schedule: "0 30 1 * * ?"
|
||||
name: "<non-security-logs-snapshot-{now/d}>"
|
||||
repository: logstash_snapshots
|
||||
config:
|
||||
indices: ["^(.*calico-|.*ceph-|.*jenkins-|.*journal-|.*kernel_syslog-|.*kubernetes-|.*libvirt-|.*logstash-|.*openvswitch-|.*utility_access-).*$"]
|
||||
ignore_unavailable: true
|
||||
include_global_state: false
|
||||
wait_for_completion: true
|
||||
max_wait: 64800
|
||||
wait_interval: 30
|
||||
ignore_empty_list: true
|
||||
continue_if_exception: true
|
||||
disable_action: false
|
||||
retention:
|
||||
expire_after: 29d
|
||||
- endpoint: _slm/policy/security-snapshots
|
||||
body:
|
||||
schedule: "0 30 1 * * ?"
|
||||
name: "<security-logs-snapshot-{now/d}>"
|
||||
repository: logstash_snapshots
|
||||
config:
|
||||
indices: ["^(.*airship-|.*audit_tsee-|.*auth-|.*flows-|.*lma-|.*openstack-).*$"]
|
||||
ignore_unavailable: true
|
||||
include_global_state: false
|
||||
wait_for_completion: true
|
||||
max_wait: 18000
|
||||
wait_interval: 30
|
||||
ignore_empty_list: true
|
||||
continue_if_exception: true
|
||||
disable_action: false
|
||||
retention:
|
||||
expire_after: 179d
|
||||
endpoints:
|
||||
cluster_domain_suffix: cluster.local
|
||||
local_image_registry:
|
||||
|
|
|
|||
Loading…
Reference in New Issue