diff --git a/elasticsearch/templates/bin/_elasticsearch.sh.tpl b/elasticsearch/templates/bin/_elasticsearch.sh.tpl index f862703c..780ec6e7 100644 --- a/elasticsearch/templates/bin/_elasticsearch.sh.tpl +++ b/elasticsearch/templates/bin/_elasticsearch.sh.tpl @@ -19,7 +19,8 @@ set -ex COMMAND="${@:-start}" function start () { - exec /usr/share/elasticsearch/bin/elasticsearch + ulimit -l unlimited + exec /docker-entrypoint.sh elasticsearch } function stop () { diff --git a/elasticsearch/templates/bin/_helm-tests.sh.tpl b/elasticsearch/templates/bin/_helm-tests.sh.tpl index b74581a9..b55e9176 100644 --- a/elasticsearch/templates/bin/_helm-tests.sh.tpl +++ b/elasticsearch/templates/bin/_helm-tests.sh.tpl @@ -19,7 +19,7 @@ limitations under the License. set -ex function create_index () { - index_result=$(curl -XPUT "$ELASTICSEARCH_ENDPOINT/test_index?pretty" -H 'Content-Type: application/json' -d' + index_result=$(curl -XPUT "${ELASTICSEARCH_ENDPOINT}test_index?pretty" -H 'Content-Type: application/json' -d' { "settings" : { "index" : { @@ -39,7 +39,7 @@ function create_index () { } function insert_test_data () { - insert_result=$(curl -XPUT "$ELASTICSEARCH_ENDPOINT/sample_index/sample_type/123/_create?pretty" -H 'Content-Type: application/json' -d' + insert_result=$(curl -XPUT "${ELASTICSEARCH_ENDPOINT}sample_index/sample_type/123/_create?pretty" -H 'Content-Type: application/json' -d' { "name" : "Elasticsearch", "message" : "Test data text entry" @@ -56,7 +56,7 @@ function insert_test_data () { function check_hits () { - total_hits=$(curl -XGET "$ELASTICSEARCH_ENDPOINT/_search?pretty" -H 'Content-Type: application/json' -d' + total_hits=$(curl -XGET "${ELASTICSEARCH_ENDPOINT}_search?pretty" -H 'Content-Type: application/json' -d' { "query" : { "bool": { diff --git a/elasticsearch/templates/configmap-etc.yaml b/elasticsearch/templates/configmap-etc.yaml index 532137b4..586ce580 100644 --- a/elasticsearch/templates/configmap-etc.yaml +++ b/elasticsearch/templates/configmap-etc.yaml @@ -21,5 +21,5 @@ metadata: data: elasticsearch.yml: |+ {{- tuple .Values.conf.elasticsearch "etc/_elasticsearch.yml.tpl" . | include "helm-toolkit.utils.configmap_templater" }} - logging.yml: |+ -{{- tuple .Values.conf.elasticsearch "etc/_logging.yml.tpl" . | include "helm-toolkit.utils.configmap_templater" }} + log4j2.properties: |+ +{{- tuple .Values.conf.elasticsearch "etc/_log4j2.properties.tpl" . | include "helm-toolkit.utils.configmap_templater" }} diff --git a/elasticsearch/templates/deployment-client.yaml b/elasticsearch/templates/deployment-client.yaml index 7f325bbf..9cd02372 100644 --- a/elasticsearch/templates/deployment-client.yaml +++ b/elasticsearch/templates/deployment-client.yaml @@ -39,6 +39,7 @@ spec: - name: memory-map-increase securityContext: privileged: true + runAsUser: 0 image: {{ .Values.images.memory_init }} imagePullPolicy: {{ .Values.images.pull_policy }} command: @@ -100,10 +101,10 @@ spec: subPath: elasticsearch.yml readOnly: true - name: elastic-etc - mountPath: /usr/share/elasticsearch/config/logging.yml - subPath: logging.yml + mountPath: /usr/share/elasticsearch/config/log4j2.properties + subPath: log4j2.properties readOnly: true - - mountPath: /data + - mountPath: /var/lib/elasticsearch/data name: storage volumes: - name: elastic-bin diff --git a/elasticsearch/templates/deployment-master.yaml b/elasticsearch/templates/deployment-master.yaml index 061613b8..4e1e65af 100644 --- a/elasticsearch/templates/deployment-master.yaml +++ b/elasticsearch/templates/deployment-master.yaml @@ -39,6 +39,7 @@ spec: - name: memory-map-increase securityContext: privileged: true + runAsUser: 0 image: {{ .Values.images.memory_init }} imagePullPolicy: {{ .Values.images.pull_policy }} command: @@ -103,7 +104,11 @@ spec: mountPath: /usr/share/elasticsearch/config/logging.yml subPath: logging.yml readOnly: true - - mountPath: /data + - name: elastic-etc + mountPath: /usr/share/elasticsearch/config/log4j2.properties + subPath: log4j2.properties + readOnly: true + - mountPath: /var/lib/elasticsearch/data name: storage volumes: - name: elastic-bin diff --git a/elasticsearch/templates/etc/_elasticsearch.yml.tpl b/elasticsearch/templates/etc/_elasticsearch.yml.tpl index 631b61c7..b150e5b6 100644 --- a/elasticsearch/templates/etc/_elasticsearch.yml.tpl +++ b/elasticsearch/templates/etc/_elasticsearch.yml.tpl @@ -21,6 +21,7 @@ node: master: ${NODE_MASTER} data: ${NODE_DATA} name: ${NODE_NAME} + max_local_storage_nodes: {{ .Values.pod.replicas.data }} network.host: {{ .Values.conf.elasticsearch.network.host }} diff --git a/elasticsearch/templates/etc/_log4j2.properties.tpl b/elasticsearch/templates/etc/_log4j2.properties.tpl new file mode 100644 index 00000000..bfc58626 --- /dev/null +++ b/elasticsearch/templates/etc/_log4j2.properties.tpl @@ -0,0 +1,34 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +appender.rolling.type = RollingFile +appender.rolling.name = rolling +appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log +appender.rolling.layout.type = PatternLayout +appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %.10000m%n +appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}.log +appender.rolling.policies.type = Policies +appender.rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling.policies.time.interval = 1 +appender.rolling.policies.time.modulate = true + +status = error + +appender.console.type = Console +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n + +rootLogger.level = info +rootLogger.appenderRef.console.ref = console diff --git a/elasticsearch/templates/etc/_logging.yml.tpl b/elasticsearch/templates/etc/_logging.yml.tpl deleted file mode 100644 index 4fc7013c..00000000 --- a/elasticsearch/templates/etc/_logging.yml.tpl +++ /dev/null @@ -1,95 +0,0 @@ -{{/* -Copyright 2017 The Openstack-Helm Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -# you can override this using by setting a system property, for example -Des.logger.level=DEBUG -es.logger.level: INFO -rootLogger: ${es.logger.level}, console, file -logger: - # log action execution errors for easier debugging - action: DEBUG - - # deprecation logging, turn to DEBUG to see them - deprecation: INFO, deprecation_log_file - - org.apache.http: INFO - - # gateway - #gateway: DEBUG - #index.gateway: DEBUG - - # peer shard recovery - #indices.recovery: DEBUG - - # discovery - #discovery: TRACE - - index.search.slowlog: TRACE, index_search_slow_log_file - index.indexing.slowlog: TRACE, index_indexing_slow_log_file - -additivity: - index.search.slowlog: false - index.indexing.slowlog: false - deprecation: false - -appender: - console: - type: console - layout: - type: consolePattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" - - file: - type: dailyRollingFile - file: ${path.logs}/${cluster.name}.log - datePattern: "'.'yyyy-MM-dd" - layout: - type: pattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %.10000m%n" - - # Use the following log4j-extras RollingFileAppender to enable gzip compression of log files. - # For more information see https://logging.apache.org/log4j/extras/apidocs/org/apache/log4j/rolling/RollingFileAppender.html - #file: - #type: extrasRollingFile - #file: ${path.logs}/${cluster.name}.log - #rollingPolicy: timeBased - #rollingPolicy.FileNamePattern: ${path.logs}/${cluster.name}.log.%d{yyyy-MM-dd}.gz - #layout: - #type: pattern - #conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" - - deprecation_log_file: - type: dailyRollingFile - file: ${path.logs}/${cluster.name}_deprecation.log - datePattern: "'.'yyyy-MM-dd" - layout: - type: pattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" - - index_search_slow_log_file: - type: dailyRollingFile - file: ${path.logs}/${cluster.name}_index_search_slowlog.log - datePattern: "'.'yyyy-MM-dd" - layout: - type: pattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" - - index_indexing_slow_log_file: - type: dailyRollingFile - file: ${path.logs}/${cluster.name}_index_indexing_slowlog.log - datePattern: "'.'yyyy-MM-dd" - layout: - type: pattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" diff --git a/elasticsearch/templates/statefulset-data.yaml b/elasticsearch/templates/statefulset-data.yaml index 15b0709e..e2f2f388 100644 --- a/elasticsearch/templates/statefulset-data.yaml +++ b/elasticsearch/templates/statefulset-data.yaml @@ -36,6 +36,7 @@ spec: - name: memory-map-increase securityContext: privileged: true + runAsUser: 0 image: {{ .Values.images.memory_init }} imagePullPolicy: {{ .Values.images.pull_policy }} {{ tuple $envAll $envAll.Values.pod.resources.data | include "helm-toolkit.snippets.kubernetes_resources" | indent 8 }} @@ -95,8 +96,8 @@ spec: subPath: elasticsearch.yml readOnly: true - name: elastic-etc - mountPath: /usr/share/elasticsearch/config/logging.yml - subPath: logging.yml + mountPath: /usr/share/elasticsearch/config/log4j2.properties + subPath: log4j2.properties readOnly: true - name: storage mountPath: /var/lib/elasticsearch/data diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index af74e700..4506ea16 100644 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -35,7 +35,7 @@ conf: min_masters: 2 env: java_opts: "-Xms256m -Xmx256m" - logging: + log4j2: override: prefix: append: @@ -60,9 +60,9 @@ endpoints: default: 9200 images: - memory_init: docker.io/kolla/ubuntu-source-elasticsearch:4.0.0 - elasticsearch: docker.io/kolla/ubuntu-source-elasticsearch:4.0.0 - helm_tests: docker.io/kolla/ubuntu-source-elasticsearch:4.0.0 + memory_init: docker.io/kolla/ubuntu-source-kolla-toolbox:4.0.0 + elasticsearch: docker.io/elasticsearch:5.4.2 + helm_tests: docker.io/kolla/ubuntu-source-kolla-toolbox:3.0.3 pull_policy: "IfNotPresent" pod: diff --git a/fluentd/templates/bin/_fluentd-test.sh.tpl b/fluentd/templates/bin/_fluentd-test.sh.tpl index 26bb59a7..b299999b 100644 --- a/fluentd/templates/bin/_fluentd-test.sh.tpl +++ b/fluentd/templates/bin/_fluentd-test.sh.tpl @@ -21,7 +21,7 @@ set -ex # Tests whether fluentd has successfully indexed data into Elasticsearch under # the logstash-* index via the fluent-elasticsearch plugin function check_logstash_index () { - total_hits=$(curl -XGET "$ELASTICSEARCH_ENDPOINT/logstash-*/fluentd/_search?pretty" -H 'Content-Type: application/json' \ + total_hits=$(curl -XGET "${ELASTICSEARCH_ENDPOINT}logstash-*/fluentd/_search?pretty" -H 'Content-Type: application/json' \ | python -c "import sys, json; print json.load(sys.stdin)['hits']['total']") if [ "$total_hits" -gt 0 ]; then echo "PASS: Successful hits on logstash-* index, provided by fluentd!" @@ -34,7 +34,7 @@ function check_logstash_index () { # Tests whether fluentd has successfully tagged data with the kubernetes.var.* # prefix via the fluent-kubernetes plugin function check_kubernetes_tag () { - total_hits=$(curl -XGET "$ELASTICSEARCH_ENDPOINT/logstash-*/fluentd/_search?q=tag:kubernetes.var.*" -H 'Content-Type: application/json' \ + total_hits=$(curl -XGET "${ELASTICSEARCH_ENDPOINT}logstash-*/fluentd/_search?q=tag:kubernetes.var.*" -H 'Content-Type: application/json' \ | python -c "import sys, json; print json.load(sys.stdin)['hits']['total']") if [ "$total_hits" -gt 0 ]; then echo "PASS: Successful hits on logstash-* index, provided by fluentd!" diff --git a/fluentd/templates/etc/_td-agent.conf.tpl b/fluentd/templates/etc/_td-agent.conf.tpl index 150a8163..1548c4ce 100644 --- a/fluentd/templates/etc/_td-agent.conf.tpl +++ b/fluentd/templates/etc/_td-agent.conf.tpl @@ -101,6 +101,7 @@ # Example: # {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text here\n","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"} + type tail path /var/lib/docker/containers/*/*-json.log @@ -111,25 +112,6 @@ read_from_head true -# Used to scan multiple lines for Python error stacktraces. Tag with stack.* - - type tail - path /var/lib/docker/containers/*/*-json.log - pos_file stack.pos - time_format %Y-%m-%dT%H:%M:%S - tag stack.* - multiline_flush_interval 1s - format multiline - format_firstline /ERROR/ - format1 /.*/ERROR/(?.*)$/ - key_name log - read_from_head true - - - - type kubernetes_metadata - - type kubernetes_metadata diff --git a/fluentd/values.yaml b/fluentd/values.yaml index 46f03b5a..889edd84 100644 --- a/fluentd/values.yaml +++ b/fluentd/values.yaml @@ -17,7 +17,7 @@ # Declare variables to be passed into your templates. images: - fluentd: gcr.io/google_containers/fluentd-elasticsearch:1.11 + fluentd: gcr.io/google_containers/fluentd-elasticsearch:1.23 helm_tests: docker.io/kolla/ubuntu-source-elasticsearch:3.0.3 pull_policy: IfNotPresent diff --git a/kibana/templates/bin/_kibana.sh.tpl b/kibana/templates/bin/_kibana.sh.tpl index 6c5e55c4..f5ac62f1 100644 --- a/kibana/templates/bin/_kibana.sh.tpl +++ b/kibana/templates/bin/_kibana.sh.tpl @@ -19,7 +19,7 @@ set -ex COMMAND="${@:-start}" function start () { - exec /opt/kibana/bin/kibana + exec kibana } function stop () { diff --git a/kibana/templates/deployment.yaml b/kibana/templates/deployment.yaml index 9fb53274..fc7d245b 100644 --- a/kibana/templates/deployment.yaml +++ b/kibana/templates/deployment.yaml @@ -47,9 +47,9 @@ spec: subPath: kibana.sh readOnly: true - name: etckibana - mountPath: /opt/kibana/config + mountPath: /etc/kibana - name: kibana-etc - mountPath: /opt/kibana/config/kibana.yml + mountPath: /etc/kibana/kibana.yml subPath: kibana.yml readOnly: true volumes: diff --git a/kibana/values.yaml b/kibana/values.yaml index bb0303dd..1ddcaaf0 100644 --- a/kibana/values.yaml +++ b/kibana/values.yaml @@ -17,7 +17,7 @@ labels: node_selector_value: images: - kibana: 'docker.io/kolla/ubuntu-source-kibana:3.0.3' + kibana: 'docker.io/kibana:5.4.2' pull_policy: IfNotPresent conf: