diff --git a/Makefile b/Makefile index 391c9aac68..91d8233a10 100644 --- a/Makefile +++ b/Makefile @@ -15,8 +15,8 @@ HELM = helm TASK = build -CHARTS = helm-toolkit bootstrap ceph mariadb etcd rabbitmq memcached -CHARTS += keystone glance cinder horizon neutron nova heat +CHARTS = helm-toolkit ceph mariadb etcd rabbitmq +CHARTS += memcached keystone glance cinder horizon neutron nova heat CHARTS += barbican mistral senlin magnum ingress all: $(CHARTS) diff --git a/bootstrap/.gitignore b/bootstrap/.gitignore deleted file mode 100644 index e1bd7e85af..0000000000 --- a/bootstrap/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -secrets/* -!secrets/.gitkeep -templates/_secrets.tpl diff --git a/bootstrap/.helmignore b/bootstrap/.helmignore deleted file mode 100644 index e8ef5ffab2..0000000000 --- a/bootstrap/.helmignore +++ /dev/null @@ -1,27 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*~ -# Various IDEs -.project -.idea/ -*.tmproj - -bin/ -etc/ -patches/ -*.py -Makefile diff --git a/bootstrap/Chart.yaml b/bootstrap/Chart.yaml deleted file mode 100644 index 2a9e7d09b3..0000000000 --- a/bootstrap/Chart.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -description: OpenStack-Helm namespace bootstrap -name: bootstrap -version: 0.1.0 diff --git a/bootstrap/requirements.yaml b/bootstrap/requirements.yaml deleted file mode 100644 index 53782e69b2..0000000000 --- a/bootstrap/requirements.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -dependencies: - - name: helm-toolkit - repository: http://localhost:8879/charts - version: 0.1.0 diff --git a/bootstrap/values.yaml b/bootstrap/values.yaml deleted file mode 100644 index 923e85d7da..0000000000 --- a/bootstrap/values.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default values for bootstrap. -# This is a YAML-formatted file. -# Declare name/value pairs to be passed into your templates. -# name: value diff --git a/ceph/templates/bin/_ceph-key.py.tpl b/ceph/templates/bin/_ceph-key.py.tpl new file mode 100644 index 0000000000..a0a279c7b2 --- /dev/null +++ b/ceph/templates/bin/_ceph-key.py.tpl @@ -0,0 +1,14 @@ +#!/bin/python +import os +import struct +import time +import base64 +key = os.urandom(16) +header = struct.pack( + ' +create_kube_key $(ceph_gen_key) ${CEPH_KEYRING_NAME} ${CEPH_KEYRING_TEMPLATE} ${KUBE_SECRET_NAME} diff --git a/ceph/templates/bin/_ceph-namespace-client-key.sh.tpl b/ceph/templates/bin/_ceph-namespace-client-key.sh.tpl new file mode 100644 index 0000000000..6817e53331 --- /dev/null +++ b/ceph/templates/bin/_ceph-namespace-client-key.sh.tpl @@ -0,0 +1,22 @@ +#!/bin/bash +set -ex + +ceph_activate_namespace() { + kube_namespace=$1 + { + cat < +create_kube_key ${CEPH_CLIENT_KEY} ${CEPH_KEYRING_NAME} ${CEPH_KEYRING_TEMPLATE} ${CEPH_KEYRING_ADMIN_NAME} + +function create_kube_storage_key () { + CEPH_KEYRING=$1 + KUBE_SECRET_NAME=$2 + + if ! kubectl get --namespace ${DEPLOYMENT_NAMESPACE} secrets ${KUBE_SECRET_NAME}; then + { + cat < +create_kube_storage_key ${CEPH_CLIENT_KEY} ${CEPH_STORAGECLASS_ADMIN_SECRET_NAME} diff --git a/ceph/templates/configmap-bin.yaml b/ceph/templates/configmap-bin.yaml new file mode 100644 index 0000000000..d8de23d0b8 --- /dev/null +++ b/ceph/templates/configmap-bin.yaml @@ -0,0 +1,31 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ceph-bin +data: +{{- if .Values.manifests_enabled.storage_secrets }} + ceph-key.py: |+ +{{ tuple "bin/_ceph-key.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + ceph-key.sh: |+ +{{ tuple "bin/_ceph-key.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + ceph-storage-key.sh: |+ +{{ tuple "bin/_ceph-storage-key.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} +{{- if .Values.manifests_enabled.client_secrets }} + ceph-namespace-client-key.sh: |+ +{{ tuple "bin/_ceph-namespace-client-key.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/bootstrap/templates/secrets.yaml b/ceph/templates/configmap-etc.yaml similarity index 53% rename from bootstrap/templates/secrets.yaml rename to ceph/templates/configmap-etc.yaml index 2bd43ac310..3a1203562a 100644 --- a/bootstrap/templates/secrets.yaml +++ b/ceph/templates/configmap-etc.yaml @@ -12,21 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +{{- if empty .Values.conf.ceph.config.global.mon_host -}} +{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_endpoint_lookup" }} +{{- $monHostDomain := default .Release.Namespace .Values.ceph.namespace }} +{{- $monHostURI := cat $monHost "." $monHostDomain | nospace -}} +{{- $monHostURI | set .Values.conf.ceph.config.global "mon_host" | quote | trunc 0 -}} +{{- end -}} + --- apiVersion: v1 -kind: Secret +kind: ConfigMap metadata: - name: "pvc-ceph-conf-combined-storageclass" -type: kubernetes.io/rbd + name: ceph-etc data: - key: | -{{ include "secrets/ceph-client-key" . | b64enc | indent 4 }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: "pvc-ceph-client-key" -type: kubernetes.io/rbd -data: - key: | -{{ include "secrets/ceph-client-key" . | b64enc | indent 4 }} + ceph.conf: |+ +{{ tuple "etc/_ceph.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/ceph/templates/configmap-templates.yaml b/ceph/templates/configmap-templates.yaml new file mode 100644 index 0000000000..ce18d56fcb --- /dev/null +++ b/ceph/templates/configmap-templates.yaml @@ -0,0 +1,31 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +{{- if .Values.manifests_enabled.storage_secrets }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ceph-templates +data: + admin.keyring: |+ +{{ tuple "templates/_admin.keyring.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + bootstrap.keyring.mds: |+ +{{ tuple "templates/_bootstrap.keyring.mds.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + bootstrap.keyring.osd: |+ +{{ tuple "templates/_bootstrap.keyring.osd.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + bootstrap.keyring.rgw: |+ +{{ tuple "templates/_bootstrap.keyring.rgw.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + mon.keyring: |+ +{{ tuple "templates/_mon.keyring.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/ceph/templates/daemonset-osd.yaml b/ceph/templates/daemonset-osd.yaml index 0092830c17..f0c5ffa54a 100644 --- a/ceph/templates/daemonset-osd.yaml +++ b/ceph/templates/daemonset-osd.yaml @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +{{- if .Values.manifests_enabled.deployment }} +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies.osd }} --- kind: DaemonSet apiVersion: extensions/v1beta1 @@ -29,47 +32,21 @@ spec: spec: nodeSelector: {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} - volumes: - - name: devices - hostPath: - path: /dev - - name: ceph - hostPath: - path: {{ .Values.storage.var_directory }} - - name: ceph-conf - secret: - secretName: ceph-conf-combined - - name: ceph-bootstrap-osd-keyring - secret: - secretName: ceph-bootstrap-osd-keyring - - name: ceph-bootstrap-mds-keyring - secret: - secretName: ceph-bootstrap-mds-keyring - - name: ceph-bootstrap-rgw-keyring - secret: - secretName: ceph-bootstrap-rgw-keyring - - name: osd-directory - hostPath: - path: {{ .Values.storage.osd_directory }} + initContainers: +{{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} containers: - name: osd-pod image: {{ .Values.images.daemon }} imagePullPolicy: {{ .Values.images.pull_policy }} - volumeMounts: - - name: devices - mountPath: /dev - - name: ceph - mountPath: /var/lib/ceph - - name: ceph-conf - mountPath: /etc/ceph - - name: ceph-bootstrap-osd-keyring - mountPath: /var/lib/ceph/bootstrap-osd - - name: ceph-bootstrap-mds-keyring - mountPath: /var/lib/ceph/bootstrap-mds - - name: ceph-bootstrap-rgw-keyring - mountPath: /var/lib/ceph/bootstrap-rgw - - name: osd-directory - mountPath: /var/lib/ceph/osd + {{- if .Values.resources.enabled }} + resources: + requests: + memory: {{ .Values.resources.osd.requests.memory | quote }} + cpu: {{ .Values.resources.osd.requests.cpu | quote }} + limits: + memory: {{ .Values.resources.osd.limits.memory | quote }} + cpu: {{ .Values.resources.osd.limits.cpu | quote }} + {{- end }} securityContext: privileged: true env: @@ -81,6 +58,10 @@ spec: value: ceph - name: CEPH_GET_ADMIN_KEY value: "1" + command: + - /entrypoint.sh + ports: + - containerPort: 6800 livenessProbe: tcpSocket: port: 6800 @@ -90,10 +71,65 @@ spec: tcpSocket: port: 6800 timeoutSeconds: 5 - resources: - requests: - memory: {{ .Values.resources.osd.requests.memory | quote }} - cpu: {{ .Values.resources.osd.requests.cpu | quote }} - limits: - memory: {{ .Values.resources.osd.limits.memory | quote }} - cpu: {{ .Values.resources.osd.limits.cpu | quote }} + volumeMounts: + - name: devices + mountPath: /dev + readOnly: false + - name: ceph + mountPath: /var/lib/ceph + readOnly: false + - name: ceph-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + - name: ceph-client-admin-keyring + mountPath: /etc/ceph/ceph.client.admin.keyring + subPath: ceph.client.admin.keyring + readOnly: false + - name: ceph-mon-keyring + mountPath: /etc/ceph/ceph.mon.keyring + subPath: ceph.mon.keyring + readOnly: false + - name: ceph-bootstrap-osd-keyring + mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring + subPath: ceph.keyring + readOnly: false + - name: ceph-bootstrap-mds-keyring + mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring + subPath: ceph.keyring + readOnly: false + - name: ceph-bootstrap-rgw-keyring + mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring + subPath: ceph.keyring + readOnly: false + - name: osd-directory + mountPath: /var/lib/ceph/osd + volumes: + - name: devices + hostPath: + path: /dev + - name: ceph + hostPath: + path: {{ .Values.ceph.storage.var_directory }} + - name: ceph-etc + configMap: + name: ceph-etc + - name: ceph-client-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin }} + - name: ceph-mon-keyring + secret: + secretName: {{ .Values.secrets.keyrings.mon }} + - name: ceph-bootstrap-osd-keyring + secret: + secretName: {{ .Values.secrets.keyrings.osd }} + - name: ceph-bootstrap-mds-keyring + secret: + secretName: {{ .Values.secrets.keyrings.mds }} + - name: ceph-bootstrap-rgw-keyring + secret: + secretName: {{ .Values.secrets.keyrings.rgw }} + - name: osd-directory + hostPath: + path: {{ .Values.ceph.storage.osd_directory }} +{{- end }} diff --git a/ceph/templates/deployment-mds.yaml b/ceph/templates/deployment-mds.yaml index 925f449dde..964677ec40 100644 --- a/ceph/templates/deployment-mds.yaml +++ b/ceph/templates/deployment-mds.yaml @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. ---- +{{- if .Values.manifests_enabled.deployment }} +{{- if .Values.ceph.enabled.mds }} +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies.mds }} kind: Deployment apiVersion: apps/v1beta1 metadata: @@ -31,24 +34,22 @@ spec: spec: nodeSelector: {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + initContainers: +{{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} serviceAccount: default - volumes: - - name: ceph-conf - secret: - secretName: ceph-conf-combined - - name: ceph-bootstrap-osd-keyring - secret: - secretName: ceph-bootstrap-osd-keyring - - name: ceph-bootstrap-mds-keyring - secret: - secretName: ceph-bootstrap-mds-keyring - - name: ceph-bootstrap-rgw-keyring - secret: - secretName: ceph-bootstrap-rgw-keyring containers: - name: ceph-mds image: {{ .Values.images.daemon }} imagePullPolicy: {{ .Values.images.pull_policy }} + {{- if .Values.resources.enabled }} + resources: + requests: + memory: {{ .Values.resources.mds.requests.memory | quote }} + cpu: {{ .Values.resources.mds.requests.cpu | quote }} + limits: + memory: {{ .Values.resources.mds.limits.memory | quote }} + cpu: {{ .Values.resources.mds.limits.cpu | quote }} + {{- end }} ports: - containerPort: 6800 env: @@ -60,15 +61,33 @@ spec: value: k8s - name: CLUSTER value: ceph + command: + - /entrypoint.sh volumeMounts: - - name: ceph-conf - mountPath: /etc/ceph + - name: ceph-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + - name: ceph-client-admin-keyring + mountPath: /etc/ceph/ceph.client.admin.keyring + subPath: ceph.client.admin.keyring + readOnly: true + - name: ceph-mon-keyring + mountPath: /etc/ceph/ceph.mon.keyring + subPath: ceph.mon.keyring + readOnly: true - name: ceph-bootstrap-osd-keyring - mountPath: /var/lib/ceph/bootstrap-osd + mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring + subPath: ceph.keyring + readOnly: false - name: ceph-bootstrap-mds-keyring - mountPath: /var/lib/ceph/bootstrap-mds + mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring + subPath: ceph.keyring + readOnly: false - name: ceph-bootstrap-rgw-keyring - mountPath: /var/lib/ceph/bootstrap-rgw + mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring + subPath: ceph.keyring + readOnly: false livenessProbe: tcpSocket: port: 6800 @@ -78,10 +97,24 @@ spec: tcpSocket: port: 6800 timeoutSeconds: 5 - resources: - requests: - memory: {{ .Values.resources.mds.requests.memory | quote }} - cpu: {{ .Values.resources.mds.requests.cpu | quote }} - limits: - memory: {{ .Values.resources.mds.limits.memory | quote }} - cpu: {{ .Values.resources.mds.limits.cpu | quote }} + volumes: + - name: ceph-etc + configMap: + name: ceph-etc + - name: ceph-client-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin }} + - name: ceph-mon-keyring + secret: + secretName: {{ .Values.secrets.keyrings.mon }} + - name: ceph-bootstrap-osd-keyring + secret: + secretName: {{ .Values.secrets.keyrings.osd }} + - name: ceph-bootstrap-mds-keyring + secret: + secretName: {{ .Values.secrets.keyrings.mds }} + - name: ceph-bootstrap-rgw-keyring + secret: + secretName: {{ .Values.secrets.keyrings.rgw }} +{{- end }} +{{- end }} diff --git a/ceph/templates/deployment-moncheck.yaml b/ceph/templates/deployment-moncheck.yaml index a62c039fdc..54228cf3ad 100644 --- a/ceph/templates/deployment-moncheck.yaml +++ b/ceph/templates/deployment-moncheck.yaml @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +{{- if .Values.manifests_enabled.deployment }} +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies.moncheck }} --- kind: Deployment apiVersion: apps/v1beta1 @@ -31,24 +34,22 @@ spec: spec: nodeSelector: {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + initContainers: +{{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} serviceAccount: default - volumes: - - name: ceph-conf - secret: - secretName: ceph-conf-combined - - name: ceph-bootstrap-osd-keyring - secret: - secretName: ceph-bootstrap-osd-keyring - - name: ceph-bootstrap-mds-keyring - secret: - secretName: ceph-bootstrap-mds-keyring - - name: ceph-bootstrap-rgw-keyring - secret: - secretName: ceph-bootstrap-rgw-keyring containers: - name: ceph-mon image: {{ .Values.images.daemon }} imagePullPolicy: {{ .Values.images.pull_policy }} + {{- if .Values.resources.enabled }} + resources: + requests: + memory: {{ .Values.resources.mon_check.requests.memory | quote }} + cpu: {{ .Values.resources.mon_check.requests.cpu | quote }} + limits: + memory: {{ .Values.resources.mon_check.limits.memory | quote }} + cpu: {{ .Values.resources.mon_check.limits.cpu | quote }} + {{- end }} ports: - containerPort: 6789 env: @@ -60,19 +61,50 @@ spec: value: "1" - name: CLUSTER value: ceph + command: + - /entrypoint.sh volumeMounts: - - name: ceph-conf - mountPath: /etc/ceph + - name: ceph-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + - name: ceph-client-admin-keyring + mountPath: /etc/ceph/ceph.client.admin.keyring + subPath: ceph.client.admin.keyring + readOnly: true + - name: ceph-mon-keyring + mountPath: /etc/ceph/ceph.mon.keyring + subPath: ceph.mon.keyring + readOnly: true - name: ceph-bootstrap-osd-keyring - mountPath: /var/lib/ceph/bootstrap-osd + mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring + subPath: ceph.keyring + readOnly: false - name: ceph-bootstrap-mds-keyring - mountPath: /var/lib/ceph/bootstrap-mds + mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring + subPath: ceph.keyring + readOnly: false - name: ceph-bootstrap-rgw-keyring - mountPath: /var/lib/ceph/bootstrap-rgw - resources: - requests: - memory: {{ .Values.resources.mon_check.requests.memory | quote }} - cpu: {{ .Values.resources.mon_check.requests.cpu | quote }} - limits: - memory: {{ .Values.resources.mon_check.limits.memory | quote }} - cpu: {{ .Values.resources.mon_check.limits.cpu | quote }} + mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring + subPath: ceph.keyring + readOnly: false + volumes: + - name: ceph-etc + configMap: + name: ceph-etc + - name: ceph-client-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin }} + - name: ceph-mon-keyring + secret: + secretName: {{ .Values.secrets.keyrings.mon }} + - name: ceph-bootstrap-osd-keyring + secret: + secretName: {{ .Values.secrets.keyrings.osd }} + - name: ceph-bootstrap-mds-keyring + secret: + secretName: {{ .Values.secrets.keyrings.mds }} + - name: ceph-bootstrap-rgw-keyring + secret: + secretName: {{ .Values.secrets.keyrings.rgw }} +{{- end }} diff --git a/ceph/templates/deployment-rgw.yaml b/ceph/templates/deployment-rgw.yaml index 6521046c85..c754b1ae40 100644 --- a/ceph/templates/deployment-rgw.yaml +++ b/ceph/templates/deployment-rgw.yaml @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -{{- if .Values.rgw.enabled }} +{{- if .Values.manifests_enabled.deployment }} +{{- if .Values.ceph.enabled.rgw }} +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies.rgw }} --- kind: Deployment apiVersion: apps/v1beta1 @@ -32,24 +35,22 @@ spec: spec: nodeSelector: {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + initContainers: +{{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} serviceAccount: default - volumes: - - name: ceph-conf - secret: - secretName: ceph-conf-combined - - name: ceph-bootstrap-osd-keyring - secret: - secretName: ceph-bootstrap-osd-keyring - - name: ceph-bootstrap-mds-keyring - secret: - secretName: ceph-bootstrap-mds-keyring - - name: ceph-bootstrap-rgw-keyring - secret: - secretName: ceph-bootstrap-rgw-keyring containers: - name: ceph-rgw image: {{ .Values.images.daemon }} imagePullPolicy: {{ .Values.images.pull_policy }} + {{- if .Values.resources.enabled }} + resources: + requests: + memory: {{ .Values.resources.rgw.requests.memory | quote }} + cpu: {{ .Values.resources.rgw.requests.cpu | quote }} + limits: + memory: {{ .Values.resources.rgw.limits.memory | quote }} + cpu: {{ .Values.resources.rgw.limits.cpu | quote }} + {{- end }} ports: - containerPort: {{ .Values.network.port.rgw_target }} env: @@ -61,15 +62,33 @@ spec: value: k8s - name: CLUSTER value: ceph + command: + - /entrypoint.sh volumeMounts: - - name: ceph-conf - mountPath: /etc/ceph + - name: ceph-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + - name: ceph-client-admin-keyring + mountPath: /etc/ceph/ceph.client.admin.keyring + subPath: ceph.client.admin.keyring + readOnly: true + - name: ceph-mon-keyring + mountPath: /etc/ceph/ceph.mon.keyring + subPath: ceph.mon.keyring + readOnly: true - name: ceph-bootstrap-osd-keyring - mountPath: /var/lib/ceph/bootstrap-osd + mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring + subPath: ceph.keyring + readOnly: false - name: ceph-bootstrap-mds-keyring - mountPath: /var/lib/ceph/bootstrap-mds + mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring + subPath: ceph.keyring + readOnly: false - name: ceph-bootstrap-rgw-keyring - mountPath: /var/lib/ceph/bootstrap-rgw + mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring + subPath: ceph.keyring + readOnly: false livenessProbe: httpGet: path: / @@ -81,11 +100,24 @@ spec: path: / port: {{ .Values.network.port.rgw_target }} timeoutSeconds: 5 - resources: - requests: - memory: {{ .Values.resources.rgw.requests.memory | quote }} - cpu: {{ .Values.resources.rgw.requests.cpu | quote }} - limits: - memory: {{ .Values.resources.rgw.limits.memory | quote }} - cpu: {{ .Values.resources.rgw.limits.cpu | quote }} + volumes: + - name: ceph-etc + configMap: + name: ceph-etc + - name: ceph-client-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin }} + - name: ceph-mon-keyring + secret: + secretName: {{ .Values.secrets.keyrings.mon }} + - name: ceph-bootstrap-osd-keyring + secret: + secretName: {{ .Values.secrets.keyrings.osd }} + - name: ceph-bootstrap-mds-keyring + secret: + secretName: {{ .Values.secrets.keyrings.mds }} + - name: ceph-bootstrap-rgw-keyring + secret: + secretName: {{ .Values.secrets.keyrings.rgw }} +{{- end }} {{- end }} diff --git a/ceph/templates/etc/_ceph.conf.tpl b/ceph/templates/etc/_ceph.conf.tpl new file mode 100644 index 0000000000..7a9dd76f28 --- /dev/null +++ b/ceph/templates/etc/_ceph.conf.tpl @@ -0,0 +1,77 @@ +[global] +fsid = {{ uuidv4 | default .Values.conf.ceph.config.global.uuid | quote }} +cephx = {{ .Values.conf.ceph.config.global.cephx | default "true" | quote }} +cephx_require_signatures = {{ .Values.conf.ceph.config.global.cephx_require_signatures | default "false" | quote }} +cephx_cluster_require_signatures = {{ .Values.conf.ceph.config.global.cephx_cluster_require_signatures | default "true" | quote }} +cephx_service_require_signatures = {{ .Values.conf.ceph.config.global.cephx_service_require_signatures | default "false" | quote }} + +# auth +max_open_files = {{ .Values.conf.ceph.config.global.max_open_files | default "131072" | quote }} + +osd_pool_default_pg_num = {{ .Values.conf.ceph.config.global.osd_pool_default_pg_num | default "128" | quote }} +osd_pool_default_pgp_num = {{ .Values.conf.ceph.config.global.osd_pool_default_pgp_num | default "128" | quote }} +osd_pool_default_size = {{ .Values.conf.ceph.config.global.osd_pool_default_size | default "3" | quote }} +osd_pool_default_min_size = {{ .Values.conf.ceph.config.global.osd_pool_default_min_size | default "1" | quote }} + +mon_osd_full_ratio = {{ .Values.conf.ceph.config.global.mon_osd_full_ratio | default ".95" | quote }} +mon_osd_nearfull_ratio = {{ .Values.conf.ceph.config.global.mon_osd_nearfull_ratio | default ".85" | quote }} +mon_host = {{ .Values.conf.ceph.config.global.mon_host | quote }} + +rgw_thread_pool_size = {{ .Values.conf.ceph.config.global.rgw_thread_pool_size | default "1024" | quote }} +rgw_num_rados_handles = {{ .Values.conf.ceph.config.global.rgw_num_rados_handles | default "100" | quote }} + +[mon] +mon_osd_down_out_interval = {{ .Values.conf.ceph.config.mon.mon_osd_down_out_interval | default "600" | quote }} +mon_osd_min_down_reporters = {{ .Values.conf.ceph.config.mon.mon_osd_min_down_reporters | default "4" | quote }} +mon_clock_drift_allowed = {{ .Values.conf.ceph.config.mon.mon_clock_drift_allowed | default "0.15" | quote }} +mon_clock_drift_warn_backoff = {{ .Values.conf.ceph.config.mon.mon_clock_drift_warn_backoff | default "30" | quote }} +mon_osd_report_timeout = {{ .Values.conf.ceph.config.mon.mon_osd_report_timeout | default "300" | quote }} + +[osd] +# network +cluster_network = {{ .Values.network.cluster | default "192.168.0.0/16" | quote }} +public_network = {{ .Values.network.public | default "192.168.0.0/16" | quote }} +osd_mon_heartbeat_interval = {{ .Values.conf.ceph.config.osd.osd_mon_heartbeat_interval | default "30" | quote }} + +# ports +ms_bind_port_min = {{ .Values.conf.ceph.config.osd.ms_bind_port_min | default "6800" | quote }} +ms_bind_port_max = {{ .Values.conf.ceph.config.osd.ms_bind_port_max | default "7100" | quote }} + +# journal +journal_size = {{ .Values.conf.ceph.config.osd.journal_size | default "100" | quote }} + +# filesystem +osd_mkfs_type = {{ .Values.conf.ceph.config.osd.osd_mkfs_type | default "xfs" | quote }} +osd_mkfs_options_xfs = {{ .Values.conf.ceph.config.osd.osd_mkfs_options_xfs | default "-f -i size=2048" | quote }} +osd_max_object_name_len = {{ .Values.conf.ceph.config.osd.osd_max_object_name_len | default "256" | quote }} + +# crush +osd_pool_default_crush_rule = {{ .Values.conf.ceph.config.osd.osd_pool_default_crush_rule | default "0" | quote }} +osd_crush_update_on_start = {{ .Values.conf.ceph.config.osd.osd_crush_update_on_start | default "true" | quote }} + +# backend +osd_objectstore = {{ .Values.conf.ceph.config.osd.osd_objectstore | default "filestore" | quote }} + +# performance tuning +filestore_merge_threshold = {{ .Values.conf.ceph.config.osd.filestore_merge_threshold | default "40" | quote }} +filestore_split_multiple = {{ .Values.conf.ceph.config.osd.filestore_split_multiple | default "8" | quote }} +osd_op_threads = {{ .Values.conf.ceph.config.osd.osd_op_threads | default "8" | quote }} +filestore_op_threads = {{ .Values.conf.ceph.config.osd.filestore_op_threads | default "8" | quote }} +filestore_max_sync_interval = {{ .Values.conf.ceph.config.osd.filestore_max_sync_interval | default "5" | quote }} +osd_max_scrubs = {{ .Values.conf.ceph.config.osd.osd_max_scrubs | default "1" | quote }} + +# recovery tuning +osd_recovery_max_active = {{ .Values.conf.ceph.config.osd.osd_recovery_max_active | default "5" | quote }} +osd_max_backfills = {{ .Values.conf.ceph.config.osd.osd_max_backfills | default "2" | quote }} +osd_recovery_op_priority = {{ .Values.conf.ceph.config.osd.osd_recovery_op_priority | default "2" | quote }} +osd_client_op_priority = {{ .Values.conf.ceph.config.osd.osd_client_op_priority | default "63" | quote }} +osd_recovery_max_chunk = {{ .Values.conf.ceph.config.osd.osd_client_op_priority | default "osd_recovery_max_chunk" | quote }} +osd_recovery_threads = {{ .Values.conf.ceph.config.osd.osd_recovery_threads | default "1" | quote }} + +[client] +rbd_cache_enabled = {{ .Values.conf.ceph.config.client.rbd_cache_enabled | default "true" | quote }} +rbd_cache_writethrough_until_flush = {{ .Values.conf.ceph.config.client.rbd_cache_writethrough_until_flush | default "true" | quote }} +rbd_default_features = {{ .Values.conf.ceph.config.client.rbd_default_features | default "1" | quote }} + +[mds] +mds_cache_size = {{ .Values.conf.ceph.config.client.mds_mds_cache_size | default "100000" | quote }} diff --git a/ceph/templates/job-keyring.yaml b/ceph/templates/job-keyring.yaml new file mode 100644 index 0000000000..677167dfaf --- /dev/null +++ b/ceph/templates/job-keyring.yaml @@ -0,0 +1,85 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.manifests_enabled.storage_secrets }} +{{- $envAll := . }} +{{- range $key1, $cephBootstrapKey := tuple "mds" "osd" "rgw" "mon" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ceph-{{ $cephBootstrapKey }}-keyring-generator +spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: ceph-secret-generator + image: {{ $envAll.Values.images.ceph_config_helper }} + imagePullPolicy: {{ $envAll.Values.images.pull_policy }} + {{- if $envAll.Values.resources.enabled }} + resources: + requests: + memory: {{ .Values.resources.jobs.secret_provisioning.requests.memory | quote }} + cpu: {{ .Values.resources.jobs.secret_provisioning.requests.cpu | quote }} + limits: + memory: {{ .Values.resources.jobs.secret_provisioning.limits.memory | quote }} + cpu: {{ .Values.resources.jobs.secret_provisioning.limits.cpu | quote }} + {{- end }} + env: + - name: DEPLOYMENT_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CEPH_GEN_DIR + value: /opt/ceph + - name: CEPH_TEMPLATES_DIR + value: /opt/ceph/templates + {{- if eq $cephBootstrapKey "mon"}} + - name: CEPH_KEYRING_NAME + value: ceph.mon.keyring + - name: CEPH_KEYRING_TEMPLATE + value: mon.keyring + {{- else }} + - name: CEPH_KEYRING_NAME + value: ceph.keyring + - name: CEPH_KEYRING_TEMPLATE + value: bootstrap.keyring.{{ $cephBootstrapKey }} + {{- end }} + - name: KUBE_SECRET_NAME + value: {{ index $envAll.Values.secrets.keyrings $cephBootstrapKey }} + command: + - /opt/ceph/ceph-key.sh + volumeMounts: + - name: ceph-bin + mountPath: /opt/ceph/ceph-key.sh + subPath: ceph-key.sh + readOnly: true + - name: ceph-bin + mountPath: /opt/ceph/ceph-key.py + subPath: ceph-key.py + readOnly: true + - name: ceph-templates + mountPath: /opt/ceph/templates + readOnly: true + volumes: + - name: ceph-bin + configMap: + name: ceph-bin + defaultMode: 0555 + - name: ceph-templates + configMap: + name: ceph-templates +{{ end }} +{{ end }} diff --git a/ceph/templates/job-namespace-client-key.yaml b/ceph/templates/job-namespace-client-key.yaml new file mode 100644 index 0000000000..aa9a2f09ed --- /dev/null +++ b/ceph/templates/job-namespace-client-key.yaml @@ -0,0 +1,61 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.manifests_enabled.client_secrets }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ceph-namespace-client-key-generator +spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: ceph-storage-keys-generator + image: {{ .Values.images.ceph_config_helper }} + imagePullPolicy: {{ .Values.images.pull_policy }} + {{- if .Values.resources.enabled }} + resources: + requests: + memory: {{ .Values.resources.jobs.secret_provisioning.requests.memory | quote }} + cpu: {{ .Values.resources.jobs.secret_provisioning.requests.cpu | quote }} + limits: + memory: {{ .Values.resources.jobs.secret_provisioning.limits.memory | quote }} + cpu: {{ .Values.resources.jobs.secret_provisioning.limits.cpu | quote }} + {{- end }} + env: + - name: DEPLOYMENT_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: PVC_CEPH_STORAGECLASS_USER_SECRET_NAME + value: {{ .Values.storageclass.user_secret_name }} + - name: PVC_CEPH_STORAGECLASS_ADMIN_SECRET_NAME + value: {{ .Values.storageclass.admin_secret_name }} + - name: PVC_CEPH_STORAGECLASS_DEPLOYED_NAMESPACE + value: {{ .Values.storageclass.admin_secret_namespace }} + command: + - /opt/ceph/ceph-namespace-client-key.sh + volumeMounts: + - name: ceph-bin + mountPath: /opt/ceph/ceph-namespace-client-key.sh + subPath: ceph-namespace-client-key.sh + readOnly: true + volumes: + - name: ceph-bin + configMap: + name: ceph-bin + defaultMode: 0555 +{{- end }} diff --git a/ceph/templates/job-storage-admin-keys.yaml b/ceph/templates/job-storage-admin-keys.yaml new file mode 100644 index 0000000000..571057b8d7 --- /dev/null +++ b/ceph/templates/job-storage-admin-keys.yaml @@ -0,0 +1,77 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.manifests_enabled.storage_secrets }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ceph-storage-keys-generator +spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: ceph-storage-keys-generator + image: {{ .Values.images.ceph_config_helper }} + imagePullPolicy: {{ .Values.images.pull_policy }} + {{- if .Values.resources.enabled }} + resources: + requests: + memory: {{ .Values.resources.jobs.secret_provisioning.requests.memory | quote }} + cpu: {{ .Values.resources.jobs.secret_provisioning.requests.cpu | quote }} + limits: + memory: {{ .Values.resources.jobs.secret_provisioning.limits.memory | quote }} + cpu: {{ .Values.resources.jobs.secret_provisioning.limits.cpu | quote }} + {{- end }} + env: + - name: DEPLOYMENT_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CEPH_GEN_DIR + value: /opt/ceph + - name: CEPH_TEMPLATES_DIR + value: /opt/ceph/templates + - name: CEPH_KEYRING_NAME + value: ceph.client.admin.keyring + - name: CEPH_KEYRING_TEMPLATE + value: admin.keyring + - name: CEPH_KEYRING_ADMIN_NAME + value: {{ .Values.secrets.keyrings.admin }} + - name: CEPH_STORAGECLASS_ADMIN_SECRET_NAME + value: {{ .Values.storageclass.admin_secret_name }} + command: + - /opt/ceph/ceph-storage-key.sh + volumeMounts: + - name: ceph-bin + mountPath: /opt/ceph/ceph-storage-key.sh + subPath: ceph-storage-key.sh + readOnly: true + - name: ceph-bin + mountPath: /opt/ceph/ceph-key.py + subPath: ceph-key.py + readOnly: true + - name: ceph-templates + mountPath: /opt/ceph/templates + readOnly: true + volumes: + - name: ceph-bin + configMap: + name: ceph-bin + defaultMode: 0555 + - name: ceph-templates + configMap: + name: ceph-templates +{{- end }} diff --git a/ceph/templates/pdb-mon.yaml b/ceph/templates/pdb-mon.yaml index 9852700e95..e357496855 100644 --- a/ceph/templates/pdb-mon.yaml +++ b/ceph/templates/pdb-mon.yaml @@ -1,3 +1,4 @@ +{{- if .Values.manifests_enabled.deployment }} apiVersion: policy/v1beta1 kind: PodDisruptionBudget metadata: @@ -7,4 +8,5 @@ spec: selector: matchLabels: app: ceph - daemon: mon \ No newline at end of file + daemon: mon +{{- end }} diff --git a/ceph/templates/secrets.yaml b/ceph/templates/secrets.yaml deleted file mode 100644 index 8c11ec2f7d..0000000000 --- a/ceph/templates/secrets.yaml +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{{- if .Values.secrets.use_common_secrets -}} ---- -apiVersion: v1 -kind: Secret -metadata: - name: "ceph-conf-combined" -type: Opaque -data: - ceph.conf: | -{{ include "secrets/ceph.conf" . | b64enc | indent 4 }} - ceph.client.admin.keyring: | -{{ include "secrets/ceph.client.admin.keyring" . | b64enc | indent 4 }} - ceph.mon.keyring: | -{{ include "secrets/ceph.mon.keyring" . | b64enc | indent 4 }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: "ceph-bootstrap-rgw-keyring" -type: Opaque -data: - ceph.keyring: | -{{ include "secrets/ceph.rgw.keyring" . | b64enc | indent 4 }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: "ceph-bootstrap-mds-keyring" -type: Opaque -data: - ceph.keyring: | -{{ include "secrets/ceph.mds.keyring" . | b64enc | indent 4 }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: "ceph-bootstrap-osd-keyring" -type: Opaque -data: - ceph.keyring: | -{{ include "secrets/ceph.osd.keyring" . | b64enc | indent 4 }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: "ceph-client-key" -type: Opaque -data: - ceph-client-key: | -{{ include "secrets/ceph-client-key" . | b64enc | indent 4 }} -{{- end -}} diff --git a/ceph/templates/service-rgw.yaml b/ceph/templates/service-rgw.yaml index 4ae4055c96..c93e82de3d 100644 --- a/ceph/templates/service-rgw.yaml +++ b/ceph/templates/service-rgw.yaml @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -{{- if .Values.rgw.enabled }} +{{- if .Values.manifests_enabled.deployment }} +{{- if .Values.ceph.enabled.rgw }} --- apiVersion: v1 kind: Service @@ -30,3 +31,4 @@ spec: app: ceph daemon: rgw {{- end }} +{{- end }} diff --git a/ceph/templates/service.yaml b/ceph/templates/service.yaml index ef6d98e0a9..e0202e557e 100644 --- a/ceph/templates/service.yaml +++ b/ceph/templates/service.yaml @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +{{- if .Values.manifests_enabled.deployment }} --- kind: Service apiVersion: v1 metadata: - name: ceph-mon + name: {{ .Values.endpoints.ceph_mon.hosts.default }} labels: app: ceph daemon: mon @@ -37,3 +38,4 @@ spec: app: ceph daemon: mon clusterIP: None +{{- end }} diff --git a/ceph/templates/statefulset-mon.yaml b/ceph/templates/statefulset-mon.yaml index 47a8ebc279..0f1ca1e169 100644 --- a/ceph/templates/statefulset-mon.yaml +++ b/ceph/templates/statefulset-mon.yaml @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +{{- if .Values.manifests_enabled.deployment }} +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies.mon }} --- apiVersion: apps/v1beta1 kind: StatefulSet @@ -21,7 +24,7 @@ metadata: daemon: mon name: ceph-mon spec: - serviceName: {{ .Values.service.mon.name | quote }} + serviceName: {{ tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_endpoint_lookup" }} replicas: {{ .Values.replicas.mon }} template: metadata: @@ -46,33 +49,22 @@ spec: weight: 10 nodeSelector: {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + initContainers: +{{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} serviceAccount: default - volumes: - - name: ceph-conf - secret: - secretName: ceph-conf-combined - - name: ceph-bootstrap-osd-keyring - secret: - secretName: ceph-bootstrap-osd-keyring - - name: ceph-bootstrap-mds-keyring - secret: - secretName: ceph-bootstrap-mds-keyring - - name: ceph-bootstrap-rgw-keyring - secret: - secretName: ceph-bootstrap-rgw-keyring - - name: ceph-monfs - hostPath: - path: {{ .Values.storage.mon_directory }} containers: - name: ceph-mon image: {{ .Values.images.daemon }} imagePullPolicy: {{ .Values.images.pull_policy }} - lifecycle: - preStop: - exec: - # remove the mon on Pod stop. - command: - - "/remove-mon.sh" + {{- if .Values.resources.enabled }} + resources: + requests: + memory: {{ .Values.resources.mon.requests.memory | quote }} + cpu: {{ .Values.resources.mon.requests.cpu | quote }} + limits: + memory: {{ .Values.resources.mon.limits.memory | quote }} + cpu: {{ .Values.resources.mon.limits.cpu | quote }} + {{- end }} ports: - containerPort: 6789 env: @@ -90,17 +82,39 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + command: + - /entrypoint.sh + lifecycle: + preStop: + exec: + # remove the mon on Pod stop. + command: + - "/remove-mon.sh" volumeMounts: - - name: ceph-conf - mountPath: /etc/ceph + - name: ceph-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + - name: ceph-client-admin-keyring + mountPath: /etc/ceph/ceph.client.admin.keyring + subPath: ceph.client.admin.keyring + readOnly: true + - name: ceph-mon-keyring + mountPath: /etc/ceph/ceph.mon.keyring + subPath: ceph.mon.keyring + readOnly: false - name: ceph-bootstrap-osd-keyring - mountPath: /var/lib/ceph/bootstrap-osd + mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring + subPath: ceph.keyring + readOnly: false - name: ceph-bootstrap-mds-keyring - mountPath: /var/lib/ceph/bootstrap-mds + mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring + subPath: ceph.keyring + readOnly: false - name: ceph-bootstrap-rgw-keyring - mountPath: /var/lib/ceph/bootstrap-rgw - - name: ceph-monfs - mountPath: /var/lib/ceph/mon + mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring + subPath: ceph.keyring + readOnly: false livenessProbe: tcpSocket: port: 6789 @@ -110,10 +124,23 @@ spec: tcpSocket: port: 6789 timeoutSeconds: 5 - resources: - requests: - memory: {{ .Values.resources.mon.requests.memory | quote }} - cpu: {{ .Values.resources.mon.requests.cpu | quote }} - limits: - memory: {{ .Values.resources.mon.limits.memory | quote }} - cpu: {{ .Values.resources.mon.limits.cpu | quote }} + volumes: + - name: ceph-etc + configMap: + name: ceph-etc + - name: ceph-client-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin }} + - name: ceph-mon-keyring + secret: + secretName: {{ .Values.secrets.keyrings.mon }} + - name: ceph-bootstrap-osd-keyring + secret: + secretName: {{ .Values.secrets.keyrings.osd }} + - name: ceph-bootstrap-mds-keyring + secret: + secretName: {{ .Values.secrets.keyrings.mds }} + - name: ceph-bootstrap-rgw-keyring + secret: + secretName: {{ .Values.secrets.keyrings.rgw }} +{{- end }} diff --git a/ceph/templates/storage.yaml b/ceph/templates/storage.yaml index b104269e56..e75fd35df6 100644 --- a/ceph/templates/storage.yaml +++ b/ceph/templates/storage.yaml @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# note that these secrets are handled by the common chart, not the ceph -# chart, as we likely want them "everywhere" +{{- if .Values.manifests_enabled.deployment }} --- apiVersion: storage.k8s.io/v1 kind: StorageClass @@ -21,12 +20,11 @@ metadata: name: {{ .Values.storageclass.name }} provisioner: kubernetes.io/rbd parameters: - monitors: {{ .Values.storageclass.monitors | default "ceph-mon.ceph:6789" }} + monitors: {{ tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.hostname_endpoint_uri_lookup" }} adminId: {{ .Values.storageclass.admin_id }} adminSecretName: {{ .Values.storageclass.admin_secret_name }} - # forcing namespace due to issue with default pipeline of "{{ .Release.Namespace }}" }} - # during helm lint - adminSecretNamespace: {{ .Values.storageclass.admin_secret_namespace | default "ceph" }} + adminSecretNamespace: {{ .Values.storageclass.admin_secret_namespace }} pool: {{ .Values.storageclass.pool }} userId: {{ .Values.storageclass.user_id }} userSecretName: {{ .Values.storageclass.user_secret_name }} +{{- end }} diff --git a/ceph/templates/templates/_admin.keyring.tpl b/ceph/templates/templates/_admin.keyring.tpl new file mode 100644 index 0000000000..7c5de607b6 --- /dev/null +++ b/ceph/templates/templates/_admin.keyring.tpl @@ -0,0 +1,6 @@ +[client.admin] + key = {{"{{"}} key {{"}}"}} + auid = 0 + caps mds = "allow" + caps mon = "allow *" + caps osd = "allow *" diff --git a/ceph/templates/templates/_bootstrap.keyring.mds.tpl b/ceph/templates/templates/_bootstrap.keyring.mds.tpl new file mode 100644 index 0000000000..c52fd6397a --- /dev/null +++ b/ceph/templates/templates/_bootstrap.keyring.mds.tpl @@ -0,0 +1,3 @@ +[client.bootstrap-mds] + key = {{"{{"}} key {{"}}"}} + caps mon = "allow profile bootstrap-mds" diff --git a/ceph/templates/templates/_bootstrap.keyring.osd.tpl b/ceph/templates/templates/_bootstrap.keyring.osd.tpl new file mode 100644 index 0000000000..c5fe618d99 --- /dev/null +++ b/ceph/templates/templates/_bootstrap.keyring.osd.tpl @@ -0,0 +1,3 @@ +[client.bootstrap-osd] + key = {{"{{"}} key {{"}}"}} + caps mon = "allow profile bootstrap-osd" diff --git a/ceph/templates/templates/_bootstrap.keyring.rgw.tpl b/ceph/templates/templates/_bootstrap.keyring.rgw.tpl new file mode 100644 index 0000000000..1f2a58d6ab --- /dev/null +++ b/ceph/templates/templates/_bootstrap.keyring.rgw.tpl @@ -0,0 +1,3 @@ +[client.bootstrap-rgw] + key = {{"{{"}} key {{"}}"}} + caps mon = "allow profile bootstrap-rgw" diff --git a/ceph/templates/templates/_mon.keyring.tpl b/ceph/templates/templates/_mon.keyring.tpl new file mode 100644 index 0000000000..f9681f2d90 --- /dev/null +++ b/ceph/templates/templates/_mon.keyring.tpl @@ -0,0 +1,3 @@ +[mon.] + key = {{"{{"}} key {{"}}"}} + caps mon = "allow *" diff --git a/ceph/values.yaml b/ceph/values.yaml index ade9681e81..ebfd7160d4 100644 --- a/ceph/values.yaml +++ b/ceph/values.yaml @@ -12,6 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +manifests_enabled: + storage_secrets: true + client_secrets: true + deployment: true + replicas: mon: 3 rgw: 3 @@ -22,8 +27,10 @@ service: name: ceph-mon images: - daemon: docker.io/library/ceph/daemon:tag-build-master-jewel-ubuntu-16.04 - pull_policy: IfNotPresent + dep_check: docker.io/kolla/ubuntu-source-kubernetes-entrypoint:4.0.0 + daemon: quay.io/attcomdev/ceph-daemon:tag-build-master-jewel-ubuntu-16.04 + ceph_config_helper: docker.io/port/ceph-config-helper:v1.6.5 + pull_policy: Always labels: node_selector_key: ceph-storage @@ -33,23 +40,125 @@ pod_disruption_budget: mon: min_available: 0 +secrets: + keyrings: + mon: ceph-mon-keyring + mds: ceph-bootstrap-mds-keyring + osd: ceph-bootstrap-osd-keyring + rgw: ceph-bootstrap-rgw-keyring + admin: ceph-client-admin-keyring + network: - public: "10.25.0.0/16" + public: "192.168.0.0/16" + cluster: "192.168.0.0/16" port: mon: 6789 rgw_ingress: 80 rgw_target: 8088 -storage: - osd_directory: /var/lib/openstack-helm/ceph/osd - var_directory: /var/lib/openstack-helm/ceph/ceph - mon_directory: /var/lib/openstack-helm/ceph/mon +conf: + ceph: + override: + append: + config: + global: + # auth + cephx: true + cephx_require_signatures: false + cephx_cluster_require_signatures: true + cephx_service_require_signatures: false + + max_open_files: 131072 + osd_pool_default_pg_num: 128 + osd_pool_default_pgp_num: 128 + osd_pool_default_size: 3 + osd_pool_default_min_size: 1 + mon_osd_full_ratio: .95 + mon_osd_nearfull_ratio: .85 + mon_host: null + mon: + mon_osd_down_out_interval: 600 + mon_osd_min_down_reporters: 4 + mon_clock_drift_allowed: .15 + mon_clock_drift_warn_backoff: 30 + mon_osd_report_timeout: 300 + osd: + journal_size: 100 + osd_mkfs_type: xfs + osd_mkfs_options_xfs: -f -i size=2048 + osd_mon_heartbeat_interval: 30 + osd_max_object_name_len: 256 + #crush + osd_pool_default_crush_rule: 0 + osd_crush_update_on_start: true + #backend + osd_objectstore: filestore + #performance tuning + filestore_merge_threshold: 40 + filestore_split_multiple: 8 + osd_op_threads: 8 + filestore_op_threads: 8 + filestore_max_sync_interval: 5 + osd_max_scrubs: 1 + #recovery tuning + osd_recovery_max_active: 5 + osd_max_backfills: 2 + osd_recovery_op_priority: 2 + osd_client_op_priority: 63 + osd_recovery_max_chunk: 1048576 + osd_recovery_threads: 1 + #ports + ms_bind_port_min: 6800 + ms_bind_port_max: 7100 + client: + rbd_cache_enabled: true + rbd_cache_writethrough_until_flush: true + rbd_default_features: "1" + mds: + mds_cache_size: 100000 + + +dependencies: + mon: + jobs: + service: + osd: + jobs: + services: + - service: ceph_mon + endpoint: internal + moncheck: + jobs: + services: + - service: ceph_mon + endpoint: internal + rgw: + jobs: + services: + - service: ceph_mon + endpoint: internal + mds: + jobs: + services: + - service: ceph_mon + endpoint: internal + + +ceph: + enabled: + mds: true + rgw: false + storage: + osd_directory: /var/lib/openstack-helm/ceph/osd + var_directory: /var/lib/openstack-helm/ceph/ceph + mon_directory: /var/lib/openstack-helm/ceph/mon # rgw is optionally disabled rgw: enabled: false resources: + enabled: false osd: requests: memory: "512Mi" @@ -85,36 +194,14 @@ resources: limits: memory: "50Mi" cpu: "500m" - -# Setting this to false will assume you will -# setup and orchestrate your own secrets and -# configmaps outside of this helm chart -# -# The list below is in the format of -# -# configMapName: -# elementKeyName -# -# ceph.conf: -# ceph.conf -# ceph.client.admin.keyring -# ceph.client.admin.keyring -# ceph.mon.keyring: -# ceph.mon.keyring -# ceph-bootstrap-rgw-keyring: -# ceph.keyring -# ceph.rgw.keyring -# ceph-bootstrap-mds-keyring: -# ceph.keyring -# ceph.mds.keyring -# ceph-bootstrap-osd-keyring: -# ceph.keyring -# ceph.osd.keyring -# ceph-client-key: -# ceph-client-key -secrets: - use_common_secrets: true - + jobs: + secret_provisioning: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "500m" # if you change provision_storage_class to false # it is presumed you manage your own storage # class definition externally @@ -125,6 +212,13 @@ storageclass: pool: rbd admin_id: admin admin_secret_name: pvc-ceph-conf-combined-storageclass - admin_secret_namespace: null + admin_secret_namespace: ceph user_id: admin user_secret_name: pvc-ceph-client-key + +endpoints: + ceph_mon: + hosts: + default: ceph-mon + port: + mon: 6789 diff --git a/cinder/templates/_funcs.tpl b/cinder/templates/_funcs.tpl index 3d62b8f9b0..8f42e5cc27 100644 --- a/cinder/templates/_funcs.tpl +++ b/cinder/templates/_funcs.tpl @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -{{- define "cinder.is_ceph_configured" -}} +{{- define "cinder.is_ceph_volume_configured" -}} {{- range $section, $values := .Values.conf.backends -}} {{- if kindIs "map" $values -}} {{- if eq $values.volume_driver "cinder.volume.drivers.rbd.RBDDriver" -}} @@ -21,3 +21,10 @@ true {{- end -}} {{- end -}} {{- end -}} + +{{- define "cinder.is_ceph_backup_configured" -}} +{{- $values := .Values.conf.cinder.default.cinder -}} +{{- if eq $values.backup_driver "cinder.backup.drivers.ceph" -}} +true +{{- end -}} +{{- end -}} diff --git a/cinder/templates/etc/_ceph-cinder.keyring.tpl b/cinder/templates/bin/_ceph-keyring.sh.tpl similarity index 81% rename from cinder/templates/etc/_ceph-cinder.keyring.tpl rename to cinder/templates/bin/_ceph-keyring.sh.tpl index 6a33704297..1d248258d9 100644 --- a/cinder/templates/etc/_ceph-cinder.keyring.tpl +++ b/cinder/templates/bin/_ceph-keyring.sh.tpl @@ -1,3 +1,5 @@ +#!/bin/bash + # Copyright 2017 The Openstack-Helm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,9 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -ex +export HOME=/tmp + +cat < /etc/ceph/ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring [client.{{ .Values.conf.backends.rbd1.rbd_user }}] {{- if .Values.conf.ceph.cinder_keyring }} key = {{ .Values.conf.ceph.cinder_keyring }} {{- else }} - key = {{- include "secrets/ceph-client-key" . -}} + key = $(cat /tmp/client-keyring) {{- end }} +EOF + +exit 0 \ No newline at end of file diff --git a/cinder/templates/configmap-bin.yaml b/cinder/templates/configmap-bin.yaml index 961363e16d..338239e269 100644 --- a/cinder/templates/configmap-bin.yaml +++ b/cinder/templates/configmap-bin.yaml @@ -33,6 +33,8 @@ data: {{ tuple "bin/_cinder-scheduler.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} cinder-volume.sh: | {{ tuple "bin/_cinder-volume.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + ceph-keyring.sh: |+ +{{ tuple "bin/_ceph-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- if .Values.bootstrap.enabled }} bootstrap.sh: |+ {{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/cinder/templates/configmap-etc.yaml b/cinder/templates/configmap-etc.yaml index 6e050e26cb..f817c45bae 100644 --- a/cinder/templates/configmap-etc.yaml +++ b/cinder/templates/configmap-etc.yaml @@ -97,17 +97,4 @@ data: {{ .Values.conf.policy.override | indent 4 }} {{- else -}} {{ tuple "etc/_policy.json.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} -{{- end }} - ceph.conf: |+ -{{- if or (include "cinder.is_ceph_configured" .) (eq .Values.conf.cinder.default.cinder.backup_driver "cinder.backup.drivers.ceph") }} -{{ if .Values.conf.ceph.override -}} -{{ .Values.conf.ceph.override | indent 4 }} -{{- else -}} -{{ tuple "etc/_ceph.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} -{{- end }} -{{- if .Values.conf.ceph.append -}} -{{ .Values.conf.ceph.append | indent 4 }} -{{- end }} - ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring: |+ -{{ tuple "etc/_ceph-cinder.keyring.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} diff --git a/cinder/templates/deployment-backup.yaml b/cinder/templates/deployment-backup.yaml index 020ab575a2..55086c2352 100644 --- a/cinder/templates/deployment-backup.yaml +++ b/cinder/templates/deployment-backup.yaml @@ -42,6 +42,23 @@ spec: {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} initContainers: {{ tuple $envAll $dependencies $mounts_cinder_backup_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + {{- if include "cinder.is_ceph_backup_configured" . }} + - name: ceph-keyring-placement + image: {{ .Values.images.backup }} + imagePullPolicy: {{ .Values.images.pull_policy }} + command: + - /tmp/ceph-keyring.sh + volumeMounts: + - name: etcceph + mountPath: /etc/ceph + - name: cinder-bin + mountPath: /tmp/ceph-keyring.sh + subPath: ceph-keyring.sh + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + {{ end }} containers: - name: cinder-backup image: {{ .Values.images.backup }} @@ -67,13 +84,15 @@ spec: subPath: cinder.conf readOnly: true {{- if eq .Values.conf.cinder.default.cinder.backup_driver "cinder.backup.drivers.ceph" }} - - name: cinder-etc + - name: etcceph + mountPath: /etc/ceph + - name: ceph-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true - - name: cinder-etc - mountPath: /etc/ceph/ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring - subPath: ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key readOnly: true {{- end -}} {{ if $mounts_cinder_backup.volumeMounts }}{{ toYaml $mounts_cinder_backup.volumeMounts | indent 12 }}{{ end }} @@ -83,4 +102,18 @@ spec: - name: cinder-etc configMap: name: cinder-etc + - name: cinder-bin + configMap: + name: cinder-bin + defaultMode: 0555 + {{- if include "cinder.is_ceph_backup_configured" . }} + - name: etcceph + emptyDir: {} + - name: ceph-etc + configMap: + name: ceph-etc + - name: ceph-keyring + secret: + secretName: pvc-ceph-client-key + {{ end }} {{ if $mounts_cinder_backup.volumes }}{{ toYaml $mounts_cinder_backup.volumes | indent 8 }}{{ end }} diff --git a/cinder/templates/deployment-volume.yaml b/cinder/templates/deployment-volume.yaml index 754452ffbe..afe59779c6 100644 --- a/cinder/templates/deployment-volume.yaml +++ b/cinder/templates/deployment-volume.yaml @@ -42,6 +42,23 @@ spec: {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} initContainers: {{ tuple $envAll $dependencies $mounts_cinder_volume_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + {{- if include "cinder.is_ceph_volume_configured" . }} + - name: ceph-keyring-placement + image: {{ .Values.images.volume }} + imagePullPolicy: {{ .Values.images.pull_policy }} + command: + - /tmp/ceph-keyring.sh + volumeMounts: + - name: etcceph + mountPath: /etc/ceph + - name: cinder-bin + mountPath: /tmp/ceph-keyring.sh + subPath: ceph-keyring.sh + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + {{ end }} containers: - name: cinder-volume image: {{ .Values.images.volume }} @@ -72,14 +89,16 @@ spec: mountPath: /etc/cinder/conf/backends.conf subPath: backends.conf readOnly: true - {{- if include "cinder.is_ceph_configured" . }} - - name: cinder-etc + {{- if include "cinder.is_ceph_volume_configured" . }} + - name: etcceph + mountPath: /etc/ceph + - name: ceph-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true - - name: cinder-etc - mountPath: /etc/ceph/ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring - subPath: ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key readOnly: true {{- end }} {{ if $mounts_cinder_volume.volumeMounts }}{{ toYaml $mounts_cinder_volume.volumeMounts | indent 12 }}{{ end }} @@ -93,4 +112,14 @@ spec: - name: cinder-etc configMap: name: cinder-etc + {{- if include "cinder.is_ceph_volume_configured" . }} + - name: etcceph + emptyDir: {} + - name: ceph-etc + configMap: + name: ceph-etc + - name: ceph-keyring + secret: + secretName: pvc-ceph-client-key + {{ end }} {{ if $mounts_cinder_volume.volumes }}{{ toYaml $mounts_cinder_volume.volumes | indent 8 }}{{ end }} diff --git a/cinder/templates/etc/_ceph.conf.tpl b/cinder/templates/etc/_ceph.conf.tpl deleted file mode 100644 index e42694fc27..0000000000 --- a/cinder/templates/etc/_ceph.conf.tpl +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[global] -rgw_thread_pool_size = 1024 -rgw_num_rados_handles = 100 -{{- if .Values.conf.ceph.monitors }} -[mon] -{{ range .Values.conf.ceph.monitors }} - [mon.{{ . }}] - host = {{ . }} - mon_addr = {{ . }} -{{ end }} -{{- else }} -mon_host = ceph-mon.ceph -{{- end }} -[client] - rbd_cache_enabled = true - rbd_cache_writethrough_until_flush = true diff --git a/cinder/values.yaml b/cinder/values.yaml index fd9ce2d6af..433cea4be7 100644 --- a/cinder/values.yaml +++ b/cinder/values.yaml @@ -23,6 +23,8 @@ replicas: scheduler: 1 backup: 1 +storage: ceph + labels: node_selector_key: openstack-control-plane node_selector_value: enabled diff --git a/doc/source/install/multinode.rst b/doc/source/install/multinode.rst index a8f3fb4a65..28ee45511c 100644 --- a/doc/source/install/multinode.rst +++ b/doc/source/install/multinode.rst @@ -141,32 +141,13 @@ completed. Installing Ceph Host Requirements ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -At some future point, we want to ensure that our solution is -cloud-native, allowing installation on any host system without a package -manager and only a container runtime (i.e. CoreOS). Until this happens, -we will need to ensure that ``ceph-common`` is installed on each of our -hosts. Using our Ubuntu example: +You need to ensure that ``ceph-common`` or equivalent is +installed on each of our hosts. Using our Ubuntu example: :: sudo apt-get install ceph-common -y -We will always attempt to keep host-specific requirements to a minimum, -and we are working with the Ceph team (Sébastien Han) to quickly address -this Ceph requirement. - -Ceph Secrets Generation -~~~~~~~~~~~~~~~~~~~~~~~ - -Another thing of interest is that our deployment assumes that you can -generate secrets at the time of the container deployment. We require the -`sigil `__ -binary on your deployment host in order to perform this action. - -:: - - curl -L https://github.com/gliderlabs/sigil/releases/download/v0.4.0/sigil_0.4.0_Linux_x86_64.tgz | sudo tar -zxC /usr/local/bin - Kubernetes Controller Manager ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -200,20 +181,18 @@ Kubernetes v1.6.5. export kube_version=v1.6.5 sudo sed -i "s|gcr.io/google_containers/kube-controller-manager-amd64:$kube_version|quay.io/attcomdev/kube-controller-manager:$kube_version|g" /etc/kubernetes/manifests/kube-controller-manager.yaml -Now you will want to ``restart`` your Kubernetes master server to +Now you will want to ``restart`` the Kubernetes master server to continue. Kube Controller Manager DNS Resolution ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Until the following `Kubernetes Pull -Request `__ is -merged, you will need to allow the Kubernetes Controller to use the -internal container ``skydns`` endpoint as a DNS server, and add the -Kubernetes search suffix into the controller's resolv.conf. As of now, -the Kubernetes controller only mirrors the host's ``resolv.conf``. This -is not sufficient if you want the controller to know how to correctly -resolve container service endpoints (in the case of DaemonSets). +You will need to allow the Kubernetes Controller to use the +Kubernetes service DNS server, and add the Kubernetes search suffix +to the controller's resolv.conf. As of now, the Kubernetes controller +only mirrors the host's ``resolv.conf``. This is not sufficient if you +want the controller to know how to correctly resolve container service +endpoints. First, find out what the IP Address of your ``kube-dns`` deployment is: @@ -224,82 +203,16 @@ First, find out what the IP Address of your ``kube-dns`` deployment is: kube-dns 10.96.0.10 53/UDP,53/TCP 1d admin@kubenode01:~$ -As you can see by this example, ``10.96.0.10`` is the -``CLUSTER-IP``\ IP. Now, have a look at the current -``kube-controller-manager-kubenode01`` ``/etc/resolv.conf``: +Then update the controller manager configuration to match: :: - admin@kubenode01:~$ kubectl exec kube-controller-manager-kubenode01 -n kube-system -- cat /etc/resolv.conf - # Dynamic resolv.conf(5) file for glibc resolver(3) generated by resolvconf(8) - # DO NOT EDIT THIS FILE BY HAND -- YOUR CHANGES WILL BE OVERWRITTEN - nameserver 192.168.1.70 - nameserver 8.8.8.8 - search jinkit.com - admin@kubenode01:~$ - -What we need is for ``kube-controller-manager-kubenode01`` -``/etc/resolv.conf`` to look like this: - -:: - - admin@kubenode01:~$ kubectl exec kube-controller-manager-kubenode01 -n kube-system -- cat /etc/resolv.conf + admin@kubenode01:~$ CONTROLLER_MANAGER_POD=$(kubectl get -n kube-system pods -l component=kube-controller-manager --no-headers -o name | head -1 | awk -F '/' '{ print $NF }') + admin@kubenode01:~$ kubectl exec -n kube-system ${CONTROLLER_MANAGER_POD} -- sh -c "cat > /etc/resolv.conf < /etc/resolv.conf - nameserver 10.96.0.10 - nameserver 192.168.1.70 - nameserver 8.8.8.8 - search svc.cluster.local jinkit.com - EOF - root@kubenode01:/# - -Now you can test your changes by deploying a service to your cluster, -and resolving this from the controller. As an example, lets deploy -something useful, like `Kubernetes -dashboard `__: - -:: - - kubectl create -f https://rawgit.com/kubernetes/dashboard/master/src/deploy/kubernetes-dashboard.yaml - -Note the ``IP`` field: - -:: - - admin@kubenode01:~$ kubectl describe svc kubernetes-dashboard -n kube-system - Name: kubernetes-dashboard - Namespace: kube-system - Labels: app=kubernetes-dashboard - Selector: app=kubernetes-dashboard - Type: NodePort - IP: 10.110.207.144 - Port: 80/TCP - NodePort: 32739/TCP - Endpoints: 10.25.178.65:9090 - Session Affinity: None - No events. - admin@kubenode01:~$ - -Now you should be able to resolve the host -``kubernetes-dashboard.kube-system.svc.cluster.local``: - -:: - - admin@kubenode01:~$ kubectl exec kube-controller-manager-kubenode01 -it -n kube-system -- ping kubernetes-dashboard.kube-system.svc.cluster.local - PING kubernetes-dashboard.kube-system.svc.cluster.local (10.110.207.144) 56(84) bytes of data. - -.. note:: - This host example above has ``iputils-ping`` installed. + search cluster.local svc.cluster.local + EOF" Kubernetes Node DNS Resolution ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -358,42 +271,15 @@ Download the latest copy of Openstack-Helm: Ceph Preparation and Installation --------------------------------- -Ceph must be aware of the OSX cluster and public networks. These CIDR +Ceph must be aware of the OSD cluster and public networks. These CIDR ranges are the exact same ranges you used earlier in your Calico -deployment yaml (our example was 10.25.0.0/16 due to our 192.168.0.0/16 -overlap). Explore this variable to your deployment environment by +deployment yaml. Export this variable to your deployment environment by issuing the following commands: :: - export osd_cluster_network=10.25.0.0/16 - export osd_public_network=10.25.0.0/16 - -Ceph Storage Volumes --------------------- - -Ceph must also have volumes to mount on each host labeled for -``ceph-storage``. On each host that you labeled, create the following -directory (can be overriden): - -:: - - mkdir -p /var/lib/openstack-helm/ceph - -*Repeat this step for each node labeled: ``ceph-storage``* - -Ceph Secrets Generation ------------------------ - -Although you can bring your own secrets, we have conveniently created a -secret generation tool for you (for greenfield deployments). You can -create secrets for your project by issuing the following: - -:: - - cd helm-toolkit/utils/secret-generator - ./generate_secrets.sh all `./generate_secrets.sh fsid` - cd ../../.. + export osd_cluster_network=192.168.0.0/16 + export osd_public_network=192.168.0.0/16 Nova Compute Instance Storage ----------------------------- @@ -468,30 +354,55 @@ the following command to install Ceph: :: - helm install --set network.public=$osd_public_network --name=ceph local/ceph --namespace=ceph + helm install --namespace=ceph local/ceph --name=ceph \ + --set manifests_enabled.client_secrets=false \ + --set network.public=$osd_public_network \ + --set network.cluster=$osd_cluster_network -Bootstrap Installation ----------------------- +Activating Control-Plane Namespace for Ceph +------------------------------------------- -At this time (and before verification of Ceph) you'll need to install -the ``bootstrap`` chart. The ``bootstrap`` chart will install secrets -for both the ``ceph`` and ``openstack`` namespaces for the general -StorageClass: +In order for Ceph to fulfill PersistentVolumeClaims within Kubernetes namespaces +outside of Ceph's namespace, a client keyring needs to be present within that +namespace. For the rest of the OpenStack and supporting core services, this guide +will be deploying the control plane to a seperate namespace ``openstack``. To +deploy the aforementioned client keyring to the ``openstack`` namespace: :: - helm install --name=bootstrap-ceph local/bootstrap --namespace=ceph - helm install --name=bootstrap-openstack local/bootstrap --namespace=openstack + helm install --namespace=openstack local/ceph --name=ceph-openstack-config \ + --set manifests_enabled.storage_secrets=false \ + --set manifests_enabled.deployment=false \ + --set ceph.namespace=ceph \ + --set network.public=$osd_public_network \ + --set network.cluster=$osd_cluster_network + +This will load the client keyring as well as the same ``ceph.conf`` into +the specified namespace. Deploying ceph.conf into this namespace allows +OpenStack services to consume this ConfigMap for their Ceph-specific +configurations. You may want to validate that Ceph is deployed successfully. For more information on this, please see the section entitled `Ceph Troubleshooting <../../operator/troubleshooting/persistent-storage.html>`__. +Ceph pool creation +------------------ + +You should now be ready to create the pools for OpenStack services to consume, +using the following commands: + +:: + + kubectl exec -n ceph ceph-mon-0 -- ceph osd pool create volumes 8 + kubectl exec -n ceph ceph-mon-0 -- ceph osd pool create images 8 + kubectl exec -n ceph ceph-mon-0 -- ceph osd pool create vms 8 + MariaDB Installation and Verification ------------------------------------- -We are using Galera to cluster MariaDB and establish a quorum. To -install the MariaDB, issue the following command: +We are using Galera to cluster MariaDB. To install MariaDB, issue the following +command: :: @@ -514,43 +425,49 @@ Now you can easily install the other services simply by going in order: :: - helm install --name=keystone local/keystone --set replicas=2 --namespace=openstack + helm install --namespace=openstack --name=keystone local/keystone \ + --set replicas=2 **Install Horizon:** :: - helm install --name=horizon local/horizon --set network.enable_node_port=true --namespace=openstack + helm install --namespace=openstack --name=horizon local/horizon \ + --set network.enable_node_port=true **Install Glance:** :: - helm install --name=glance local/glance --set replicas.api=2,replicas.registry=2 --namespace=openstack + helm install --namespace=openstack --name=glance local/glance \ + --set replicas.api=2,replicas.registry=2 **Install Heat:** :: - helm install --name=heat local/heat --namespace=openstack + helm install --namespace=openstack --name=heat local/heat **Install Neutron:** :: - helm install --name=neutron local/neutron --set replicas.server=2 --namespace=openstack + helm install --namespace=openstack --name=neutron local/neutron \ + --set replicas.server=2 **Install Nova:** :: - helm install --name=nova local/nova --set control_replicas=2 --namespace=openstack + helm install --namespace=openstack --name=nova local/nova \ + --set control_replicas=2 **Install Cinder:** :: - helm install --name=cinder local/cinder --set replicas.api=2 --namespace=openstack + helm install --namespace=openstack --name=cinder local/cinder \ + --set replicas.api=2 Final Checks ------------ diff --git a/glance/templates/etc/_ceph.client.glance.keyring.tpl b/glance/templates/bin/_ceph-keyring.sh.tpl similarity index 79% rename from glance/templates/etc/_ceph.client.glance.keyring.tpl rename to glance/templates/bin/_ceph-keyring.sh.tpl index 0e0e639a2e..411b2bb639 100644 --- a/glance/templates/etc/_ceph.client.glance.keyring.tpl +++ b/glance/templates/bin/_ceph-keyring.sh.tpl @@ -1,3 +1,5 @@ +#!/bin/bash + # Copyright 2017 The Openstack-Helm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,9 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -ex +export HOME=/tmp + +cat < /etc/ceph/ceph.client.{{ .Values.conf.glance.glance_store.glance.store.rbd_store_user }}.keyring [client.{{ .Values.conf.glance.glance_store.glance.store.rbd_store_user }}] {{- if .Values.conf.ceph.keyring }} key = {{ .Values.conf.ceph.keyring }} {{- else }} - key = {{- include "secrets/ceph-client-key" . -}} + key = $(cat /tmp/client-keyring) {{- end }} +EOF + +exit 0 diff --git a/glance/templates/configmap-bin.yaml b/glance/templates/configmap-bin.yaml index b5d90d5f2a..b104293f73 100644 --- a/glance/templates/configmap-bin.yaml +++ b/glance/templates/configmap-bin.yaml @@ -35,3 +35,5 @@ data: {{ tuple "bin/_glance-registry.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} bootstrap.sh: |+ {{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + ceph-keyring.sh: |+ +{{ tuple "bin/_ceph-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/glance/templates/configmap-etc.yaml b/glance/templates/configmap-etc.yaml index 2999cd472b..13655b773a 100644 --- a/glance/templates/configmap-etc.yaml +++ b/glance/templates/configmap-etc.yaml @@ -78,30 +78,6 @@ data: {{- end }} {{- if .Values.conf.rally_tests.append -}} {{ .Values.conf.rally_tests.append | indent 4 }} -{{- end }} - ceph.conf: |+ -{{ if .Values.conf.ceph.override -}} -{{ .Values.conf.ceph.override | indent 4 }} -{{- else -}} -{{- if .Values.conf.ceph.prefix -}} -{{ .Values.conf.ceph.prefix | indent 4 }} -{{- end }} -{{ tuple "etc/_ceph.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} -{{- end }} -{{- if .Values.conf.ceph.append -}} -{{ .Values.conf.ceph.append | indent 4 }} -{{- end }} - ceph.client.{{ .Values.conf.glance.glance_store.glance.store.rbd_store_user }}.keyring: |+ -{{ if .Values.conf.ceph_client.override -}} -{{ .Values.conf.ceph_client.override | indent 4 }} -{{- else -}} -{{- if .Values.conf.ceph_client.prefix -}} -{{ .Values.conf.ceph_client.prefix | indent 4 }} -{{- end }} -{{ tuple "etc/_ceph.client.glance.keyring.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} -{{- end }} -{{- if .Values.conf.ceph_client.append -}} -{{ .Values.conf.ceph_client.append | indent 4 }} {{- end }} glance-api.conf: |+ {{ if .Values.conf.glance.override -}} diff --git a/glance/templates/deployment-api.yaml b/glance/templates/deployment-api.yaml index 5a6c015b6d..83d5fdca62 100644 --- a/glance/templates/deployment-api.yaml +++ b/glance/templates/deployment-api.yaml @@ -43,6 +43,47 @@ spec: terminationGracePeriodSeconds: {{ .Values.termination_grace_period.api.timeout | default "600" }} initContainers: {{ tuple $envAll $dependencies $mounts_glance_api_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + {{- if eq .Values.storage "pvc" }} + - name: glance-perms + image: {{ .Values.images.api }} + imagePullPolicy: {{ .Values.images.pull_policy }} + securityContext: + runAsUser: 0 + {{- if .Values.resources.enabled }} + resources: + limits: + cpu: {{ .Values.resources.api.limits.cpu | quote }} + memory: {{ .Values.resources.api.limits.memory | quote }} + requests: + cpu: {{ .Values.resources.api.requests.cpu | quote }} + memory: {{ .Values.resources.api.requests.memory | quote }} + {{- end }} + command: + - chown + - -R + - "glance:" + - {{ .Values.conf.glance.glance_store.glance.store.filesystem_store_datadir }} + volumeMounts: + - name: glance-images + mountPath: {{ .Values.conf.glance.glance_store.glance.store.filesystem_store_datadir }} + {{- end }} + {{ if eq .Values.storage "ceph" }} + - name: ceph-keyring-placement + image: {{ .Values.images.api }} + imagePullPolicy: {{ .Values.images.pull_policy }} + command: + - /tmp/ceph-keyring.sh + volumeMounts: + - name: etcceph + mountPath: /etc/ceph + - name: glance-bin + mountPath: /tmp/ceph-keyring.sh + subPath: ceph-keyring.sh + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + {{ end }} containers: - name: glance-api image: {{ .Values.images.api }} @@ -93,13 +134,15 @@ spec: - name: glance-images mountPath: {{ .Values.conf.glance.glance_store.glance.store.filesystem_store_datadir }} {{- else }} - - name: glance-etc + - name: etcceph + mountPath: /etc/ceph + - name: ceph-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true - - name: glance-etc - mountPath: /etc/ceph/ceph.client.{{ .Values.conf.glance.glance_store.glance.store.rbd_store_user }}.keyring - subPath: ceph.client.{{ .Values.conf.glance.glance_store.glance.store.rbd_store_user }}.keyring + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key readOnly: true {{- end }} {{ if $mounts_glance_api.volumeMounts }}{{ toYaml $mounts_glance_api.volumeMounts | indent 12 }}{{ end }} @@ -117,5 +160,14 @@ spec: - name: glance-images persistentVolumeClaim: claimName: glance-images +{{ else }} + - name: etcceph + emptyDir: {} + - name: ceph-etc + configMap: + name: ceph-etc + - name: ceph-keyring + secret: + secretName: pvc-ceph-client-key {{- end }} {{ if $mounts_glance_api.volumes }}{{ toYaml $mounts_glance_api.volumes | indent 8 }}{{ end }} diff --git a/glance/templates/etc/_ceph.conf.tpl b/glance/templates/etc/_ceph.conf.tpl deleted file mode 100644 index e42694fc27..0000000000 --- a/glance/templates/etc/_ceph.conf.tpl +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[global] -rgw_thread_pool_size = 1024 -rgw_num_rados_handles = 100 -{{- if .Values.conf.ceph.monitors }} -[mon] -{{ range .Values.conf.ceph.monitors }} - [mon.{{ . }}] - host = {{ . }} - mon_addr = {{ . }} -{{ end }} -{{- else }} -mon_host = ceph-mon.ceph -{{- end }} -[client] - rbd_cache_enabled = true - rbd_cache_writethrough_until_flush = true diff --git a/helm-toolkit/secrets/.gitkeep b/helm-toolkit/secrets/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/helm-toolkit/utils/secret-generator/README.rst b/helm-toolkit/utils/secret-generator/README.rst deleted file mode 100644 index ace977e6bb..0000000000 --- a/helm-toolkit/utils/secret-generator/README.rst +++ /dev/null @@ -1,78 +0,0 @@ -Ceph Kubernetes Secret Generation -================================= - -This script will generate ceph keyrings and configs as Kubernetes -secrets. - -Sigil is required for template handling and must be installed in system -``PATH``. Instructions can be found`here -`__ - -The following functions are provided: - -Generate raw FSID (can be used for other functions) ---------------------------------------------------- - -.. code:: bash - - ./generate_secrets.sh fsid - -Generate raw ceph.conf (For verification) ------------------------------------------ - -.. code:: bash - - ./generate_secrets.sh ceph-conf-raw "overridekey=value" - -Take a look at ``ceph/ceph.conf.tmpl`` for the default values - -Generate encoded ceph.conf secret ---------------------------------- - -.. code:: bash - - ./generate_secrets.sh ceph-conf "overridekey=value" - -Generate encoded admin keyring secret -------------------------------------- - -.. code:: bash - - ./generate_secrets.sh admin-keyring - -Generate encoded mon keyring secret ------------------------------------ - -.. code:: bash - - ./generate_secrets.sh mon-keyring - -Generate a combined secret --------------------------- - -Contains ceph.conf, admin keyring and mon keyring. Useful for generating -the ``/etc/ceph`` directory - -.. code:: bash - - ./generate_secrets.sh combined-conf - -Generate encoded boostrap keyring secret ----------------------------------------- - -.. code:: bash - - ./generate_secrets.sh bootstrap-keyring - -Kubernetes workflow -=================== - -.. code:: bash - - ./generator/generate_secrets.sh all `./generate_secrets.sh fsid` - - kubectl create secret generic ceph-conf-combined --from-file=ceph.conf --from-file=ceph.client.admin.keyring --from-file=ceph.mon.keyring --namespace=ceph - kubectl create secret generic ceph-bootstrap-rgw-keyring --from-file=ceph.keyring=ceph.rgw.keyring --namespace=ceph - kubectl create secret generic ceph-bootstrap-mds-keyring --from-file=ceph.keyring=ceph.mds.keyring --namespace=ceph - kubectl create secret generic ceph-bootstrap-osd-keyring --from-file=ceph.keyring=ceph.osd.keyring --namespace=ceph - kubectl create secret generic ceph-client-key --from-file=ceph-client-key --namespace=ceph diff --git a/helm-toolkit/utils/secret-generator/generate_secrets.sh b/helm-toolkit/utils/secret-generator/generate_secrets.sh deleted file mode 100755 index a74bf1d040..0000000000 --- a/helm-toolkit/utils/secret-generator/generate_secrets.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/bash - -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -gen-fsid() { - echo "$(uuidgen)" -} - -gen-ceph-conf-raw() { - fsid=${1:?} - shift - conf=$(sigil -p -f templates/ceph/ceph.conf.tmpl "fsid=${fsid}" $@) - echo "${conf}" -} - -gen-ceph-conf() { - fsid=${1:?} - shift - conf=$(sigil -p -f templates/ceph/ceph.conf.tmpl "fsid=${fsid}" $@) - echo "${conf}" -} - -gen-admin-keyring() { - key=$(python ceph-key.py) - keyring=$(sigil -f templates/ceph/admin.keyring.tmpl "key=${key}") - echo "${keyring}" -} - -gen-mon-keyring() { - key=$(python ceph-key.py) - keyring=$(sigil -f templates/ceph/mon.keyring.tmpl "key=${key}") - echo "${keyring}" -} - -gen-combined-conf() { - fsid=${1:?} - shift - conf=$(sigil -p -f templates/ceph/ceph.conf.tmpl "fsid=${fsid}" $@) - echo "${conf}" > ../../secrets/ceph.conf - - key=$(python ceph-key.py) - keyring=$(sigil -f templates/ceph/admin.keyring.tmpl "key=${key}") - echo "${key}" > ../../secrets/ceph-client-key - echo "${keyring}" > ../../secrets/ceph.client.admin.keyring - - key=$(python ceph-key.py) - keyring=$(sigil -f templates/ceph/mon.keyring.tmpl "key=${key}") - echo "${keyring}" > ../../secrets/ceph.mon.keyring -} - -gen-bootstrap-keyring() { - service="${1:-osd}" - key=$(python ceph-key.py) - bootstrap=$(sigil -f templates/ceph/bootstrap.keyring.tmpl "key=${key}" "service=${service}") - echo "${bootstrap}" -} - -gen-all-bootstrap-keyrings() { - gen-bootstrap-keyring osd > ../../secrets/ceph.osd.keyring - gen-bootstrap-keyring mds > ../../secrets/ceph.mds.keyring - gen-bootstrap-keyring rgw > ../../secrets/ceph.rgw.keyring -} - -gen-all() { - gen-combined-conf $@ - gen-all-bootstrap-keyrings -} - - -main() { - set -eo pipefail - case "$1" in - fsid) shift; gen-fsid $@;; - ceph-conf-raw) shift; gen-ceph-conf-raw $@;; - ceph-conf) shift; gen-ceph-conf $@;; - admin-keyring) shift; gen-admin-keyring $@;; - mon-keyring) shift; gen-mon-keyring $@;; - bootstrap-keyring) shift; gen-bootstrap-keyring $@;; - combined-conf) shift; gen-combined-conf $@;; - all) shift; gen-all $@;; - esac -} - -main "$@" diff --git a/helm-toolkit/utils/secret-generator/templates/ceph/admin.keyring.tmpl b/helm-toolkit/utils/secret-generator/templates/ceph/admin.keyring.tmpl deleted file mode 100644 index b0f04dba26..0000000000 --- a/helm-toolkit/utils/secret-generator/templates/ceph/admin.keyring.tmpl +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[client.admin] - key = {{ $key }} - auid = 0 - caps mds = "allow" - caps mon = "allow *" - caps osd = "allow *" diff --git a/helm-toolkit/utils/secret-generator/templates/ceph/bootstrap.keyring.tmpl b/helm-toolkit/utils/secret-generator/templates/ceph/bootstrap.keyring.tmpl deleted file mode 100644 index 639a1010be..0000000000 --- a/helm-toolkit/utils/secret-generator/templates/ceph/bootstrap.keyring.tmpl +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[client.bootstrap-{{ $service }}] - key = {{ $key }} - caps mon = "allow profile bootstrap-{{ $service }}" diff --git a/helm-toolkit/utils/secret-generator/templates/ceph/ceph.conf.tmpl b/helm-toolkit/utils/secret-generator/templates/ceph/ceph.conf.tmpl deleted file mode 100644 index b7c925bf9f..0000000000 --- a/helm-toolkit/utils/secret-generator/templates/ceph/ceph.conf.tmpl +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[global] -fsid = ${fsid:?} -cephx = ${auth_cephx:-"true"} -cephx_require_signatures = ${auth_cephx_require_signatures:-"false"} -cephx_cluster_require_signatures = ${auth_cephx_cluster_require_signatures:-"true"} -cephx_service_require_signatures = ${auth_cephx_service_require_signatures:-"false"} - -# auth -max_open_files = ${global_max_open_files:-"131072"} -osd_pool_default_pg_num = ${global_osd_pool_default_pg_num:-"128"} -osd_pool_default_pgp_num = ${global_osd_pool_default_pgp_num:-"128"} -osd_pool_default_size = ${global_osd_pool_default_size:-"3"} -osd_pool_default_min_size = ${global_osd_pool_default_min_size:-"1"} - -mon_osd_full_ratio = ${global_mon_osd_full_ratio:-".95"} -mon_osd_nearfull_ratio = ${global_mon_osd_nearfull_ratio:-".85"} - -mon_host = ${global_mon_host:-'ceph-mon'} - -[mon] -mon_osd_down_out_interval = ${mon_mon_osd_down_out_interval:-"600"} -mon_osd_min_down_reporters = ${mon_mon_osd_min_down_reporters:-"4"} -mon_clock_drift_allowed = ${mon_mon_clock_drift_allowed:-".15"} -mon_clock_drift_warn_backoff = ${mon_mon_clock_drift_warn_backoff:-"30"} -mon_osd_report_timeout = ${mon_mon_osd_report_timeout:-"300"} - - -[osd] -journal_size = ${osd_journal_size:-"100"} -cluster_network = ${osd_cluster_network:-'192.168.0.0/16'} -public_network = ${osd_public_network:-'192.168.0.0/16'} -osd_mkfs_type = ${osd_osd_mkfs_type:-"xfs"} -osd_mkfs_options_xfs = ${osd_osd_mkfs_options_xfs:-"-f -i size=2048"} -osd_mon_heartbeat_interval = ${osd_osd_mon_heartbeat_interval:-"30"} -osd_max_object_name_len = ${osd_max_object_name_len:-"256"} - -#crush -osd_pool_default_crush_rule = ${osd_pool_default_crush_rule:-"0"} -osd_crush_update_on_start = ${osd_osd_crush_update_on_start:-"true"} - -#backend -osd_objectstore = ${osd_osd_objectstore:-"filestore"} - -#performance tuning -filestore_merge_threshold = ${osd_filestore_merge_threshold:-"40"} -filestore_split_multiple = ${osd_filestore_split_multiple:-"8"} -osd_op_threads = ${osd_osd_op_threads:-"8"} -filestore_op_threads = ${osd_filestore_op_threads:-"8"} -filestore_max_sync_interval = ${osd_filestore_max_sync_interval:-"5"} -osd_max_scrubs = ${osd_osd_max_scrubs:-"1"} - - -#recovery tuning -osd_recovery_max_active = ${osd_osd_recovery_max_active:-"5"} -osd_max_backfills = ${osd_osd_max_backfills:-"2"} -osd_recovery_op_priority = ${osd_osd_recovery_op_priority:-"2"} -osd_client_op_priority = ${osd_osd_client_op_priority:-"63"} -osd_recovery_max_chunk = ${osd_osd_recovery_max_chunk:-"1048576"} -osd_recovery_threads = ${osd_osd_recovery_threads:-"1"} - -#ports -ms_bind_port_min = ${osd_ms_bind_port_min:-"6800"} -ms_bind_port_max = ${osd_ms_bind_port_max:-"7100"} - -[client] -rbd_cache_enabled = ${client_rbd_cache_enabled:-"true"} -rbd_cache_writethrough_until_flush = ${client_rbd_cache_writethrough_until_flush:-"true"} -rbd_default_features = ${client_rbd_default_features:-"1"} - -[mds] -mds_cache_size = ${mds_mds_cache_size:-"100000"} diff --git a/helm-toolkit/utils/secret-generator/templates/ceph/mon.keyring.tmpl b/helm-toolkit/utils/secret-generator/templates/ceph/mon.keyring.tmpl deleted file mode 100644 index 9a88a2421c..0000000000 --- a/helm-toolkit/utils/secret-generator/templates/ceph/mon.keyring.tmpl +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[mon.] - key = {{ $key }} - caps mon = "allow *" diff --git a/helm-toolkit/utils/test/ceph-rbd-test.yaml b/helm-toolkit/utils/test/ceph-rbd-test.yaml deleted file mode 100644 index 32b91d2a28..0000000000 --- a/helm-toolkit/utils/test/ceph-rbd-test.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Pod -metadata: - name: ceph-rbd-test -spec: - containers: - - name: cephrbd-rw - image: busybox - command: - - sh - - -c - - while true; do sleep 1; done - volumeMounts: - - mountPath: "/mnt/cephrbd" - name: cephrbd - volumes: - - name: cephrbd - rbd: - monitors: -#This only works if you have skyDNS resolveable from the kubernetes node. Otherwise you must manually put in one or more mon pod ips. - - ceph-mon.ceph:6789 - user: admin - image: ceph-rbd-test - pool: rbd - secretRef: - name: ceph-client-key diff --git a/nova/templates/etc/_ceph.client.cinder.keyring.yaml.tpl b/nova/templates/bin/_ceph-keyring.sh.tpl similarity index 85% rename from nova/templates/etc/_ceph.client.cinder.keyring.yaml.tpl rename to nova/templates/bin/_ceph-keyring.sh.tpl index 1a0200d098..0407e50271 100644 --- a/nova/templates/etc/_ceph.client.cinder.keyring.yaml.tpl +++ b/nova/templates/bin/_ceph-keyring.sh.tpl @@ -1,3 +1,5 @@ +#!/bin/bash + # Copyright 2017 The Openstack-Helm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,9 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -ex +export HOME=/tmp + +cat < /etc/ceph/ceph.client.keyring [client.{{ .Values.ceph.cinder_user }}] {{- if .Values.ceph.cinder_keyring }} key = {{ .Values.ceph.cinder_keyring }} {{- else }} - key = {{- include "secrets/ceph-client-key" . -}} + key = $(cat /tmp/client-keyring) {{- end }} +EOF + +exit 0 diff --git a/nova/templates/configmap-bin.yaml b/nova/templates/configmap-bin.yaml index 76188e5c67..aee5c9ea23 100644 --- a/nova/templates/configmap-bin.yaml +++ b/nova/templates/configmap-bin.yaml @@ -33,7 +33,7 @@ data: {{ tuple "bin/_libvirt.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- if .Values.ceph.enabled }} ceph-secret-define.sh: | -{{ tuple "bin/_ceph-secret-define.sh.tpl" . | include "helm-toolkit.template" | indent 4 }} +{{ tuple "bin/_ceph-secret-define.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} bootstrap.sh: | {{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} @@ -51,3 +51,5 @@ data: {{ tuple "bin/_nova-scheduler.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} fake-iptables.sh: | {{ tuple "bin/_fake-iptables.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + ceph-keyring.sh: |+ +{{ tuple "bin/_ceph-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/nova/templates/configmap-etc.yaml b/nova/templates/configmap-etc.yaml index bf7e5b5801..706fb2ef41 100644 --- a/nova/templates/configmap-etc.yaml +++ b/nova/templates/configmap-etc.yaml @@ -92,8 +92,6 @@ data: {{- else -}} {{ tuple "etc/_policy.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} - ceph.client.cinder.keyring.yaml: |+ -{{ tuple "etc/_ceph.client.cinder.keyring.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} resolv.conf: |+ {{ tuple "etc/_resolv.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} libvirtd.conf: |+ diff --git a/nova/templates/daemonset-compute.yaml b/nova/templates/daemonset-compute.yaml index a6f82fea62..7f1bd6e162 100644 --- a/nova/templates/daemonset-compute.yaml +++ b/nova/templates/daemonset-compute.yaml @@ -39,6 +39,23 @@ spec: dnsPolicy: ClusterFirst initContainers: {{ tuple $envAll $dependencies $mounts_nova_compute_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + {{- if .Values.ceph.enabled }} + - name: ceph-keyring-placement + image: {{ .Values.images.compute }} + imagePullPolicy: {{ .Values.images.pull_policy }} + command: + - /tmp/ceph-keyring.sh + volumeMounts: + - name: etcceph + mountPath: /etc/ceph + - name: nova-bin + mountPath: /tmp/ceph-keyring.sh + subPath: ceph-keyring.sh + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + {{ end }} containers: - name: nova-compute image: {{ .Values.images.compute }} @@ -77,14 +94,18 @@ spec: mountPath: /etc/resolv.conf subPath: resolv.conf readOnly: true - - name: nova-etc + {{- if .Values.ceph.enabled }} + - name: etcceph + mountPath: /etc/ceph + - name: ceph-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true - - name: nova-etc - mountPath: /etc/ceph/ceph.client.keyring - subPath: ceph.client.keyring + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key readOnly: true + {{ end }} - mountPath: /lib/modules name: libmodules readOnly: true @@ -105,6 +126,16 @@ spec: - name: nova-etc configMap: name: nova-etc + {{- if .Values.ceph.enabled }} + - name: etcceph + emptyDir: {} + - name: ceph-etc + configMap: + name: ceph-etc + - name: ceph-keyring + secret: + secretName: pvc-ceph-client-key + {{ end }} - name: libmodules hostPath: path: /lib/modules diff --git a/nova/templates/daemonset-libvirt.yaml b/nova/templates/daemonset-libvirt.yaml index 32e4b7aba3..dce550c71e 100644 --- a/nova/templates/daemonset-libvirt.yaml +++ b/nova/templates/daemonset-libvirt.yaml @@ -38,6 +38,23 @@ spec: dnsPolicy: ClusterFirst initContainers: {{ tuple $envAll $dependencies $mounts_nova_libvirt_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + {{- if .Values.ceph.enabled }} + - name: ceph-keyring-placement + image: {{ .Values.images.libvirt }} + imagePullPolicy: {{ .Values.images.pull_policy }} + command: + - /tmp/ceph-keyring.sh + volumeMounts: + - name: etcceph + mountPath: /etc/ceph + - name: nova-bin + mountPath: /tmp/ceph-keyring.sh + subPath: ceph-keyring.sh + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + {{ end }} containers: - name: nova-libvirt image: {{ .Values.images.libvirt }} @@ -101,13 +118,15 @@ spec: - name: cgroup mountPath: /sys/fs/cgroup {{- if .Values.ceph.enabled }} - - name: nova-etc + - name: etcceph + mountPath: /etc/ceph + - name: ceph-etc mountPath: /etc/ceph/ceph.conf subPath: ceph.conf readOnly: true - - name: nova-etc - mountPath: /etc/ceph/ceph.client.keyring - subPath: ceph.client.keyring + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key readOnly: true - name: nova-bin mountPath: /tmp/ceph-secret-define.sh @@ -123,6 +142,16 @@ spec: - name: nova-etc configMap: name: nova-etc + {{- if .Values.ceph.enabled }} + - name: etcceph + emptyDir: {} + - name: ceph-etc + configMap: + name: ceph-etc + - name: ceph-keyring + secret: + secretName: pvc-ceph-client-key + {{ end }} - name: libmodules hostPath: path: /lib/modules diff --git a/nova/templates/etc/_ceph.conf.tpl b/nova/templates/etc/_ceph.conf.tpl deleted file mode 100644 index f8fbaf64a3..0000000000 --- a/nova/templates/etc/_ceph.conf.tpl +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[global] -rgw_thread_pool_size = 1024 -rgw_num_rados_handles = 100 -{{- if .Values.ceph.enabled }} -[mon] -{{- if .Values.ceph.monitors }} -{{ range .Values.ceph.monitors }} - [mon.{{ . }}] - host = {{ . }} - mon_addr = {{ . }} -{{ end }} -{{- else }} -mon_host = ceph-mon.ceph -{{- end }} -{{- end }} -[client] - rbd_cache_enabled = true - rbd_cache_writethrough_until_flush = true diff --git a/nova/values.yaml b/nova/values.yaml index df1e99ed98..b0239e30a1 100644 --- a/nova/values.yaml +++ b/nova/values.yaml @@ -130,7 +130,6 @@ keystone: admin_region_name: "RegionOne" nova_user_role: "admin" - ceph: enabled: false monitors: [] diff --git a/tests/pvc-test.yaml b/tests/pvc-test.yaml index 475d995ece..a448ecb63e 100644 --- a/tests/pvc-test.yaml +++ b/tests/pvc-test.yaml @@ -12,15 +12,42 @@ # See the License for the specific language governing permissions and # limitations under the License. +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + version: v0.1.0 + test: ceph + name: ceph-test-job +spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: test + image: docker.io/alpine:latest + imagePullPolicy: Always + command: + - /bin/sh + - -ec + - | + echo "Ceph PVC Mount Test Passed" + volumeMounts: + - name: ceph-mount + mountPath: /mnt/ceph + volumes: + - name: ceph-mount + persistentVolumeClaim: + claimName: ceph-test --- kind: PersistentVolumeClaim apiVersion: v1 metadata: - name: pvc-test + name: ceph-test spec: - accessModes: - - ReadWriteOnce + storageClassName: general + accessModes: [ "ReadWriteOnce" ] resources: requests: storage: 1Gi - storageClassName: general diff --git a/tools/gate/README.rst b/tools/gate/README.rst index a73422beb7..bdaed3b23e 100644 --- a/tools/gate/README.rst +++ b/tools/gate/README.rst @@ -17,6 +17,7 @@ integration test is below: export INTEGRATION=aio export INTEGRATION_TYPE=basic + export PVC_BACKEND=ceph ./tools/gate/setup_gate.sh Supported Platforms diff --git a/tools/gate/basic_launch.sh b/tools/gate/basic_launch.sh index 810c86ce1e..3a721bd19b 100755 --- a/tools/gate/basic_launch.sh +++ b/tools/gate/basic_launch.sh @@ -29,14 +29,53 @@ if [ "x$HOST_OS" == "xfedora" ]; then sudo modprobe ip6_tables fi +if [ "x$PVC_BACKEND" == "xceph" ]; then + kubectl label nodes ceph-storage=enabled --all + CONTROLLER_MANAGER_POD=$(kubectl get -n kube-system pods -l component=kube-controller-manager --no-headers -o name | head -1 | awk -F '/' '{ print $NF }') + kubectl exec -n kube-system ${CONTROLLER_MANAGER_POD} -- sh -c "cat > /etc/resolv.conf < \ + ${LOGS_DIR}/k8s/pvc/$NAMESPACE-$NAME.txt +done + mkdir -p ${LOGS_DIR}/k8s/rbac for OBJECT_TYPE in clusterroles \ roles \ @@ -76,5 +85,6 @@ sudo iptables-save > ${LOGS_DIR}/nodes/$(hostname)/iptables.txt sudo ip a > ${LOGS_DIR}/nodes/$(hostname)/ip.txt sudo route -n > ${LOGS_DIR}/nodes/$(hostname)/routes.txt arp -a > ${LOGS_DIR}/nodes/$(hostname)/arp.txt +cat /etc/resolv.conf > ${LOGS_DIR}/nodes/$(hostname)/resolv.conf exit $1 diff --git a/tools/gate/funcs/common.sh b/tools/gate/funcs/common.sh index 90176491e8..72d88d49f3 100644 --- a/tools/gate/funcs/common.sh +++ b/tools/gate/funcs/common.sh @@ -29,3 +29,17 @@ function base_install { iptables fi } + +function ceph_support_install { + if [ "x$HOST_OS" == "xubuntu" ]; then + sudo apt-get update -y + sudo apt-get install -y --no-install-recommends -qq \ + ceph-common + elif [ "x$HOST_OS" == "xcentos" ]; then + sudo yum install -y \ + ceph + elif [ "x$HOST_OS" == "xfedora" ]; then + sudo dnf install -y \ + ceph + fi +} diff --git a/tools/gate/funcs/kube.sh b/tools/gate/funcs/kube.sh index 75a445421d..900b49b15e 100755 --- a/tools/gate/funcs/kube.sh +++ b/tools/gate/funcs/kube.sh @@ -130,3 +130,9 @@ function kubeadm_aio_launch { kube_wait_for_pods kube-system 240 kube_wait_for_pods default 240 } + +function ceph_kube_controller_manager_replace { + sudo docker pull ${CEPH_KUBE_CONTROLLER_MANAGER_IMAGE} + IMAGE_ID=$(sudo docker images ${CEPH_KUBE_CONTROLLER_MANAGER_IMAGE} -q) + sudo docker tag ${IMAGE_ID} ${BASE_KUBE_CONTROLLER_MANAGER_IMAGE} +} diff --git a/tools/gate/funcs/network.sh b/tools/gate/funcs/network.sh index 85f724914c..1b1d0d9dfb 100755 --- a/tools/gate/funcs/network.sh +++ b/tools/gate/funcs/network.sh @@ -42,3 +42,4 @@ function net_hosts_pre_kube { function net_hosts_post_kube { sudo cp -f /etc/hosts-pre-kube /etc/hosts } + diff --git a/tools/gate/kubeadm_aio.sh b/tools/gate/kubeadm_aio.sh index 7dd8d4acb6..b60d5900dd 100755 --- a/tools/gate/kubeadm_aio.sh +++ b/tools/gate/kubeadm_aio.sh @@ -18,5 +18,11 @@ source ${WORK_DIR}/tools/gate/funcs/kube.sh kubeadm_aio_reqs_install sudo docker pull ${KUBEADM_IMAGE} || kubeadm_aio_build + +if [ "x$PVC_BACKEND" == "xceph" ]; then + ceph_kube_controller_manager_replace + sudo modprobe rbd +fi + kubeadm_aio_launch net_resolv_kube diff --git a/tools/gate/provision_gate_worker_node.sh b/tools/gate/provision_gate_worker_node.sh index 1f236163f7..e1cd5e4ee4 100755 --- a/tools/gate/provision_gate_worker_node.sh +++ b/tools/gate/provision_gate_worker_node.sh @@ -37,8 +37,15 @@ sudo rm -rfv \ /var/lib/etcd \ /var/etcd \ /var/lib/kubelet/* \ + /var/lib/nova \ + /var/lib/openstack-helm \ /run/openvswitch || true +# Load ceph kernel module if required +if [ "x$PVC_BACKEND" == "xceph" ]; then + sudo modprobe rbd +fi + # Launch Container sudo docker run \ -dt \ diff --git a/tools/gate/setup_gate.sh b/tools/gate/setup_gate.sh index e31dd17a9e..f9b72bf0d8 100755 --- a/tools/gate/setup_gate.sh +++ b/tools/gate/setup_gate.sh @@ -13,10 +13,12 @@ # limitations under the License. set -ex -export HELM_VERSION=${2:-v2.3.1} +export HELM_VERSION=${2:-v2.4.1} export KUBE_VERSION=${3:-v1.6.5} export KUBECONFIG=${HOME}/.kubeadm-aio/admin.conf -export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:${KUBE_VERSION} +export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:${KUBE_VERSION}-ceph +export BASE_KUBE_CONTROLLER_MANAGER_IMAGE=gcr.io/google_containers/kube-controller-manager-amd64:${KUBE_VERSION} +export CEPH_KUBE_CONTROLLER_MANAGER_IMAGE=quay.io/attcomdev/kube-controller-manager:${KUBE_VERSION} export WORK_DIR=$(pwd) source /etc/os-release @@ -24,6 +26,7 @@ export HOST_OS=${ID} source ${WORK_DIR}/tools/gate/funcs/common.sh source ${WORK_DIR}/tools/gate/funcs/network.sh source ${WORK_DIR}/tools/gate/funcs/helm.sh +export PVC_BACKEND=ceph # Setup the logging location: by default use the working dir as the root. export LOGS_DIR=${LOGS_DIR:-"${WORK_DIR}/logs"} @@ -34,14 +37,17 @@ function dump_logs () { } trap 'dump_logs "$?"' ERR -# Install base requirements -base_install - # Moving the ws-linter here to avoid it blocking all the jobs just for ws if [ "x$INTEGRATION_TYPE" == "xlinter" ]; then bash ${WORK_DIR}/tools/gate/whitespace.sh fi +# Install base requirements +base_install +if [ "x$PVC_BACKEND" == "xceph" ]; then + ceph_support_install +fi + # We setup the network for pre kube here, to enable cluster restarts on # development machines net_resolv_pre_kube diff --git a/tools/gate/setup_gate_worker_nodes.sh b/tools/gate/setup_gate_worker_nodes.sh index da6c6683f0..91db2904b7 100755 --- a/tools/gate/setup_gate_worker_nodes.sh +++ b/tools/gate/setup_gate_worker_nodes.sh @@ -31,6 +31,7 @@ cat /etc/nodepool/sub_nodes_private | while read SUB_NODE; do export KUBEADM_TOKEN=${KUBEADM_TOKEN}; \ export PRIMARY_NODE_IP=${PRIMARY_NODE_IP}; \ export KUBEADM_IMAGE=${KUBEADM_IMAGE}; \ + export PVC_BACKEND=${PVC_BACKEND}; \ bash ${WORK_DIR}/tools/gate/provision_gate_worker_node.sh" EOS done diff --git a/tools/images/ceph-config-helper/Dockerfile b/tools/images/ceph-config-helper/Dockerfile new file mode 100644 index 0000000000..f310618847 --- /dev/null +++ b/tools/images/ceph-config-helper/Dockerfile @@ -0,0 +1,20 @@ +FROM ubuntu:16.04 +MAINTAINER pete.birley@att.com + +ARG KUBE_VERSION=v1.6.5 + +RUN set -x \ + && TMP_DIR=$(mktemp --directory) \ + && cd ${TMP_DIR} \ + && apt-get update \ + && apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + python \ + jq \ +# Install kubectl: + && curl -sSL https://dl.k8s.io/${KUBE_VERSION}/kubernetes-client-linux-amd64.tar.gz | tar -zxv --strip-components=1 \ + && mv ${TMP_DIR}/client/bin/kubectl /usr/bin/kubectl \ + && chmod +x /usr/bin/kubectl \ + && rm -rf ${TMP_DIR} diff --git a/tools/images/ceph-config-helper/README.rst b/tools/images/ceph-config-helper/README.rst new file mode 100644 index 0000000000..7b7196b368 --- /dev/null +++ b/tools/images/ceph-config-helper/README.rst @@ -0,0 +1,39 @@ +Ceph Config Helper Container +===================== + +This container builds a small image with kubectl and some other utilites for +use in the ceph-config chart. + +Instructions +------------ + +OS Specific Host setup: +~~~~~~~~~~~~~~~~~~~~~~~ + +Ubuntu: +^^^^^^^aa + +From a freshly provisioned Ubuntu 16.04 LTS host run: + +.. code:: bash + + sudo apt-get update -y + sudo apt-get install -y \ + docker.io \ + git + +Build the Ceph-Helper Image environment (optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A known good image is published to dockerhub on a fairly regular basis, but if +you wish to build your own image, from the root directory of the OpenStack-Helm +repo run: + +.. code:: bash + + export KUBE_VERSION=v1.6.5 + sudo docker build \ + --build-arg KUBE_VERSION=${KUBE_VERSION} \ + -t docker.io/port/ceph-config-helper:${KUBE_VERSION} \ + tools/images/ceph-config-helper + sudo docker push docker.io/port/ceph-config-helper:${KUBE_VERSION} diff --git a/tools/kubeadm-aio/Dockerfile b/tools/kubeadm-aio/Dockerfile index 90e9d32ea2..ef14dfc670 100644 --- a/tools/kubeadm-aio/Dockerfile +++ b/tools/kubeadm-aio/Dockerfile @@ -61,8 +61,9 @@ RUN set -x \ git \ vim \ jq \ -# Install nfs utils for development PVC provisioner +# Install utils for PVC provisioners nfs-common \ + ceph-common \ # Tweak Systemd units and targets for running in a container && find /lib/systemd/system/sysinit.target.wants/ ! -name 'systemd-tmpfiles-setup.service' -type l -exec rm -fv {} + \ && rm -fv \ diff --git a/tools/kubeadm-aio/README.rst b/tools/kubeadm-aio/README.rst index dd9ac2ce18..f2ba595249 100644 --- a/tools/kubeadm-aio/README.rst +++ b/tools/kubeadm-aio/README.rst @@ -57,7 +57,7 @@ repo run: .. code:: bash - export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:v1.6.4 + export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:v1.6.5 sudo docker build --pull -t ${KUBEADM_IMAGE} tools/kubeadm-aio Deploy the AIO environment diff --git a/tools/kubeadm-aio/assets/usr/bin/kubelet b/tools/kubeadm-aio/assets/usr/bin/kubelet index 0972ffc339..a851686625 100755 --- a/tools/kubeadm-aio/assets/usr/bin/kubelet +++ b/tools/kubeadm-aio/assets/usr/bin/kubelet @@ -36,7 +36,8 @@ else --restart=always \ --volume=/sys/fs/cgroup:/sys/fs/cgroup:ro \ --volume=/:/rootfs:ro \ - --volume=/dev/net:/dev/net:rw \ + --volume=/dev:/dev:rshared \ + --volume=/lib/modules:/lib/modules:ro \ --volume=/var/run/netns:/var/run/netns:rw \ --volume=/sys:/sys:ro \ --volume=/etc/machine-id:/etc/machine-id:ro \ diff --git a/tools/kubeadm-aio/kubeadm-aio-launcher.sh b/tools/kubeadm-aio/kubeadm-aio-launcher.sh index d9bab230e9..1a9224befa 100755 --- a/tools/kubeadm-aio/kubeadm-aio-launcher.sh +++ b/tools/kubeadm-aio/kubeadm-aio-launcher.sh @@ -33,6 +33,7 @@ sudo rm -rfv \ /run/openvswitch \ /var/lib/nova \ ${HOME}/.kubeadm-aio/admin.conf \ + /var/lib/openstack-helm \ /var/lib/nfs-provisioner || true # Launch Container diff --git a/helm-toolkit/utils/secret-generator/ceph-key.py b/tools/overrides/mvp/cinder.yaml similarity index 58% rename from helm-toolkit/utils/secret-generator/ceph-key.py rename to tools/overrides/mvp/cinder.yaml index 761f41a86f..bc44c55300 100644 --- a/helm-toolkit/utils/secret-generator/ceph-key.py +++ b/tools/overrides/mvp/cinder.yaml @@ -1,5 +1,3 @@ -#!/bin/python - # Copyright 2017 The Openstack-Helm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,17 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -import struct -import time -import base64 +# MVP values for glance. +# This file contains overrides to launch a MVP deployment of glance for the +# OpenStack-Helm Single node gates, and local development use. It should be +# kept to the bare minimum required for this purpose. + +storage: "gate" + +conf: + cinder: + default: + cinder: + backup_driver: "cinder.backup.drivers.nfs" + backends: + rbd1: + volume_driver: "cinder.volume.drivers.nfs.NfsDriver" -key = os.urandom(16) -header = struct.pack( - '