Refactor Ceph secret generation

This PS refactors the ceph chart and secret generation process.
The updated chart replaces the existing "bootstrap" chart.
Additionally, Ceph manifests and deployment guides were modified
accordingly.

Change-Id: I6f5bb88fc0f40cfee8865d9dab83859d765e7537
Co-Authored-By: Larry Rensing <lr699s@att.com>

Makefile

@@ -15,8 +15,8 @@
 HELM = helm
 TASK = build
 
-CHARTS = helm-toolkit bootstrap ceph mariadb etcd rabbitmq memcached
+CHARTS = helm-toolkit ceph mariadb etcd rabbitmq
-CHARTS += keystone glance cinder horizon neutron nova heat
+CHARTS += memcached keystone glance cinder horizon neutron nova heat
 CHARTS += barbican mistral senlin magnum ingress
 
 all: $(CHARTS)

bootstrap/.gitignore

@@ -1,3 +0,0 @@
-secrets/*
-!secrets/.gitkeep 
-templates/_secrets.tpl

bootstrap/.helmignore

@@ -1,27 +0,0 @@
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-
-bin/
-etc/
-patches/
-*.py
-Makefile

bootstrap/Chart.yaml

@@ -1,18 +0,0 @@
-# Copyright 2017 The Openstack-Helm Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-apiVersion: v1
-description: OpenStack-Helm namespace bootstrap
-name: bootstrap
-version: 0.1.0

bootstrap/requirements.yaml

@@ -1,18 +0,0 @@
-# Copyright 2017 The Openstack-Helm Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-dependencies:
-  - name: helm-toolkit
-    repository: http://localhost:8879/charts
-    version: 0.1.0

bootstrap/values.yaml

@@ -1,18 +0,0 @@
-# Copyright 2017 The Openstack-Helm Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Default values for bootstrap.
-# This is a YAML-formatted file.
-# Declare name/value pairs to be passed into your templates.
-# name: value

ceph/templates/bin/_ceph-key.py.tpl

@@ -0,0 +1,14 @@
+#!/bin/python
+import os
+import struct
+import time
+import base64
+key = os.urandom(16)
+header = struct.pack(
+    '<hiih',
+    1,                 # le16 type: CEPH_CRYPTO_AES
+    int(time.time()),  # le32 created: seconds
+    0,                 # le32 created: nanoseconds,
+    len(key),          # le16: len(key)
+)
+print(base64.b64encode(header + key).decode('ascii'))

ceph/templates/bin/_ceph-key.sh.tpl

@@ -0,0 +1,37 @@
+#!/bin/bash
+set -ex
+
+function ceph_gen_key () {
+  python ${CEPH_GEN_DIR}/ceph-key.py
+}
+
+function kube_ceph_keyring_gen () {
+  CEPH_KEY=$1
+  CEPH_KEY_TEMPLATE=$2
+  sed "s|{{"{{"}} key {{"}}"}}|${CEPH_KEY}|" ${CEPH_TEMPLATES_DIR}/${CEPH_KEY_TEMPLATE} | base64 | tr -d '\n'
+}
+
+function create_kube_key () {
+  CEPH_KEYRING=$1
+  CEPH_KEYRING_NAME=$2
+  CEPH_KEYRING_TEMPLATE=$3
+  KUBE_SECRET_NAME=$4
+  if ! kubectl get --namespace ${DEPLOYMENT_NAMESPACE} secrets ${KUBE_SECRET_NAME}; then
+    {
+      cat <<EOF
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ${KUBE_SECRET_NAME}
+type: Opaque
+data:
+  ${CEPH_KEYRING_NAME}: |
+    $( kube_ceph_keyring_gen ${CEPH_KEYRING} ${CEPH_KEYRING_TEMPLATE} )
+EOF
+    } | kubectl create --namespace ${DEPLOYMENT_NAMESPACE} -f -
+  fi
+}
+
+#create_kube_key <ceph_key> <ceph_keyring_name> <ceph_keyring_template> <kube_secret_name>
+create_kube_key $(ceph_gen_key) ${CEPH_KEYRING_NAME} ${CEPH_KEYRING_TEMPLATE} ${KUBE_SECRET_NAME}

ceph/templates/bin/_ceph-namespace-client-key.sh.tpl

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -ex
+
+ceph_activate_namespace() {
+  kube_namespace=$1
+  {
+  cat <<EOF
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "${PVC_CEPH_STORAGECLASS_USER_SECRET_NAME}"
+type: kubernetes.io/rbd
+data:
+  key: |
+    $(kubectl get secret ${PVC_CEPH_STORAGECLASS_ADMIN_SECRET_NAME} \
+        --namespace=${PVC_CEPH_STORAGECLASS_DEPLOYED_NAMESPACE} \
+        -o json | jq -r '.data | .[]')
+EOF
+  } | kubectl create --namespace ${kube_namespace} -f -
+}
+
+ceph_activate_namespace ${DEPLOYMENT_NAMESPACE}

ceph/templates/bin/_ceph-storage-key.sh.tpl

@@ -0,0 +1,62 @@
+#!/bin/bash
+set -ex
+
+function ceph_gen_key () {
+  python ${CEPH_GEN_DIR}/ceph-key.py
+}
+
+function kube_ceph_keyring_gen () {
+  CEPH_KEY=$1
+  CEPH_KEY_TEMPLATE=$2
+  sed "s|{{"{{"}} key {{"}}"}}|${CEPH_KEY}|" ${CEPH_TEMPLATES_DIR}/${CEPH_KEY_TEMPLATE} | base64 | tr -d '\n'
+}
+
+CEPH_CLIENT_KEY=$(ceph_gen_key)
+
+function create_kube_key () {
+  CEPH_KEYRING=$1
+  CEPH_KEYRING_NAME=$2
+  CEPH_KEYRING_TEMPLATE=$3
+  KUBE_SECRET_NAME=$4
+
+  if ! kubectl get --namespace ${DEPLOYMENT_NAMESPACE} secrets ${KUBE_SECRET_NAME}; then
+    {
+      cat <<EOF
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ${KUBE_SECRET_NAME}
+type: Opaque
+data:
+  ${CEPH_KEYRING_NAME}: |
+    $( kube_ceph_keyring_gen ${CEPH_KEYRING} ${CEPH_KEYRING_TEMPLATE} )
+EOF
+    } | kubectl create --namespace ${DEPLOYMENT_NAMESPACE} -f -
+  fi
+}
+#create_kube_key <ceph_key> <ceph_keyring_name> <ceph_keyring_template> <kube_secret_name>
+create_kube_key ${CEPH_CLIENT_KEY} ${CEPH_KEYRING_NAME} ${CEPH_KEYRING_TEMPLATE} ${CEPH_KEYRING_ADMIN_NAME}
+
+function create_kube_storage_key () {
+  CEPH_KEYRING=$1
+  KUBE_SECRET_NAME=$2
+
+  if ! kubectl get --namespace ${DEPLOYMENT_NAMESPACE} secrets ${KUBE_SECRET_NAME}; then
+    {
+      cat <<EOF
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ${KUBE_SECRET_NAME}
+type: kubernetes.io/rbd
+data:
+  key: |
+    $( echo ${CEPH_KEYRING} | base64 | tr -d '\n' )
+EOF
+    } | kubectl create --namespace ${DEPLOYMENT_NAMESPACE} -f -
+  fi
+}
+#create_kube_storage_key <ceph_key> <kube_secret_name>
+create_kube_storage_key ${CEPH_CLIENT_KEY} ${CEPH_STORAGECLASS_ADMIN_SECRET_NAME}

ceph/templates/configmap-bin.yaml

@@ -0,0 +1,31 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ceph-bin
+data:
+{{- if .Values.manifests_enabled.storage_secrets }}
+  ceph-key.py: |+
+{{ tuple "bin/_ceph-key.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  ceph-key.sh: |+
+{{ tuple "bin/_ceph-key.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  ceph-storage-key.sh: |+
+{{ tuple "bin/_ceph-storage-key.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}
+{{- if .Values.manifests_enabled.client_secrets }}
+  ceph-namespace-client-key.sh: |+
+{{ tuple "bin/_ceph-namespace-client-key.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}

ceph/templates/configmap-etc.yaml

@@ -12,21 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+{{- if empty .Values.conf.ceph.config.global.mon_host -}}
+{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_endpoint_lookup" }}
+{{- $monHostDomain := default .Release.Namespace .Values.ceph.namespace }}
+{{- $monHostURI := cat $monHost "." $monHostDomain | nospace -}}
+{{- $monHostURI |  set .Values.conf.ceph.config.global "mon_host" | quote | trunc 0 -}}
+{{- end -}}
+
 ---
 apiVersion: v1
-kind: Secret
+kind: ConfigMap
 metadata:
-  name: "pvc-ceph-conf-combined-storageclass"
+  name: ceph-etc
-type: kubernetes.io/rbd
 data:
-  key: |
+  ceph.conf: |+
-{{ include "secrets/ceph-client-key" . | b64enc | indent 4 }}
+{{ tuple "etc/_ceph.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: "pvc-ceph-client-key"
-type: kubernetes.io/rbd
-data:
-  key: |
-{{ include "secrets/ceph-client-key" . | b64enc | indent 4 }}

ceph/templates/configmap-templates.yaml

@@ -0,0 +1,31 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+{{- if .Values.manifests_enabled.storage_secrets }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ceph-templates
+data:
+  admin.keyring: |+
+{{ tuple "templates/_admin.keyring.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  bootstrap.keyring.mds: |+
+{{ tuple "templates/_bootstrap.keyring.mds.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  bootstrap.keyring.osd: |+
+{{ tuple "templates/_bootstrap.keyring.osd.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  bootstrap.keyring.rgw: |+
+{{ tuple "templates/_bootstrap.keyring.rgw.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  mon.keyring: |+
+{{ tuple "templates/_mon.keyring.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+{{- end }}

ceph/templates/daemonset-osd.yaml

@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+{{- if .Values.manifests_enabled.deployment }}
+{{- $envAll := . }}
+{{- $dependencies := .Values.dependencies.osd }}
 ---
 kind: DaemonSet
 apiVersion: extensions/v1beta1
@@ -29,47 +32,21 @@ spec:
     spec:
       nodeSelector:
         {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
-      volumes:
+      initContainers:
-        - name: devices
+{{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
-          hostPath:
-            path: /dev
-        - name: ceph
-          hostPath:
-            path: {{ .Values.storage.var_directory }}
-        - name: ceph-conf
-          secret:
-            secretName: ceph-conf-combined
-        - name: ceph-bootstrap-osd-keyring
-          secret:
-            secretName: ceph-bootstrap-osd-keyring
-        - name: ceph-bootstrap-mds-keyring
-          secret:
-            secretName: ceph-bootstrap-mds-keyring
-        - name: ceph-bootstrap-rgw-keyring
-          secret:
-            secretName: ceph-bootstrap-rgw-keyring
-        - name: osd-directory
-          hostPath:
-            path: {{ .Values.storage.osd_directory }}
       containers:
         - name: osd-pod
           image: {{ .Values.images.daemon }}
           imagePullPolicy: {{ .Values.images.pull_policy }}
-          volumeMounts:
+          {{- if .Values.resources.enabled }}
-            - name: devices
+          resources:
-              mountPath: /dev
+            requests:
-            - name: ceph
+              memory: {{ .Values.resources.osd.requests.memory | quote }}
-              mountPath: /var/lib/ceph
+              cpu: {{ .Values.resources.osd.requests.cpu | quote }}
-            - name: ceph-conf
+            limits:
-              mountPath: /etc/ceph
+              memory: {{ .Values.resources.osd.limits.memory | quote }}
-            - name: ceph-bootstrap-osd-keyring
+              cpu: {{ .Values.resources.osd.limits.cpu | quote }}
-              mountPath: /var/lib/ceph/bootstrap-osd
+          {{- end }}
-            - name: ceph-bootstrap-mds-keyring
-              mountPath: /var/lib/ceph/bootstrap-mds
-            - name: ceph-bootstrap-rgw-keyring
-              mountPath: /var/lib/ceph/bootstrap-rgw
-            - name: osd-directory
-              mountPath: /var/lib/ceph/osd
           securityContext:
             privileged: true
           env:
@@ -81,6 +58,10 @@ spec:
               value: ceph
             - name: CEPH_GET_ADMIN_KEY
               value: "1"
+          command:
+            - /entrypoint.sh
+          ports:
+            - containerPort: 6800
           livenessProbe:
               tcpSocket:
                 port: 6800
@@ -90,10 +71,65 @@ spec:
               tcpSocket:
                 port: 6800
               timeoutSeconds: 5
-          resources:
+          volumeMounts:
-            requests:
+            - name: devices
-              memory: {{ .Values.resources.osd.requests.memory | quote }}
+              mountPath: /dev
-              cpu: {{ .Values.resources.osd.requests.cpu | quote }}
+              readOnly: false
-            limits:
+            - name: ceph
-              memory: {{ .Values.resources.osd.limits.memory | quote }}
+              mountPath: /var/lib/ceph
-              cpu: {{ .Values.resources.osd.limits.cpu | quote }}
+              readOnly: false
+            - name: ceph-etc
+              mountPath: /etc/ceph/ceph.conf
+              subPath: ceph.conf
+              readOnly: true
+            - name: ceph-client-admin-keyring
+              mountPath: /etc/ceph/ceph.client.admin.keyring
+              subPath: ceph.client.admin.keyring
+              readOnly: false
+            - name: ceph-mon-keyring
+              mountPath: /etc/ceph/ceph.mon.keyring
+              subPath: ceph.mon.keyring
+              readOnly: false
+            - name: ceph-bootstrap-osd-keyring
+              mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring
+              subPath: ceph.keyring
+              readOnly: false
+            - name: ceph-bootstrap-mds-keyring
+              mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring
+              subPath: ceph.keyring
+              readOnly: false
+            - name: ceph-bootstrap-rgw-keyring
+              mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring
+              subPath: ceph.keyring
+              readOnly: false
+            - name: osd-directory
+              mountPath: /var/lib/ceph/osd
+      volumes:
+        - name: devices
+          hostPath:
+            path: /dev
+        - name: ceph
+          hostPath:
+            path: {{ .Values.ceph.storage.var_directory }}
+        - name: ceph-etc
+          configMap:
+            name: ceph-etc
+        - name: ceph-client-admin-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.admin }}
+        - name: ceph-mon-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.mon }}
+        - name: ceph-bootstrap-osd-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.osd }}
+        - name: ceph-bootstrap-mds-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.mds }}
+        - name: ceph-bootstrap-rgw-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.rgw }}
+        - name: osd-directory
+          hostPath:
+            path: {{ .Values.ceph.storage.osd_directory }}
+{{- end }}

ceph/templates/deployment-mds.yaml

@@ -12,7 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
----
+{{- if .Values.manifests_enabled.deployment }}
+{{- if .Values.ceph.enabled.mds }}
+{{- $envAll := . }}
+{{- $dependencies := .Values.dependencies.mds }}
 kind: Deployment
 apiVersion: apps/v1beta1
 metadata:
@@ -31,24 +34,22 @@ spec:
     spec:
       nodeSelector:
         {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
+      initContainers:
+{{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
       serviceAccount: default
-      volumes:
-        - name: ceph-conf
-          secret:
-            secretName: ceph-conf-combined
-        - name: ceph-bootstrap-osd-keyring
-          secret:
-            secretName: ceph-bootstrap-osd-keyring
-        - name: ceph-bootstrap-mds-keyring
-          secret:
-            secretName: ceph-bootstrap-mds-keyring
-        - name: ceph-bootstrap-rgw-keyring
-          secret:
-            secretName: ceph-bootstrap-rgw-keyring
       containers:
         - name: ceph-mds
           image: {{ .Values.images.daemon }}
           imagePullPolicy: {{ .Values.images.pull_policy }}
+          {{- if .Values.resources.enabled }}
+          resources:
+            requests:
+              memory: {{ .Values.resources.mds.requests.memory | quote }}
+              cpu: {{ .Values.resources.mds.requests.cpu | quote }}
+            limits:
+              memory: {{ .Values.resources.mds.limits.memory | quote }}
+              cpu: {{ .Values.resources.mds.limits.cpu | quote }}
+          {{- end }}
           ports:
             - containerPort: 6800
           env:
@@ -60,15 +61,33 @@ spec:
               value: k8s
             - name: CLUSTER
               value: ceph
+          command:
+            - /entrypoint.sh
           volumeMounts:
-            - name: ceph-conf
+            - name: ceph-etc
-              mountPath: /etc/ceph
+              mountPath: /etc/ceph/ceph.conf
+              subPath: ceph.conf
+              readOnly: true
+            - name: ceph-client-admin-keyring
+              mountPath: /etc/ceph/ceph.client.admin.keyring
+              subPath: ceph.client.admin.keyring
+              readOnly: true
+            - name: ceph-mon-keyring
+              mountPath: /etc/ceph/ceph.mon.keyring
+              subPath: ceph.mon.keyring
+              readOnly: true
             - name: ceph-bootstrap-osd-keyring
-              mountPath: /var/lib/ceph/bootstrap-osd
+              mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring
+              subPath: ceph.keyring
+              readOnly: false
             - name: ceph-bootstrap-mds-keyring
-              mountPath: /var/lib/ceph/bootstrap-mds
+              mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring
+              subPath: ceph.keyring
+              readOnly: false
             - name: ceph-bootstrap-rgw-keyring
-              mountPath: /var/lib/ceph/bootstrap-rgw
+              mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring
+              subPath: ceph.keyring
+              readOnly: false
           livenessProbe:
               tcpSocket:
                 port: 6800
@@ -78,10 +97,24 @@ spec:
               tcpSocket:
                 port: 6800
               timeoutSeconds: 5
-          resources:
+      volumes:
-            requests:
+        - name: ceph-etc
-              memory: {{ .Values.resources.mds.requests.memory | quote }}
+          configMap:
-              cpu: {{ .Values.resources.mds.requests.cpu | quote }}
+            name: ceph-etc
-            limits:
+        - name: ceph-client-admin-keyring
-              memory: {{ .Values.resources.mds.limits.memory | quote }}
+          secret:
-              cpu: {{ .Values.resources.mds.limits.cpu | quote }}
+            secretName: {{ .Values.secrets.keyrings.admin }}
+        - name: ceph-mon-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.mon }}
+        - name: ceph-bootstrap-osd-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.osd }}
+        - name: ceph-bootstrap-mds-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.mds }}
+        - name: ceph-bootstrap-rgw-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.rgw }}
+{{- end }}
+{{- end }}

ceph/templates/deployment-moncheck.yaml

@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+{{- if .Values.manifests_enabled.deployment }}
+{{- $envAll := . }}
+{{- $dependencies := .Values.dependencies.moncheck }}
 ---
 kind: Deployment
 apiVersion: apps/v1beta1
@@ -31,24 +34,22 @@ spec:
     spec:
       nodeSelector:
         {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
+      initContainers:
+{{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
       serviceAccount: default
-      volumes:
-        - name: ceph-conf
-          secret:
-            secretName: ceph-conf-combined
-        - name: ceph-bootstrap-osd-keyring
-          secret:
-            secretName: ceph-bootstrap-osd-keyring
-        - name: ceph-bootstrap-mds-keyring
-          secret:
-            secretName: ceph-bootstrap-mds-keyring
-        - name: ceph-bootstrap-rgw-keyring
-          secret:
-            secretName: ceph-bootstrap-rgw-keyring
       containers:
         - name: ceph-mon
           image: {{ .Values.images.daemon }}
           imagePullPolicy: {{ .Values.images.pull_policy }}
+          {{- if .Values.resources.enabled }}
+          resources:
+            requests:
+              memory: {{ .Values.resources.mon_check.requests.memory | quote }}
+              cpu: {{ .Values.resources.mon_check.requests.cpu | quote }}
+            limits:
+              memory: {{ .Values.resources.mon_check.limits.memory | quote }}
+              cpu: {{ .Values.resources.mon_check.limits.cpu | quote }}
+          {{- end }}
           ports:
             - containerPort: 6789
           env:
@@ -60,19 +61,50 @@ spec:
               value: "1"
             - name: CLUSTER
               value: ceph
+          command:
+            - /entrypoint.sh
           volumeMounts:
-            - name: ceph-conf
+            - name: ceph-etc
-              mountPath: /etc/ceph
+              mountPath: /etc/ceph/ceph.conf
+              subPath: ceph.conf
+              readOnly: true
+            - name: ceph-client-admin-keyring
+              mountPath: /etc/ceph/ceph.client.admin.keyring
+              subPath: ceph.client.admin.keyring
+              readOnly: true
+            - name: ceph-mon-keyring
+              mountPath: /etc/ceph/ceph.mon.keyring
+              subPath: ceph.mon.keyring
+              readOnly: true
             - name: ceph-bootstrap-osd-keyring
-              mountPath: /var/lib/ceph/bootstrap-osd
+              mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring
+              subPath: ceph.keyring
+              readOnly: false
             - name: ceph-bootstrap-mds-keyring
-              mountPath: /var/lib/ceph/bootstrap-mds
+              mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring
+              subPath: ceph.keyring
+              readOnly: false
             - name: ceph-bootstrap-rgw-keyring
-              mountPath: /var/lib/ceph/bootstrap-rgw
+              mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring
-          resources:
+              subPath: ceph.keyring
-            requests:
+              readOnly: false
-              memory: {{ .Values.resources.mon_check.requests.memory | quote }}
+      volumes:
-              cpu: {{ .Values.resources.mon_check.requests.cpu | quote }}
+        - name: ceph-etc
-            limits:
+          configMap:
-              memory: {{ .Values.resources.mon_check.limits.memory | quote }}
+            name: ceph-etc
-              cpu: {{ .Values.resources.mon_check.limits.cpu | quote }}
+        - name: ceph-client-admin-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.admin }}
+        - name: ceph-mon-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.mon }}
+        - name: ceph-bootstrap-osd-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.osd }}
+        - name: ceph-bootstrap-mds-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.mds }}
+        - name: ceph-bootstrap-rgw-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.rgw }}
+{{- end }}

ceph/templates/deployment-rgw.yaml

@@ -12,7 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-{{- if .Values.rgw.enabled }}
+{{- if .Values.manifests_enabled.deployment }}
+{{- if .Values.ceph.enabled.rgw }}
+{{- $envAll := . }}
+{{- $dependencies := .Values.dependencies.rgw }}
 ---
 kind: Deployment
 apiVersion: apps/v1beta1
@@ -32,24 +35,22 @@ spec:
     spec:
       nodeSelector:
         {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
+              initContainers:
+{{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
       serviceAccount: default
-      volumes:
-        - name: ceph-conf
-          secret:
-            secretName: ceph-conf-combined
-        - name: ceph-bootstrap-osd-keyring
-          secret:
-            secretName: ceph-bootstrap-osd-keyring
-        - name: ceph-bootstrap-mds-keyring
-          secret:
-            secretName: ceph-bootstrap-mds-keyring
-        - name: ceph-bootstrap-rgw-keyring
-          secret:
-            secretName: ceph-bootstrap-rgw-keyring
       containers:
         - name: ceph-rgw
           image: {{ .Values.images.daemon }}
           imagePullPolicy: {{ .Values.images.pull_policy }}
+          {{- if .Values.resources.enabled }}
+          resources:
+            requests:
+              memory: {{ .Values.resources.rgw.requests.memory | quote }}
+              cpu: {{ .Values.resources.rgw.requests.cpu | quote }}
+            limits:
+              memory: {{ .Values.resources.rgw.limits.memory | quote }}
+              cpu: {{ .Values.resources.rgw.limits.cpu | quote }}
+          {{- end }}
           ports:
             - containerPort: {{ .Values.network.port.rgw_target }}
           env:
@@ -61,15 +62,33 @@ spec:
               value: k8s
             - name: CLUSTER
               value: ceph
+          command:
+            - /entrypoint.sh
           volumeMounts:
-            - name: ceph-conf
+            - name: ceph-etc
-              mountPath: /etc/ceph
+              mountPath: /etc/ceph/ceph.conf
+              subPath: ceph.conf
+              readOnly: true
+            - name: ceph-client-admin-keyring
+              mountPath: /etc/ceph/ceph.client.admin.keyring
+              subPath: ceph.client.admin.keyring
+              readOnly: true
+            - name: ceph-mon-keyring
+              mountPath: /etc/ceph/ceph.mon.keyring
+              subPath: ceph.mon.keyring
+              readOnly: true
             - name: ceph-bootstrap-osd-keyring
-              mountPath: /var/lib/ceph/bootstrap-osd
+              mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring
+              subPath: ceph.keyring
+              readOnly: false
             - name: ceph-bootstrap-mds-keyring
-              mountPath: /var/lib/ceph/bootstrap-mds
+              mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring
+              subPath: ceph.keyring
+              readOnly: false
             - name: ceph-bootstrap-rgw-keyring
-              mountPath: /var/lib/ceph/bootstrap-rgw
+              mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring
+              subPath: ceph.keyring
+              readOnly: false
           livenessProbe:
               httpGet:
                 path: /
@@ -81,11 +100,24 @@ spec:
                 path: /
                 port: {{ .Values.network.port.rgw_target }}
               timeoutSeconds: 5
-          resources:
+      volumes:
-            requests:
+        - name: ceph-etc
-              memory: {{ .Values.resources.rgw.requests.memory | quote }}
+          configMap:
-              cpu: {{ .Values.resources.rgw.requests.cpu | quote }}
+            name: ceph-etc
-            limits:
+        - name: ceph-client-admin-keyring
-              memory: {{ .Values.resources.rgw.limits.memory | quote }}
+          secret:
-              cpu: {{ .Values.resources.rgw.limits.cpu | quote }}
+            secretName: {{ .Values.secrets.keyrings.admin }}
+        - name: ceph-mon-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.mon }}
+        - name: ceph-bootstrap-osd-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.osd }}
+        - name: ceph-bootstrap-mds-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.mds }}
+        - name: ceph-bootstrap-rgw-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.rgw }}
+{{- end }}
 {{- end }}

ceph/templates/etc/_ceph.conf.tpl

@@ -0,0 +1,77 @@
+[global]
+fsid = {{ uuidv4 | default .Values.conf.ceph.config.global.uuid | quote }}
+cephx = {{ .Values.conf.ceph.config.global.cephx | default "true" | quote }}
+cephx_require_signatures = {{ .Values.conf.ceph.config.global.cephx_require_signatures | default "false" | quote }}
+cephx_cluster_require_signatures = {{ .Values.conf.ceph.config.global.cephx_cluster_require_signatures | default "true" | quote }}
+cephx_service_require_signatures = {{ .Values.conf.ceph.config.global.cephx_service_require_signatures | default "false" | quote }}
+
+# auth
+max_open_files = {{ .Values.conf.ceph.config.global.max_open_files | default "131072" | quote }}
+
+osd_pool_default_pg_num = {{ .Values.conf.ceph.config.global.osd_pool_default_pg_num | default "128" | quote }}
+osd_pool_default_pgp_num = {{ .Values.conf.ceph.config.global.osd_pool_default_pgp_num | default "128" | quote }}
+osd_pool_default_size = {{ .Values.conf.ceph.config.global.osd_pool_default_size | default "3" | quote }}
+osd_pool_default_min_size = {{ .Values.conf.ceph.config.global.osd_pool_default_min_size | default "1" | quote }}
+
+mon_osd_full_ratio = {{ .Values.conf.ceph.config.global.mon_osd_full_ratio | default ".95" | quote }}
+mon_osd_nearfull_ratio = {{ .Values.conf.ceph.config.global.mon_osd_nearfull_ratio | default ".85" | quote }}
+mon_host = {{ .Values.conf.ceph.config.global.mon_host | quote }}
+
+rgw_thread_pool_size = {{ .Values.conf.ceph.config.global.rgw_thread_pool_size | default "1024" | quote }}
+rgw_num_rados_handles = {{ .Values.conf.ceph.config.global.rgw_num_rados_handles | default "100" | quote }}
+
+[mon]
+mon_osd_down_out_interval = {{ .Values.conf.ceph.config.mon.mon_osd_down_out_interval | default "600" | quote }}
+mon_osd_min_down_reporters = {{ .Values.conf.ceph.config.mon.mon_osd_min_down_reporters | default "4" | quote }}
+mon_clock_drift_allowed = {{ .Values.conf.ceph.config.mon.mon_clock_drift_allowed | default "0.15" | quote }}
+mon_clock_drift_warn_backoff = {{ .Values.conf.ceph.config.mon.mon_clock_drift_warn_backoff | default "30" | quote }}
+mon_osd_report_timeout = {{ .Values.conf.ceph.config.mon.mon_osd_report_timeout | default "300" | quote }}
+
+[osd]
+# network
+cluster_network = {{ .Values.network.cluster | default "192.168.0.0/16" | quote }}
+public_network = {{ .Values.network.public | default "192.168.0.0/16" | quote }}
+osd_mon_heartbeat_interval = {{ .Values.conf.ceph.config.osd.osd_mon_heartbeat_interval | default "30" | quote }}
+
+# ports
+ms_bind_port_min = {{ .Values.conf.ceph.config.osd.ms_bind_port_min | default "6800" | quote }}
+ms_bind_port_max = {{ .Values.conf.ceph.config.osd.ms_bind_port_max | default "7100" | quote }}
+
+# journal
+journal_size = {{ .Values.conf.ceph.config.osd.journal_size | default "100" | quote }}
+
+# filesystem
+osd_mkfs_type = {{ .Values.conf.ceph.config.osd.osd_mkfs_type | default "xfs" | quote }}
+osd_mkfs_options_xfs = {{ .Values.conf.ceph.config.osd.osd_mkfs_options_xfs | default "-f -i size=2048" | quote }}
+osd_max_object_name_len = {{ .Values.conf.ceph.config.osd.osd_max_object_name_len | default "256" | quote }}
+
+# crush
+osd_pool_default_crush_rule = {{ .Values.conf.ceph.config.osd.osd_pool_default_crush_rule | default "0" | quote }}
+osd_crush_update_on_start = {{ .Values.conf.ceph.config.osd.osd_crush_update_on_start | default "true" | quote }}
+
+# backend
+osd_objectstore = {{ .Values.conf.ceph.config.osd.osd_objectstore | default "filestore" | quote }}
+
+# performance tuning
+filestore_merge_threshold = {{ .Values.conf.ceph.config.osd.filestore_merge_threshold | default "40" | quote }}
+filestore_split_multiple = {{ .Values.conf.ceph.config.osd.filestore_split_multiple | default "8" | quote }}
+osd_op_threads = {{ .Values.conf.ceph.config.osd.osd_op_threads | default "8" | quote }}
+filestore_op_threads = {{ .Values.conf.ceph.config.osd.filestore_op_threads | default "8" | quote }}
+filestore_max_sync_interval = {{ .Values.conf.ceph.config.osd.filestore_max_sync_interval | default "5" | quote }}
+osd_max_scrubs = {{ .Values.conf.ceph.config.osd.osd_max_scrubs | default "1" | quote }}
+
+# recovery tuning
+osd_recovery_max_active = {{ .Values.conf.ceph.config.osd.osd_recovery_max_active | default "5" | quote }}
+osd_max_backfills = {{ .Values.conf.ceph.config.osd.osd_max_backfills | default "2" | quote }}
+osd_recovery_op_priority = {{ .Values.conf.ceph.config.osd.osd_recovery_op_priority | default "2" | quote }}
+osd_client_op_priority = {{ .Values.conf.ceph.config.osd.osd_client_op_priority | default "63" | quote }}
+osd_recovery_max_chunk = {{ .Values.conf.ceph.config.osd.osd_client_op_priority | default "osd_recovery_max_chunk" | quote }}
+osd_recovery_threads = {{ .Values.conf.ceph.config.osd.osd_recovery_threads | default "1" | quote }}
+
+[client]
+rbd_cache_enabled = {{ .Values.conf.ceph.config.client.rbd_cache_enabled | default "true" | quote }}
+rbd_cache_writethrough_until_flush = {{ .Values.conf.ceph.config.client.rbd_cache_writethrough_until_flush | default "true" | quote }}
+rbd_default_features = {{ .Values.conf.ceph.config.client.rbd_default_features | default "1" | quote }}
+
+[mds]
+mds_cache_size = {{ .Values.conf.ceph.config.client.mds_mds_cache_size | default "100000" | quote }}

ceph/templates/job-keyring.yaml

@@ -0,0 +1,85 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{- if .Values.manifests_enabled.storage_secrets }}
+{{- $envAll := . }}
+{{- range $key1, $cephBootstrapKey := tuple "mds" "osd" "rgw" "mon" }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ceph-{{ $cephBootstrapKey }}-keyring-generator
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name:  ceph-secret-generator
+          image: {{ $envAll.Values.images.ceph_config_helper }}
+          imagePullPolicy: {{ $envAll.Values.images.pull_policy }}
+          {{- if $envAll.Values.resources.enabled }}
+          resources:
+            requests:
+              memory: {{ .Values.resources.jobs.secret_provisioning.requests.memory | quote }}
+              cpu: {{ .Values.resources.jobs.secret_provisioning.requests.cpu | quote }}
+            limits:
+              memory: {{ .Values.resources.jobs.secret_provisioning.limits.memory | quote }}
+              cpu: {{ .Values.resources.jobs.secret_provisioning.limits.cpu | quote }}
+          {{- end }}
+          env:
+            - name: DEPLOYMENT_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: CEPH_GEN_DIR
+              value: /opt/ceph
+            - name: CEPH_TEMPLATES_DIR
+              value: /opt/ceph/templates
+            {{- if eq $cephBootstrapKey "mon"}}
+            - name: CEPH_KEYRING_NAME
+              value: ceph.mon.keyring
+            - name: CEPH_KEYRING_TEMPLATE
+              value: mon.keyring
+            {{- else }}
+            - name: CEPH_KEYRING_NAME
+              value: ceph.keyring
+            - name: CEPH_KEYRING_TEMPLATE
+              value: bootstrap.keyring.{{ $cephBootstrapKey }}
+            {{- end }}
+            - name: KUBE_SECRET_NAME
+              value: {{  index $envAll.Values.secrets.keyrings $cephBootstrapKey }}
+          command:
+            - /opt/ceph/ceph-key.sh
+          volumeMounts:
+            - name: ceph-bin
+              mountPath: /opt/ceph/ceph-key.sh
+              subPath: ceph-key.sh
+              readOnly: true
+            - name: ceph-bin
+              mountPath: /opt/ceph/ceph-key.py
+              subPath: ceph-key.py
+              readOnly: true
+            - name: ceph-templates
+              mountPath: /opt/ceph/templates
+              readOnly: true
+      volumes:
+        - name: ceph-bin
+          configMap:
+            name: ceph-bin
+            defaultMode: 0555
+        - name: ceph-templates
+          configMap:
+            name: ceph-templates
+{{ end }}
+{{ end }}

ceph/templates/job-namespace-client-key.yaml

@@ -0,0 +1,61 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{- if .Values.manifests_enabled.client_secrets }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ceph-namespace-client-key-generator
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name:  ceph-storage-keys-generator
+          image: {{ .Values.images.ceph_config_helper }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+          {{- if .Values.resources.enabled }}
+          resources:
+            requests:
+              memory: {{ .Values.resources.jobs.secret_provisioning.requests.memory | quote }}
+              cpu: {{ .Values.resources.jobs.secret_provisioning.requests.cpu | quote }}
+            limits:
+              memory: {{ .Values.resources.jobs.secret_provisioning.limits.memory | quote }}
+              cpu: {{ .Values.resources.jobs.secret_provisioning.limits.cpu | quote }}
+          {{- end }}
+          env:
+            - name: DEPLOYMENT_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: PVC_CEPH_STORAGECLASS_USER_SECRET_NAME
+              value: {{ .Values.storageclass.user_secret_name }}
+            - name: PVC_CEPH_STORAGECLASS_ADMIN_SECRET_NAME
+              value: {{ .Values.storageclass.admin_secret_name }}
+            - name: PVC_CEPH_STORAGECLASS_DEPLOYED_NAMESPACE
+              value: {{ .Values.storageclass.admin_secret_namespace }}
+          command:
+            - /opt/ceph/ceph-namespace-client-key.sh
+          volumeMounts:
+            - name: ceph-bin
+              mountPath: /opt/ceph/ceph-namespace-client-key.sh
+              subPath: ceph-namespace-client-key.sh
+              readOnly: true
+      volumes:
+        - name: ceph-bin
+          configMap:
+            name: ceph-bin
+            defaultMode: 0555
+{{- end }}

ceph/templates/job-storage-admin-keys.yaml

@@ -0,0 +1,77 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{- if .Values.manifests_enabled.storage_secrets }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ceph-storage-keys-generator
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name:  ceph-storage-keys-generator
+          image: {{ .Values.images.ceph_config_helper }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+          {{- if .Values.resources.enabled }}
+          resources:
+            requests:
+              memory: {{ .Values.resources.jobs.secret_provisioning.requests.memory | quote }}
+              cpu: {{ .Values.resources.jobs.secret_provisioning.requests.cpu | quote }}
+            limits:
+              memory: {{ .Values.resources.jobs.secret_provisioning.limits.memory | quote }}
+              cpu: {{ .Values.resources.jobs.secret_provisioning.limits.cpu | quote }}
+          {{- end }}
+          env:
+            - name: DEPLOYMENT_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: CEPH_GEN_DIR
+              value: /opt/ceph
+            - name: CEPH_TEMPLATES_DIR
+              value: /opt/ceph/templates
+            - name: CEPH_KEYRING_NAME
+              value: ceph.client.admin.keyring
+            - name: CEPH_KEYRING_TEMPLATE
+              value: admin.keyring
+            - name: CEPH_KEYRING_ADMIN_NAME
+              value: {{ .Values.secrets.keyrings.admin }}
+            - name: CEPH_STORAGECLASS_ADMIN_SECRET_NAME
+              value: {{ .Values.storageclass.admin_secret_name }}
+          command:
+            - /opt/ceph/ceph-storage-key.sh
+          volumeMounts:
+            - name: ceph-bin
+              mountPath: /opt/ceph/ceph-storage-key.sh
+              subPath: ceph-storage-key.sh
+              readOnly: true
+            - name: ceph-bin
+              mountPath: /opt/ceph/ceph-key.py
+              subPath: ceph-key.py
+              readOnly: true
+            - name: ceph-templates
+              mountPath: /opt/ceph/templates
+              readOnly: true
+      volumes:
+        - name: ceph-bin
+          configMap:
+            name: ceph-bin
+            defaultMode: 0555
+        - name: ceph-templates
+          configMap:
+            name: ceph-templates
+{{- end }}

ceph/templates/pdb-mon.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.manifests_enabled.deployment }}
 apiVersion: policy/v1beta1
 kind: PodDisruptionBudget
 metadata:
@@ -7,4 +8,5 @@ spec:
   selector:
     matchLabels:
       app: ceph
-      daemon: mon
+      daemon: mon
+{{- end }}

ceph/templates/secrets.yaml

@@ -1,65 +0,0 @@
-# Copyright 2017 The Openstack-Helm Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-{{- if .Values.secrets.use_common_secrets -}}
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: "ceph-conf-combined"
-type: Opaque
-data:
-  ceph.conf: |
-{{ include "secrets/ceph.conf" . | b64enc | indent 4 }}
-  ceph.client.admin.keyring: |
-{{ include "secrets/ceph.client.admin.keyring" . | b64enc | indent 4 }}
-  ceph.mon.keyring: |
-{{ include "secrets/ceph.mon.keyring" . | b64enc | indent 4 }}
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: "ceph-bootstrap-rgw-keyring"
-type: Opaque
-data:
-  ceph.keyring: |
-{{ include "secrets/ceph.rgw.keyring" . | b64enc | indent 4 }}
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: "ceph-bootstrap-mds-keyring"
-type: Opaque
-data:
-  ceph.keyring: |
-{{ include "secrets/ceph.mds.keyring" . | b64enc | indent 4 }}
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: "ceph-bootstrap-osd-keyring"
-type: Opaque
-data:
-  ceph.keyring: |
-{{ include "secrets/ceph.osd.keyring" . | b64enc | indent 4 }}
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: "ceph-client-key"
-type: Opaque
-data:
-  ceph-client-key: |
-{{ include "secrets/ceph-client-key" . | b64enc | indent 4 }}
-{{- end -}}

ceph/templates/service-rgw.yaml

@@ -12,7 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-{{- if .Values.rgw.enabled }}
+{{- if .Values.manifests_enabled.deployment }}
+{{- if .Values.ceph.enabled.rgw }}
 ---
 apiVersion: v1
 kind: Service
@@ -30,3 +31,4 @@ spec:
     app: ceph
     daemon: rgw
 {{- end }}
+{{- end }}

ceph/templates/service.yaml

@@ -12,11 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+{{- if .Values.manifests_enabled.deployment }}
 ---
 kind: Service
 apiVersion: v1
 metadata:
-  name: ceph-mon
+  name: {{ .Values.endpoints.ceph_mon.hosts.default }}
   labels:
     app: ceph
     daemon: mon
@@ -37,3 +38,4 @@ spec:
     app: ceph
     daemon: mon
   clusterIP: None
+{{- end }}

ceph/templates/statefulset-mon.yaml

@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+{{- if .Values.manifests_enabled.deployment }}
+{{- $envAll := . }}
+{{- $dependencies := .Values.dependencies.mon }}
 ---
 apiVersion: apps/v1beta1
 kind: StatefulSet
@@ -21,7 +24,7 @@ metadata:
     daemon: mon
   name: ceph-mon
 spec:
-  serviceName: {{ .Values.service.mon.name | quote }}
+  serviceName: {{ tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_endpoint_lookup" }}
   replicas: {{ .Values.replicas.mon }}
   template:
     metadata:
@@ -46,33 +49,22 @@ spec:
             weight: 10
       nodeSelector:
         {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
+      initContainers:
+{{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
       serviceAccount: default
-      volumes:
-        - name: ceph-conf
-          secret:
-            secretName: ceph-conf-combined
-        - name: ceph-bootstrap-osd-keyring
-          secret:
-            secretName: ceph-bootstrap-osd-keyring
-        - name: ceph-bootstrap-mds-keyring
-          secret:
-            secretName: ceph-bootstrap-mds-keyring
-        - name: ceph-bootstrap-rgw-keyring
-          secret:
-            secretName: ceph-bootstrap-rgw-keyring
-        - name: ceph-monfs
-          hostPath:
-            path: {{ .Values.storage.mon_directory }}
       containers:
         - name: ceph-mon
           image: {{ .Values.images.daemon }}
           imagePullPolicy: {{ .Values.images.pull_policy }}
-          lifecycle:
+          {{- if .Values.resources.enabled }}
-            preStop:
+          resources:
-                exec:
+            requests:
-                  # remove the mon on Pod stop.
+              memory: {{ .Values.resources.mon.requests.memory | quote }}
-                  command:
+              cpu: {{ .Values.resources.mon.requests.cpu | quote }}
-                    - "/remove-mon.sh"
+            limits:
+              memory: {{ .Values.resources.mon.limits.memory | quote }}
+              cpu: {{ .Values.resources.mon.limits.cpu | quote }}
+          {{- end }}
           ports:
             - containerPort: 6789
           env:
@@ -90,17 +82,39 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+          command:
+            - /entrypoint.sh
+          lifecycle:
+            preStop:
+                exec:
+                  # remove the mon on Pod stop.
+                  command:
+                    - "/remove-mon.sh"
           volumeMounts:
-            - name: ceph-conf
+            - name: ceph-etc
-              mountPath: /etc/ceph
+              mountPath: /etc/ceph/ceph.conf
+              subPath: ceph.conf
+              readOnly: true
+            - name: ceph-client-admin-keyring
+              mountPath: /etc/ceph/ceph.client.admin.keyring
+              subPath: ceph.client.admin.keyring
+              readOnly: true
+            - name: ceph-mon-keyring
+              mountPath: /etc/ceph/ceph.mon.keyring
+              subPath: ceph.mon.keyring
+              readOnly: false
             - name: ceph-bootstrap-osd-keyring
-              mountPath: /var/lib/ceph/bootstrap-osd
+              mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring
+              subPath: ceph.keyring
+              readOnly: false
             - name: ceph-bootstrap-mds-keyring
-              mountPath: /var/lib/ceph/bootstrap-mds
+              mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring
+              subPath: ceph.keyring
+              readOnly: false
             - name: ceph-bootstrap-rgw-keyring
-              mountPath: /var/lib/ceph/bootstrap-rgw
+              mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring
-            - name: ceph-monfs
+              subPath: ceph.keyring
-              mountPath: /var/lib/ceph/mon
+              readOnly: false
           livenessProbe:
               tcpSocket:
                 port: 6789
@@ -110,10 +124,23 @@ spec:
               tcpSocket:
                 port: 6789
               timeoutSeconds: 5
-          resources:
+      volumes:
-            requests:
+        - name: ceph-etc
-              memory: {{ .Values.resources.mon.requests.memory | quote }}
+          configMap:
-              cpu: {{ .Values.resources.mon.requests.cpu | quote }}
+            name: ceph-etc
-            limits:
+        - name: ceph-client-admin-keyring
-              memory: {{ .Values.resources.mon.limits.memory | quote }}
+          secret:
-              cpu: {{ .Values.resources.mon.limits.cpu | quote }}
+            secretName: {{ .Values.secrets.keyrings.admin }}
+        - name: ceph-mon-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.mon }}
+        - name: ceph-bootstrap-osd-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.osd }}
+        - name: ceph-bootstrap-mds-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.mds }}
+        - name: ceph-bootstrap-rgw-keyring
+          secret:
+            secretName: {{ .Values.secrets.keyrings.rgw }}
+{{- end }}

ceph/templates/storage.yaml

@@ -12,8 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# note that these secrets are handled by the common chart, not the ceph
+{{- if .Values.manifests_enabled.deployment }}
-# chart, as we likely want them "everywhere"
 ---
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
@@ -21,12 +20,11 @@ metadata:
     name: {{ .Values.storageclass.name }}
 provisioner: kubernetes.io/rbd
 parameters:
-    monitors: {{ .Values.storageclass.monitors | default "ceph-mon.ceph:6789" }}
+    monitors: {{ tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.hostname_endpoint_uri_lookup" }}
     adminId: {{ .Values.storageclass.admin_id }}
     adminSecretName: {{ .Values.storageclass.admin_secret_name }}
-    # forcing namespace due to issue with default pipeline of "{{ .Release.Namespace }}" }}
+    adminSecretNamespace: {{ .Values.storageclass.admin_secret_namespace }}
-    # during helm lint
-    adminSecretNamespace: {{ .Values.storageclass.admin_secret_namespace | default "ceph" }}
     pool: {{ .Values.storageclass.pool }}
     userId: {{ .Values.storageclass.user_id }}
     userSecretName: {{ .Values.storageclass.user_secret_name }}
+{{- end }}

ceph/templates/templates/_admin.keyring.tpl

@@ -0,0 +1,6 @@
+[client.admin]
+  key = {{"{{"}} key {{"}}"}}
+  auid = 0
+  caps mds = "allow"
+  caps mon = "allow *"
+  caps osd = "allow *"

ceph/templates/templates/_bootstrap.keyring.mds.tpl

@@ -0,0 +1,3 @@
+[client.bootstrap-mds]
+  key = {{"{{"}} key {{"}}"}}
+  caps mon = "allow profile bootstrap-mds"

ceph/templates/templates/_bootstrap.keyring.osd.tpl

@@ -0,0 +1,3 @@
+[client.bootstrap-osd]
+  key = {{"{{"}} key {{"}}"}}
+  caps mon = "allow profile bootstrap-osd"

ceph/templates/templates/_bootstrap.keyring.rgw.tpl

@@ -0,0 +1,3 @@
+[client.bootstrap-rgw]
+  key = {{"{{"}} key {{"}}"}}
+  caps mon = "allow profile bootstrap-rgw"

ceph/templates/templates/_mon.keyring.tpl

@@ -0,0 +1,3 @@
+[mon.]
+  key = {{"{{"}} key {{"}}"}}
+  caps mon = "allow *"

ceph/values.yaml

@@ -12,6 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+manifests_enabled:
+  storage_secrets: true
+  client_secrets: true
+  deployment: true
+
 replicas:
   mon: 3
   rgw: 3
@@ -22,8 +27,10 @@ service:
     name: ceph-mon
 
 images:
-  daemon: docker.io/library/ceph/daemon:tag-build-master-jewel-ubuntu-16.04
+  dep_check: docker.io/kolla/ubuntu-source-kubernetes-entrypoint:4.0.0
-  pull_policy: IfNotPresent
+  daemon: quay.io/attcomdev/ceph-daemon:tag-build-master-jewel-ubuntu-16.04
+  ceph_config_helper: docker.io/port/ceph-config-helper:v1.6.5
+  pull_policy: Always
 
 labels:
   node_selector_key: ceph-storage
@@ -33,23 +40,125 @@ pod_disruption_budget:
   mon:
     min_available: 0
 
+secrets:
+  keyrings:
+    mon: ceph-mon-keyring
+    mds: ceph-bootstrap-mds-keyring
+    osd: ceph-bootstrap-osd-keyring
+    rgw: ceph-bootstrap-rgw-keyring
+    admin: ceph-client-admin-keyring
+
 network:
-  public: "10.25.0.0/16"
+  public: "192.168.0.0/16"
+  cluster: "192.168.0.0/16"
   port:
     mon: 6789
     rgw_ingress: 80
     rgw_target: 8088
 
-storage:
+conf:
-  osd_directory: /var/lib/openstack-helm/ceph/osd
+  ceph:
-  var_directory: /var/lib/openstack-helm/ceph/ceph
+    override:
-  mon_directory: /var/lib/openstack-helm/ceph/mon
+    append:
+    config:
+      global:
+        # auth
+        cephx: true
+        cephx_require_signatures: false
+        cephx_cluster_require_signatures: true
+        cephx_service_require_signatures: false
+
+        max_open_files: 131072
+        osd_pool_default_pg_num: 128
+        osd_pool_default_pgp_num: 128
+        osd_pool_default_size: 3
+        osd_pool_default_min_size: 1
+        mon_osd_full_ratio: .95
+        mon_osd_nearfull_ratio: .85
+        mon_host: null
+      mon:
+        mon_osd_down_out_interval: 600
+        mon_osd_min_down_reporters: 4
+        mon_clock_drift_allowed: .15
+        mon_clock_drift_warn_backoff: 30
+        mon_osd_report_timeout: 300
+      osd:
+        journal_size: 100
+        osd_mkfs_type: xfs
+        osd_mkfs_options_xfs: -f -i size=2048
+        osd_mon_heartbeat_interval: 30
+        osd_max_object_name_len: 256
+        #crush
+        osd_pool_default_crush_rule: 0
+        osd_crush_update_on_start: true
+        #backend
+        osd_objectstore: filestore
+        #performance tuning
+        filestore_merge_threshold: 40
+        filestore_split_multiple: 8
+        osd_op_threads: 8
+        filestore_op_threads: 8
+        filestore_max_sync_interval: 5
+        osd_max_scrubs: 1
+        #recovery tuning
+        osd_recovery_max_active: 5
+        osd_max_backfills: 2
+        osd_recovery_op_priority: 2
+        osd_client_op_priority: 63
+        osd_recovery_max_chunk: 1048576
+        osd_recovery_threads: 1
+        #ports
+        ms_bind_port_min: 6800
+        ms_bind_port_max: 7100
+      client:
+        rbd_cache_enabled: true
+        rbd_cache_writethrough_until_flush: true
+        rbd_default_features: "1"
+      mds:
+        mds_cache_size: 100000
+
+
+dependencies:
+  mon:
+    jobs:
+    service:
+  osd:
+    jobs:
+    services:
+    - service: ceph_mon
+      endpoint: internal
+  moncheck:
+    jobs:
+    services:
+    - service: ceph_mon
+      endpoint: internal
+  rgw:
+    jobs:
+    services:
+    - service: ceph_mon
+      endpoint: internal
+  mds:
+    jobs:
+    services:
+    - service: ceph_mon
+      endpoint: internal
+
+
+ceph:
+  enabled:
+    mds: true
+    rgw: false
+  storage:
+    osd_directory: /var/lib/openstack-helm/ceph/osd
+    var_directory: /var/lib/openstack-helm/ceph/ceph
+    mon_directory: /var/lib/openstack-helm/ceph/mon
 
 # rgw is optionally disabled
 rgw:
   enabled: false
 
 resources:
+  enabled: false
   osd:
     requests:
       memory: "512Mi"
@@ -85,36 +194,14 @@ resources:
     limits:
       memory: "50Mi"
       cpu: "500m"
-
+  jobs:
-# Setting this to false will assume you will
+    secret_provisioning:
-# setup and orchestrate your own secrets and
+      limits:
-# configmaps outside of this helm chart
+        memory: "1024Mi"
-#
+        cpu: "2000m"
-# The list below is in the format of
+      requests:
-#
+        memory: "128Mi"
-#   configMapName:
+        cpu: "500m"
-#      elementKeyName
-#
-#   ceph.conf:
-#      ceph.conf
-#      ceph.client.admin.keyring
-#      ceph.client.admin.keyring
-#   ceph.mon.keyring:
-#      ceph.mon.keyring
-#   ceph-bootstrap-rgw-keyring:
-#      ceph.keyring
-#      ceph.rgw.keyring
-#   ceph-bootstrap-mds-keyring:
-#      ceph.keyring
-#      ceph.mds.keyring
-#   ceph-bootstrap-osd-keyring:
-#      ceph.keyring
-#      ceph.osd.keyring
-#   ceph-client-key:
-#      ceph-client-key
-secrets:
-  use_common_secrets: true
-
 # if you change provision_storage_class to false
 # it is presumed you manage your own storage
 # class definition externally
@@ -125,6 +212,13 @@ storageclass:
   pool: rbd
   admin_id: admin
   admin_secret_name: pvc-ceph-conf-combined-storageclass
-  admin_secret_namespace: null
+  admin_secret_namespace: ceph
   user_id: admin
   user_secret_name: pvc-ceph-client-key
+
+endpoints:
+  ceph_mon:
+    hosts:
+      default: ceph-mon
+    port:
+      mon: 6789

cinder/templates/_funcs.tpl

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-{{- define "cinder.is_ceph_configured" -}}
+{{- define "cinder.is_ceph_volume_configured" -}}
 {{- range $section, $values := .Values.conf.backends -}}
 {{- if kindIs "map" $values -}}
 {{- if eq $values.volume_driver "cinder.volume.drivers.rbd.RBDDriver" -}}
@@ -21,3 +21,10 @@ true
 {{- end -}}
 {{- end -}}
 {{- end -}}
+
+{{- define "cinder.is_ceph_backup_configured" -}}
+{{- $values := .Values.conf.cinder.default.cinder -}}
+{{- if eq $values.backup_driver "cinder.backup.drivers.ceph" -}}
+true
+{{- end -}}
+{{- end -}}

cinder/templates/bin/_ceph-keyring.sh.tpl

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 # Copyright 2017 The Openstack-Helm Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,9 +14,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -ex
+export HOME=/tmp
+
+cat <<EOF > /etc/ceph/ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring
 [client.{{ .Values.conf.backends.rbd1.rbd_user }}]
 {{- if .Values.conf.ceph.cinder_keyring }}
     key = {{ .Values.conf.ceph.cinder_keyring }}
 {{- else }}
-    key = {{- include "secrets/ceph-client-key" . -}}
+    key = $(cat /tmp/client-keyring)
 {{- end }}
+EOF
+
+exit 0

cinder/templates/configmap-bin.yaml

@@ -33,6 +33,8 @@ data:
 {{ tuple "bin/_cinder-scheduler.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   cinder-volume.sh: |
 {{ tuple "bin/_cinder-volume.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  ceph-keyring.sh: |+
+{{ tuple "bin/_ceph-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- if .Values.bootstrap.enabled }}
   bootstrap.sh: |+
 {{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

cinder/templates/configmap-etc.yaml

@@ -97,17 +97,4 @@ data:
 {{ .Values.conf.policy.override | indent 4 }}
 {{- else -}}
 {{ tuple "etc/_policy.json.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-{{- end }}
-  ceph.conf: |+
-{{- if or (include "cinder.is_ceph_configured" .) (eq .Values.conf.cinder.default.cinder.backup_driver "cinder.backup.drivers.ceph") }}
-{{ if .Values.conf.ceph.override -}}
-{{ .Values.conf.ceph.override | indent 4 }}
-{{- else -}}
-{{ tuple "etc/_ceph.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-{{- end }}
-{{- if .Values.conf.ceph.append -}}
-{{ .Values.conf.ceph.append | indent 4 }}
-{{- end }}
-  ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring: |+
-{{ tuple "etc/_ceph-cinder.keyring.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}

cinder/templates/deployment-backup.yaml

@@ -42,6 +42,23 @@ spec:
         {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
       initContainers:
 {{ tuple $envAll $dependencies $mounts_cinder_backup_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+        {{- if include "cinder.is_ceph_backup_configured" . }}
+        - name: ceph-keyring-placement
+          image: {{ .Values.images.backup }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+          command:
+            - /tmp/ceph-keyring.sh
+          volumeMounts:
+            - name: etcceph
+              mountPath: /etc/ceph
+            - name: cinder-bin
+              mountPath: /tmp/ceph-keyring.sh
+              subPath: ceph-keyring.sh
+            - name: ceph-keyring
+              mountPath: /tmp/client-keyring
+              subPath: key
+              readOnly: true
+        {{ end }}
       containers:
         - name: cinder-backup
           image: {{ .Values.images.backup }}
@@ -67,13 +84,15 @@ spec:
               subPath: cinder.conf
               readOnly: true
             {{- if eq .Values.conf.cinder.default.cinder.backup_driver "cinder.backup.drivers.ceph" }}
-            - name: cinder-etc
+            - name: etcceph
+              mountPath: /etc/ceph
+            - name: ceph-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
               readOnly: true
-            - name: cinder-etc
+            - name: ceph-keyring
-              mountPath: /etc/ceph/ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring
+              mountPath: /tmp/client-keyring
-              subPath: ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring
+              subPath: key
               readOnly: true
             {{- end -}}
 {{ if $mounts_cinder_backup.volumeMounts }}{{ toYaml $mounts_cinder_backup.volumeMounts | indent 12 }}{{ end }}
@@ -83,4 +102,18 @@ spec:
         - name: cinder-etc
           configMap:
             name: cinder-etc
+        - name: cinder-bin
+          configMap:
+            name: cinder-bin
+            defaultMode: 0555
+        {{- if include "cinder.is_ceph_backup_configured" . }}
+        - name: etcceph
+          emptyDir: {}
+        - name: ceph-etc
+          configMap:
+            name: ceph-etc
+        - name: ceph-keyring
+          secret:
+            secretName: pvc-ceph-client-key
+        {{ end }}
 {{ if $mounts_cinder_backup.volumes }}{{ toYaml $mounts_cinder_backup.volumes | indent 8 }}{{ end }}

cinder/templates/deployment-volume.yaml

@@ -42,6 +42,23 @@ spec:
         {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
       initContainers:
 {{ tuple $envAll $dependencies $mounts_cinder_volume_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+        {{- if include "cinder.is_ceph_volume_configured" . }}
+        - name: ceph-keyring-placement
+          image: {{ .Values.images.volume }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+          command:
+            - /tmp/ceph-keyring.sh
+          volumeMounts:
+            - name: etcceph
+              mountPath: /etc/ceph
+            - name: cinder-bin
+              mountPath: /tmp/ceph-keyring.sh
+              subPath: ceph-keyring.sh
+            - name: ceph-keyring
+              mountPath: /tmp/client-keyring
+              subPath: key
+              readOnly: true
+        {{ end }}
       containers:
         - name: cinder-volume
           image: {{ .Values.images.volume }}
@@ -72,14 +89,16 @@ spec:
               mountPath: /etc/cinder/conf/backends.conf
               subPath: backends.conf
               readOnly: true
-            {{- if include "cinder.is_ceph_configured" . }}
+            {{- if include "cinder.is_ceph_volume_configured" . }}
-            - name: cinder-etc
+            - name: etcceph
+              mountPath: /etc/ceph
+            - name: ceph-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
               readOnly: true
-            - name: cinder-etc
+            - name: ceph-keyring
-              mountPath: /etc/ceph/ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring
+              mountPath: /tmp/client-keyring
-              subPath: ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring
+              subPath: key
               readOnly: true
             {{- end }}
 {{ if $mounts_cinder_volume.volumeMounts }}{{ toYaml $mounts_cinder_volume.volumeMounts | indent 12 }}{{ end }}
@@ -93,4 +112,14 @@ spec:
         - name: cinder-etc
           configMap:
             name: cinder-etc
+        {{- if include "cinder.is_ceph_volume_configured" . }}
+        - name: etcceph
+          emptyDir: {}
+        - name: ceph-etc
+          configMap:
+            name: ceph-etc
+        - name: ceph-keyring
+          secret:
+            secretName: pvc-ceph-client-key
+        {{ end }}
 {{ if $mounts_cinder_volume.volumes }}{{ toYaml $mounts_cinder_volume.volumes | indent 8 }}{{ end }}

cinder/templates/etc/_ceph.conf.tpl

@@ -1,30 +0,0 @@
-# Copyright 2017 The Openstack-Helm Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-[global]
-rgw_thread_pool_size = 1024
-rgw_num_rados_handles = 100
-{{- if .Values.conf.ceph.monitors }}
-[mon]
-{{ range .Values.conf.ceph.monitors }}
-    [mon.{{ . }}]
-      host = {{ . }}
-      mon_addr = {{ . }}
-{{ end }}
-{{- else }}
-mon_host = ceph-mon.ceph
-{{- end }}
-[client]
-  rbd_cache_enabled = true
-  rbd_cache_writethrough_until_flush = true

cinder/values.yaml

@@ -23,6 +23,8 @@ replicas:
   scheduler: 1
   backup: 1
 
+storage: ceph
+
 labels:
   node_selector_key: openstack-control-plane
   node_selector_value: enabled

doc/source/install/multinode.rst

@@ -141,32 +141,13 @@ completed.
 Installing Ceph Host Requirements
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-At some future point, we want to ensure that our solution is
+You need to ensure that ``ceph-common`` or equivalent is
-cloud-native, allowing installation on any host system without a package
+installed on each of our hosts. Using our Ubuntu example:
-manager and only a container runtime (i.e. CoreOS). Until this happens,
-we will need to ensure that ``ceph-common`` is installed on each of our
-hosts. Using our Ubuntu example:
 
 ::
 
     sudo apt-get install ceph-common -y
 
-We will always attempt to keep host-specific requirements to a minimum,
-and we are working with the Ceph team (Sébastien Han) to quickly address
-this Ceph requirement.
-
-Ceph Secrets Generation
-~~~~~~~~~~~~~~~~~~~~~~~
-
-Another thing of interest is that our deployment assumes that you can
-generate secrets at the time of the container deployment. We require the
-`sigil <https://github.com/gliderlabs/sigil/releases/download/v0.4.0/sigil_0.4.0_Linux_x86_64.tgz>`__
-binary on your deployment host in order to perform this action.
-
-::
-
-    curl -L https://github.com/gliderlabs/sigil/releases/download/v0.4.0/sigil_0.4.0_Linux_x86_64.tgz | sudo tar -zxC /usr/local/bin
-
 Kubernetes Controller Manager
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -200,20 +181,18 @@ Kubernetes v1.6.5.
     export kube_version=v1.6.5
     sudo sed -i "s|gcr.io/google_containers/kube-controller-manager-amd64:$kube_version|quay.io/attcomdev/kube-controller-manager:$kube_version|g" /etc/kubernetes/manifests/kube-controller-manager.yaml
 
-Now you will want to ``restart`` your Kubernetes master server to
+Now you will want to ``restart`` the Kubernetes master server to
 continue.
 
 Kube Controller Manager DNS Resolution
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Until the following `Kubernetes Pull
+You will need to allow the Kubernetes Controller to use the
-Request <https://github.com/kubernetes/kubernetes/issues/17406>`__ is
+Kubernetes service DNS server, and add the Kubernetes search suffix
-merged, you will need to allow the Kubernetes Controller to use the
+to the controller's resolv.conf. As of now, the Kubernetes controller
-internal container ``skydns`` endpoint as a DNS server, and add the
+only mirrors the host's ``resolv.conf``. This is not sufficient if you
-Kubernetes search suffix into the controller's resolv.conf. As of now,
+want the controller to know how to correctly resolve container service
-the Kubernetes controller only mirrors the host's ``resolv.conf``. This
+endpoints.
-is not sufficient if you want the controller to know how to correctly
-resolve container service endpoints (in the case of DaemonSets).
 
 First, find out what the IP Address of your ``kube-dns`` deployment is:
 
@@ -224,82 +203,16 @@ First, find out what the IP Address of your ``kube-dns`` deployment is:
     kube-dns   10.96.0.10   <none>        53/UDP,53/TCP   1d
     admin@kubenode01:~$
 
-As you can see by this example, ``10.96.0.10`` is the
+Then update the controller manager configuration to match:
-``CLUSTER-IP``\ IP. Now, have a look at the current
-``kube-controller-manager-kubenode01`` ``/etc/resolv.conf``:
 
 ::
 
-    admin@kubenode01:~$ kubectl exec kube-controller-manager-kubenode01 -n kube-system -- cat /etc/resolv.conf
+    admin@kubenode01:~$ CONTROLLER_MANAGER_POD=$(kubectl get -n kube-system pods -l component=kube-controller-manager --no-headers -o name | head -1 | awk -F '/' '{ print $NF }')
-    # Dynamic resolv.conf(5) file for glibc resolver(3) generated by resolvconf(8)
+    admin@kubenode01:~$ kubectl exec -n kube-system ${CONTROLLER_MANAGER_POD} -- sh -c "cat > /etc/resolv.conf <<EOF
-    #     DO NOT EDIT THIS FILE BY HAND -- YOUR CHANGES WILL BE OVERWRITTEN
-    nameserver 192.168.1.70
-    nameserver 8.8.8.8
-    search jinkit.com
-    admin@kubenode01:~$
-
-What we need is for ``kube-controller-manager-kubenode01``
-``/etc/resolv.conf`` to look like this:
-
-::
-
-    admin@kubenode01:~$ kubectl exec kube-controller-manager-kubenode01 -n kube-system -- cat /etc/resolv.conf
     nameserver 10.96.0.10
-    nameserver 192.168.1.70
     nameserver 8.8.8.8
-    search svc.cluster.local jinkit.com
+    search cluster.local svc.cluster.local
-    admin@kubenode01:~$
+    EOF"
-
-You can change this by doing the following:
-
-::
-
-    admin@kubenode01:~$ kubectl exec kube-controller-manager-kubenode01 -it -n kube-system -- /bin/bash
-    root@kubenode01:/# cat <<EOF > /etc/resolv.conf
-    nameserver 10.96.0.10
-    nameserver 192.168.1.70
-    nameserver 8.8.8.8
-    search svc.cluster.local jinkit.com
-    EOF
-    root@kubenode01:/#
-
-Now you can test your changes by deploying a service to your cluster,
-and resolving this from the controller. As an example, lets deploy
-something useful, like `Kubernetes
-dashboard <https://github.com/kubernetes/dashboard>`__:
-
-::
-
-    kubectl create -f https://rawgit.com/kubernetes/dashboard/master/src/deploy/kubernetes-dashboard.yaml
-
-Note the ``IP`` field:
-
-::
-
-    admin@kubenode01:~$ kubectl describe svc kubernetes-dashboard -n kube-system
-    Name:               kubernetes-dashboard
-    Namespace:          kube-system
-    Labels:             app=kubernetes-dashboard
-    Selector:           app=kubernetes-dashboard
-    Type:               NodePort
-    IP:                 10.110.207.144
-    Port:               <unset> 80/TCP
-    NodePort:           <unset> 32739/TCP
-    Endpoints:          10.25.178.65:9090
-    Session Affinity:   None
-    No events.
-    admin@kubenode01:~$
-
-Now you should be able to resolve the host
-``kubernetes-dashboard.kube-system.svc.cluster.local``:
-
-::
-
-    admin@kubenode01:~$ kubectl exec kube-controller-manager-kubenode01 -it -n kube-system -- ping kubernetes-dashboard.kube-system.svc.cluster.local
-    PING kubernetes-dashboard.kube-system.svc.cluster.local (10.110.207.144) 56(84) bytes of data.
-
-.. note::
-  This host example above has ``iputils-ping`` installed.
 
 Kubernetes Node DNS Resolution
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -358,42 +271,15 @@ Download the latest copy of Openstack-Helm:
 Ceph Preparation and Installation
 ---------------------------------
 
-Ceph must be aware of the OSX cluster and public networks. These CIDR
+Ceph must be aware of the OSD cluster and public networks. These CIDR
 ranges are the exact same ranges you used earlier in your Calico
-deployment yaml (our example was 10.25.0.0/16 due to our 192.168.0.0/16
+deployment yaml. Export this variable to your deployment environment by
-overlap). Explore this variable to your deployment environment by
 issuing the following commands:
 
 ::
 
-    export osd_cluster_network=10.25.0.0/16
+    export osd_cluster_network=192.168.0.0/16
-    export osd_public_network=10.25.0.0/16
+    export osd_public_network=192.168.0.0/16
-
-Ceph Storage Volumes
---------------------
-
-Ceph must also have volumes to mount on each host labeled for
-``ceph-storage``. On each host that you labeled, create the following
-directory (can be overriden):
-
-::
-
-    mkdir -p /var/lib/openstack-helm/ceph
-
-*Repeat this step for each node labeled: ``ceph-storage``*
-
-Ceph Secrets Generation
------------------------
-
-Although you can bring your own secrets, we have conveniently created a
-secret generation tool for you (for greenfield deployments). You can
-create secrets for your project by issuing the following:
-
-::
-
-    cd helm-toolkit/utils/secret-generator
-    ./generate_secrets.sh all `./generate_secrets.sh fsid`
-    cd ../../..
 
 Nova Compute Instance Storage
 -----------------------------
@@ -468,30 +354,55 @@ the following command to install Ceph:
 
 ::
 
-    helm install --set network.public=$osd_public_network --name=ceph local/ceph --namespace=ceph
+    helm install --namespace=ceph local/ceph --name=ceph \
+      --set manifests_enabled.client_secrets=false \
+      --set network.public=$osd_public_network \
+      --set network.cluster=$osd_cluster_network
 
-Bootstrap Installation
+Activating Control-Plane Namespace for Ceph
-----------------------
+-------------------------------------------
 
-At this time (and before verification of Ceph) you'll need to install
+In order for Ceph to fulfill PersistentVolumeClaims within Kubernetes namespaces
-the ``bootstrap`` chart. The ``bootstrap`` chart will install secrets
+outside of Ceph's namespace, a client keyring needs to be present within that
-for both the ``ceph`` and ``openstack`` namespaces for the general
+namespace.  For the rest of the OpenStack and supporting core services, this guide
-StorageClass:
+will be deploying the control plane to a seperate namespace ``openstack``.  To
+deploy the aforementioned client keyring to the ``openstack`` namespace:
 
 ::
 
-    helm install --name=bootstrap-ceph local/bootstrap --namespace=ceph
+    helm install --namespace=openstack local/ceph --name=ceph-openstack-config \
-    helm install --name=bootstrap-openstack local/bootstrap --namespace=openstack
+      --set manifests_enabled.storage_secrets=false \
+      --set manifests_enabled.deployment=false \
+      --set ceph.namespace=ceph \
+      --set network.public=$osd_public_network \
+      --set network.cluster=$osd_cluster_network
+
+This will load the client keyring as well as the same ``ceph.conf`` into
+the specified namespace. Deploying ceph.conf into this namespace allows
+OpenStack services to consume this ConfigMap for their Ceph-specific
+configurations.
 
 You may want to validate that Ceph is deployed successfully. For more
 information on this, please see the section entitled `Ceph
 Troubleshooting <../../operator/troubleshooting/persistent-storage.html>`__.
 
+Ceph pool creation
+------------------
+
+You should now be ready to create the pools for OpenStack services to consume,
+using the following commands:
+
+::
+
+    kubectl exec -n ceph ceph-mon-0 -- ceph osd pool create volumes 8
+    kubectl exec -n ceph ceph-mon-0 -- ceph osd pool create images 8
+    kubectl exec -n ceph ceph-mon-0 -- ceph osd pool create vms 8
+
 MariaDB Installation and Verification
 -------------------------------------
 
-We are using Galera to cluster MariaDB and establish a quorum. To
+We are using Galera to cluster MariaDB. To install MariaDB, issue the following
-install the MariaDB, issue the following command:
+command:
 
 ::
 
@@ -514,43 +425,49 @@ Now you can easily install the other services simply by going in order:
 
 ::
 
-    helm install --name=keystone local/keystone --set replicas=2 --namespace=openstack
+    helm install --namespace=openstack --name=keystone local/keystone \
+      --set replicas=2
 
 **Install Horizon:**
 
 ::
 
-    helm install --name=horizon local/horizon --set network.enable_node_port=true --namespace=openstack
+    helm install --namespace=openstack --name=horizon local/horizon \
+      --set network.enable_node_port=true
 
 **Install Glance:**
 
 ::
 
-    helm install --name=glance local/glance --set replicas.api=2,replicas.registry=2 --namespace=openstack
+    helm install --namespace=openstack --name=glance local/glance \
+      --set replicas.api=2,replicas.registry=2
 
 **Install Heat:**
 
 ::
 
-    helm install --name=heat local/heat --namespace=openstack
+    helm install --namespace=openstack --name=heat local/heat
 
 **Install Neutron:**
 
 ::
 
-    helm install --name=neutron local/neutron --set replicas.server=2 --namespace=openstack
+    helm install --namespace=openstack --name=neutron local/neutron \
+      --set replicas.server=2
 
 **Install Nova:**
 
 ::
 
-    helm install --name=nova local/nova --set control_replicas=2 --namespace=openstack
+    helm install --namespace=openstack --name=nova local/nova \
+      --set control_replicas=2
 
 **Install Cinder:**
 
 ::
 
-    helm install --name=cinder local/cinder --set replicas.api=2 --namespace=openstack
+    helm install --namespace=openstack --name=cinder local/cinder \
+      --set replicas.api=2
 
 Final Checks
 ------------

glance/templates/bin/_ceph-keyring.sh.tpl

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 # Copyright 2017 The Openstack-Helm Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,9 +14,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -ex
+export HOME=/tmp
+
+cat <<EOF > /etc/ceph/ceph.client.{{ .Values.conf.glance.glance_store.glance.store.rbd_store_user }}.keyring
 [client.{{ .Values.conf.glance.glance_store.glance.store.rbd_store_user }}]
 {{- if .Values.conf.ceph.keyring }}
     key = {{ .Values.conf.ceph.keyring }}
 {{- else }}
-    key = {{- include "secrets/ceph-client-key" . -}}
+    key = $(cat /tmp/client-keyring)
 {{- end }}
+EOF
+
+exit 0

glance/templates/configmap-bin.yaml

@@ -35,3 +35,5 @@ data:
 {{ tuple "bin/_glance-registry.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   bootstrap.sh: |+
 {{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  ceph-keyring.sh: |+
+{{ tuple "bin/_ceph-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

glance/templates/configmap-etc.yaml

@@ -78,30 +78,6 @@ data:
 {{- end }}
 {{- if .Values.conf.rally_tests.append -}}
 {{ .Values.conf.rally_tests.append | indent 4 }}
-{{- end }}
-  ceph.conf: |+
-{{ if .Values.conf.ceph.override -}}
-{{ .Values.conf.ceph.override | indent 4 }}
-{{- else -}}
-{{- if .Values.conf.ceph.prefix -}}
-{{ .Values.conf.ceph.prefix | indent 4 }}
-{{- end }}
-{{ tuple "etc/_ceph.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-{{- end }}
-{{- if .Values.conf.ceph.append -}}
-{{ .Values.conf.ceph.append | indent 4 }}
-{{- end }}
-  ceph.client.{{ .Values.conf.glance.glance_store.glance.store.rbd_store_user }}.keyring: |+
-{{ if .Values.conf.ceph_client.override -}}
-{{ .Values.conf.ceph_client.override | indent 4 }}
-{{- else -}}
-{{- if .Values.conf.ceph_client.prefix -}}
-{{ .Values.conf.ceph_client.prefix | indent 4 }}
-{{- end }}
-{{ tuple "etc/_ceph.client.glance.keyring.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-{{- end }}
-{{- if .Values.conf.ceph_client.append -}}
-{{ .Values.conf.ceph_client.append | indent 4 }}
 {{- end }}
   glance-api.conf: |+
 {{ if .Values.conf.glance.override -}}

glance/templates/deployment-api.yaml

@@ -43,6 +43,47 @@ spec:
       terminationGracePeriodSeconds: {{ .Values.termination_grace_period.api.timeout | default "600" }}
       initContainers:
 {{ tuple $envAll $dependencies $mounts_glance_api_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+        {{- if eq .Values.storage "pvc" }}
+        - name: glance-perms
+          image: {{ .Values.images.api }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+          securityContext:
+            runAsUser: 0
+          {{- if .Values.resources.enabled }}
+          resources:
+            limits:
+              cpu: {{ .Values.resources.api.limits.cpu | quote }}
+              memory: {{ .Values.resources.api.limits.memory | quote }}
+            requests:
+              cpu: {{ .Values.resources.api.requests.cpu | quote }}
+              memory: {{ .Values.resources.api.requests.memory | quote }}
+          {{- end }}
+          command:
+            - chown
+            - -R
+            - "glance:"
+            - {{ .Values.conf.glance.glance_store.glance.store.filesystem_store_datadir }}
+          volumeMounts:
+            - name: glance-images
+              mountPath: {{ .Values.conf.glance.glance_store.glance.store.filesystem_store_datadir }}
+        {{- end }}
+        {{ if eq .Values.storage "ceph" }}
+        - name: ceph-keyring-placement
+          image: {{ .Values.images.api }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+          command:
+            - /tmp/ceph-keyring.sh
+          volumeMounts:
+            - name: etcceph
+              mountPath: /etc/ceph
+            - name: glance-bin
+              mountPath: /tmp/ceph-keyring.sh
+              subPath: ceph-keyring.sh
+            - name: ceph-keyring
+              mountPath: /tmp/client-keyring
+              subPath: key
+              readOnly: true
+        {{ end }}
       containers:
         - name: glance-api
           image: {{ .Values.images.api }}
@@ -93,13 +134,15 @@ spec:
             - name: glance-images
               mountPath: {{ .Values.conf.glance.glance_store.glance.store.filesystem_store_datadir }}
 {{- else }}
-            - name: glance-etc
+            - name: etcceph
+              mountPath: /etc/ceph
+            - name: ceph-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
               readOnly: true
-            - name: glance-etc
+            - name: ceph-keyring
-              mountPath: /etc/ceph/ceph.client.{{ .Values.conf.glance.glance_store.glance.store.rbd_store_user }}.keyring
+              mountPath: /tmp/client-keyring
-              subPath: ceph.client.{{ .Values.conf.glance.glance_store.glance.store.rbd_store_user }}.keyring
+              subPath: key
               readOnly: true
 {{- end }}
 {{ if $mounts_glance_api.volumeMounts }}{{ toYaml $mounts_glance_api.volumeMounts | indent 12 }}{{ end }}
@@ -117,5 +160,14 @@ spec:
         - name: glance-images
           persistentVolumeClaim:
             claimName: glance-images
+{{ else }}
+        - name: etcceph
+          emptyDir: {}
+        - name: ceph-etc
+          configMap:
+            name: ceph-etc
+        - name: ceph-keyring
+          secret:
+            secretName: pvc-ceph-client-key
 {{- end }}
 {{ if $mounts_glance_api.volumes }}{{ toYaml $mounts_glance_api.volumes | indent 8 }}{{ end }}

glance/templates/etc/_ceph.conf.tpl

@@ -1,30 +0,0 @@
-# Copyright 2017 The Openstack-Helm Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-[global]
-rgw_thread_pool_size = 1024
-rgw_num_rados_handles = 100
-{{- if .Values.conf.ceph.monitors }}
-[mon]
-{{ range .Values.conf.ceph.monitors }}
-    [mon.{{ . }}]
-      host = {{ . }}
-      mon_addr = {{ . }}
-{{ end }}
-{{- else }}
-mon_host = ceph-mon.ceph
-{{- end }}
-[client]
-  rbd_cache_enabled = true
-  rbd_cache_writethrough_until_flush = true

helm-toolkit/utils/secret-generator/README.rst

@@ -1,78 +0,0 @@
-Ceph Kubernetes Secret Generation
-=================================
-
-This script will generate ceph keyrings and configs as Kubernetes
-secrets.
-
-Sigil is required for template handling and must be installed in system
-``PATH``. Instructions can be found`here
-<https://github.com/gliderlabs/sigil>`__
-
-The following functions are provided:
-
-Generate raw FSID (can be used for other functions)
----------------------------------------------------
-
-.. code:: bash
-
-    ./generate_secrets.sh fsid
-
-Generate raw ceph.conf (For verification)
------------------------------------------
-
-.. code:: bash
-
-    ./generate_secrets.sh ceph-conf-raw <fsid> "overridekey=value"
-
-Take a look at ``ceph/ceph.conf.tmpl`` for the default values
-
-Generate encoded ceph.conf secret
----------------------------------
-
-.. code:: bash
-
-    ./generate_secrets.sh ceph-conf <fsid> "overridekey=value"
-
-Generate encoded admin keyring secret
--------------------------------------
-
-.. code:: bash
-
-    ./generate_secrets.sh admin-keyring
-
-Generate encoded mon keyring secret
------------------------------------
-
-.. code:: bash
-
-    ./generate_secrets.sh mon-keyring
-
-Generate a combined secret
---------------------------
-
-Contains ceph.conf, admin keyring and mon keyring. Useful for generating
-the ``/etc/ceph`` directory
-
-.. code:: bash
-
-    ./generate_secrets.sh combined-conf
-
-Generate encoded boostrap keyring secret
-----------------------------------------
-
-.. code:: bash
-
-    ./generate_secrets.sh bootstrap-keyring <osd|mds|rgw>
-
-Kubernetes workflow
-===================
-
-.. code:: bash
-
-    ./generator/generate_secrets.sh all `./generate_secrets.sh fsid`
-
-    kubectl create secret generic ceph-conf-combined --from-file=ceph.conf --from-file=ceph.client.admin.keyring --from-file=ceph.mon.keyring --namespace=ceph
-    kubectl create secret generic ceph-bootstrap-rgw-keyring --from-file=ceph.keyring=ceph.rgw.keyring --namespace=ceph
-    kubectl create secret generic ceph-bootstrap-mds-keyring --from-file=ceph.keyring=ceph.mds.keyring --namespace=ceph
-    kubectl create secret generic ceph-bootstrap-osd-keyring --from-file=ceph.keyring=ceph.osd.keyring --namespace=ceph
-    kubectl create secret generic ceph-client-key --from-file=ceph-client-key --namespace=ceph

helm-toolkit/utils/secret-generator/generate_secrets.sh

@@ -1,96 +0,0 @@
-#!/bin/bash
-
-# Copyright 2017 The Openstack-Helm Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-gen-fsid() {
-  echo "$(uuidgen)"
-}
-
-gen-ceph-conf-raw() {
-  fsid=${1:?}
-  shift
-  conf=$(sigil -p -f templates/ceph/ceph.conf.tmpl "fsid=${fsid}" $@)
-  echo "${conf}"
-}
-
-gen-ceph-conf() {
-  fsid=${1:?}
-  shift
-  conf=$(sigil -p -f templates/ceph/ceph.conf.tmpl "fsid=${fsid}" $@)
-  echo "${conf}"
-}
-
-gen-admin-keyring() {
-  key=$(python ceph-key.py)
-  keyring=$(sigil -f templates/ceph/admin.keyring.tmpl "key=${key}")
-  echo "${keyring}"
-}
-
-gen-mon-keyring() {
-  key=$(python ceph-key.py)
-  keyring=$(sigil -f templates/ceph/mon.keyring.tmpl "key=${key}")
-  echo "${keyring}"
-}
-
-gen-combined-conf() {
-  fsid=${1:?}
-  shift
-  conf=$(sigil -p -f templates/ceph/ceph.conf.tmpl "fsid=${fsid}" $@)
-  echo "${conf}" > ../../secrets/ceph.conf
-
-  key=$(python ceph-key.py)
-  keyring=$(sigil -f templates/ceph/admin.keyring.tmpl "key=${key}")
-  echo "${key}" > ../../secrets/ceph-client-key
-  echo "${keyring}" > ../../secrets/ceph.client.admin.keyring
-
-  key=$(python ceph-key.py)
-  keyring=$(sigil -f templates/ceph/mon.keyring.tmpl "key=${key}")
-  echo "${keyring}" > ../../secrets/ceph.mon.keyring
-}
-
-gen-bootstrap-keyring() {
-  service="${1:-osd}"
-  key=$(python ceph-key.py)
-  bootstrap=$(sigil -f templates/ceph/bootstrap.keyring.tmpl "key=${key}" "service=${service}")
-  echo "${bootstrap}"
-}
-
-gen-all-bootstrap-keyrings() {
-  gen-bootstrap-keyring osd > ../../secrets/ceph.osd.keyring
-  gen-bootstrap-keyring mds > ../../secrets/ceph.mds.keyring
-  gen-bootstrap-keyring rgw > ../../secrets/ceph.rgw.keyring
-}
-
-gen-all() {
-  gen-combined-conf $@
-  gen-all-bootstrap-keyrings
-}
-
-
-main() {
-  set -eo pipefail
-  case "$1" in
-  fsid)            shift; gen-fsid $@;;
-  ceph-conf-raw)            shift; gen-ceph-conf-raw $@;;
-  ceph-conf)            shift; gen-ceph-conf $@;;
-  admin-keyring)            shift; gen-admin-keyring $@;;
-  mon-keyring)            shift; gen-mon-keyring $@;;
-  bootstrap-keyring)            shift; gen-bootstrap-keyring $@;;
-  combined-conf)               shift; gen-combined-conf $@;;
-  all)                         shift; gen-all $@;;
-  esac
-}
-
-main "$@"

helm-toolkit/utils/secret-generator/templates/ceph/admin.keyring.tmpl

@@ -1,20 +0,0 @@
-# Copyright 2017 The Openstack-Helm Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-[client.admin]
-  key = {{ $key }}
-  auid = 0
-  caps mds = "allow"
-  caps mon = "allow *"
-  caps osd = "allow *"

helm-toolkit/utils/secret-generator/templates/ceph/bootstrap.keyring.tmpl

@@ -1,17 +0,0 @@
-# Copyright 2017 The Openstack-Helm Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-[client.bootstrap-{{ $service }}]
-  key = {{ $key }}
-  caps mon = "allow profile bootstrap-{{ $service }}"

helm-toolkit/utils/secret-generator/templates/ceph/ceph.conf.tmpl

@@ -1,85 +0,0 @@
-# Copyright 2017 The Openstack-Helm Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-[global]
-fsid = ${fsid:?}
-cephx = ${auth_cephx:-"true"}
-cephx_require_signatures = ${auth_cephx_require_signatures:-"false"}
-cephx_cluster_require_signatures = ${auth_cephx_cluster_require_signatures:-"true"}
-cephx_service_require_signatures = ${auth_cephx_service_require_signatures:-"false"}
-
-# auth
-max_open_files = ${global_max_open_files:-"131072"}
-osd_pool_default_pg_num = ${global_osd_pool_default_pg_num:-"128"}
-osd_pool_default_pgp_num = ${global_osd_pool_default_pgp_num:-"128"}
-osd_pool_default_size = ${global_osd_pool_default_size:-"3"}
-osd_pool_default_min_size = ${global_osd_pool_default_min_size:-"1"}
-
-mon_osd_full_ratio = ${global_mon_osd_full_ratio:-".95"}
-mon_osd_nearfull_ratio = ${global_mon_osd_nearfull_ratio:-".85"}
-
-mon_host = ${global_mon_host:-'ceph-mon'}
-
-[mon]
-mon_osd_down_out_interval = ${mon_mon_osd_down_out_interval:-"600"}
-mon_osd_min_down_reporters = ${mon_mon_osd_min_down_reporters:-"4"}
-mon_clock_drift_allowed = ${mon_mon_clock_drift_allowed:-".15"}
-mon_clock_drift_warn_backoff = ${mon_mon_clock_drift_warn_backoff:-"30"}
-mon_osd_report_timeout = ${mon_mon_osd_report_timeout:-"300"}
-
-
-[osd]
-journal_size = ${osd_journal_size:-"100"}
-cluster_network = ${osd_cluster_network:-'192.168.0.0/16'}
-public_network = ${osd_public_network:-'192.168.0.0/16'}
-osd_mkfs_type = ${osd_osd_mkfs_type:-"xfs"}
-osd_mkfs_options_xfs = ${osd_osd_mkfs_options_xfs:-"-f -i size=2048"}
-osd_mon_heartbeat_interval = ${osd_osd_mon_heartbeat_interval:-"30"}
-osd_max_object_name_len = ${osd_max_object_name_len:-"256"}
-
-#crush
-osd_pool_default_crush_rule = ${osd_pool_default_crush_rule:-"0"}
-osd_crush_update_on_start = ${osd_osd_crush_update_on_start:-"true"}
-
-#backend
-osd_objectstore = ${osd_osd_objectstore:-"filestore"}
-
-#performance tuning
-filestore_merge_threshold = ${osd_filestore_merge_threshold:-"40"}
-filestore_split_multiple = ${osd_filestore_split_multiple:-"8"}
-osd_op_threads = ${osd_osd_op_threads:-"8"}
-filestore_op_threads = ${osd_filestore_op_threads:-"8"}
-filestore_max_sync_interval = ${osd_filestore_max_sync_interval:-"5"}
-osd_max_scrubs = ${osd_osd_max_scrubs:-"1"}
-
-
-#recovery tuning
-osd_recovery_max_active = ${osd_osd_recovery_max_active:-"5"}
-osd_max_backfills = ${osd_osd_max_backfills:-"2"}
-osd_recovery_op_priority = ${osd_osd_recovery_op_priority:-"2"}
-osd_client_op_priority = ${osd_osd_client_op_priority:-"63"}
-osd_recovery_max_chunk = ${osd_osd_recovery_max_chunk:-"1048576"}
-osd_recovery_threads = ${osd_osd_recovery_threads:-"1"}
-
-#ports
-ms_bind_port_min = ${osd_ms_bind_port_min:-"6800"}
-ms_bind_port_max = ${osd_ms_bind_port_max:-"7100"}
-
-[client]
-rbd_cache_enabled = ${client_rbd_cache_enabled:-"true"}
-rbd_cache_writethrough_until_flush = ${client_rbd_cache_writethrough_until_flush:-"true"}
-rbd_default_features = ${client_rbd_default_features:-"1"}
-
-[mds]
-mds_cache_size = ${mds_mds_cache_size:-"100000"}

helm-toolkit/utils/secret-generator/templates/ceph/mon.keyring.tmpl

@@ -1,17 +0,0 @@
-# Copyright 2017 The Openstack-Helm Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-[mon.]
-  key = {{ $key }}
-  caps mon = "allow *"

helm-toolkit/utils/test/ceph-rbd-test.yaml

@@ -1,40 +0,0 @@
-# Copyright 2017 The Openstack-Helm Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-apiVersion: v1
-kind: Pod
-metadata:
-  name: ceph-rbd-test
-spec:
-  containers:
-  - name: cephrbd-rw
-    image: busybox
-    command:
-    - sh
-    - -c
-    - while true; do sleep 1; done
-    volumeMounts:
-    - mountPath: "/mnt/cephrbd"
-      name: cephrbd
-  volumes:
-  - name: cephrbd
-    rbd:
-      monitors:
-#This only works if you have skyDNS resolveable from the kubernetes node. Otherwise you must manually put in one or more mon pod ips.
-      - ceph-mon.ceph:6789
-      user: admin
-      image: ceph-rbd-test
-      pool: rbd
-      secretRef:
-        name: ceph-client-key

nova/templates/bin/_ceph-keyring.sh.tpl

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 # Copyright 2017 The Openstack-Helm Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,9 +14,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -ex
+export HOME=/tmp
+
+cat <<EOF > /etc/ceph/ceph.client.keyring
 [client.{{ .Values.ceph.cinder_user }}]
 {{- if .Values.ceph.cinder_keyring }}
     key = {{ .Values.ceph.cinder_keyring }}
 {{- else }}
-    key = {{- include "secrets/ceph-client-key" . -}}
+    key = $(cat /tmp/client-keyring)
 {{- end }}
+EOF
+
+exit 0

nova/templates/configmap-bin.yaml

@@ -33,7 +33,7 @@ data:
 {{ tuple "bin/_libvirt.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- if .Values.ceph.enabled }}
   ceph-secret-define.sh: |
-{{ tuple "bin/_ceph-secret-define.sh.tpl" . | include "helm-toolkit.template" | indent 4 }}
+{{ tuple "bin/_ceph-secret-define.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}
   bootstrap.sh: |
 {{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
@@ -51,3 +51,5 @@ data:
 {{ tuple "bin/_nova-scheduler.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   fake-iptables.sh: |
 {{ tuple "bin/_fake-iptables.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
+  ceph-keyring.sh: |+
+{{ tuple "bin/_ceph-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

nova/templates/configmap-etc.yaml

@@ -92,8 +92,6 @@ data:
 {{- else -}}
 {{ tuple "etc/_policy.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}
-  ceph.client.cinder.keyring.yaml: |+
-{{ tuple "etc/_ceph.client.cinder.keyring.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   resolv.conf: |+
 {{ tuple "etc/_resolv.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   libvirtd.conf: |+

nova/templates/daemonset-compute.yaml

@@ -39,6 +39,23 @@ spec:
       dnsPolicy: ClusterFirst
       initContainers:
 {{ tuple $envAll $dependencies $mounts_nova_compute_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+        {{- if .Values.ceph.enabled }}
+        - name: ceph-keyring-placement
+          image: {{ .Values.images.compute }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+          command:
+            - /tmp/ceph-keyring.sh
+          volumeMounts:
+            - name: etcceph
+              mountPath: /etc/ceph
+            - name: nova-bin
+              mountPath: /tmp/ceph-keyring.sh
+              subPath: ceph-keyring.sh
+            - name: ceph-keyring
+              mountPath: /tmp/client-keyring
+              subPath: key
+              readOnly: true
+        {{ end }}
       containers:
         - name: nova-compute
           image: {{ .Values.images.compute }}
@@ -77,14 +94,18 @@ spec:
               mountPath: /etc/resolv.conf
               subPath: resolv.conf
               readOnly: true
-            - name: nova-etc
+            {{- if .Values.ceph.enabled }}
+            - name: etcceph
+              mountPath: /etc/ceph
+            - name: ceph-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
               readOnly: true
-            - name: nova-etc
+            - name: ceph-keyring
-              mountPath: /etc/ceph/ceph.client.keyring
+              mountPath: /tmp/client-keyring
-              subPath: ceph.client.keyring
+              subPath: key
               readOnly: true
+            {{ end }}
             - mountPath: /lib/modules
               name: libmodules
               readOnly: true
@@ -105,6 +126,16 @@ spec:
         - name: nova-etc
           configMap:
             name: nova-etc
+        {{- if .Values.ceph.enabled }}
+        - name: etcceph
+          emptyDir: {}
+        - name: ceph-etc
+          configMap:
+            name: ceph-etc
+        - name: ceph-keyring
+          secret:
+            secretName: pvc-ceph-client-key
+        {{ end }}
         - name: libmodules
           hostPath:
             path: /lib/modules

nova/templates/daemonset-libvirt.yaml

@@ -38,6 +38,23 @@ spec:
       dnsPolicy: ClusterFirst
       initContainers:
 {{ tuple $envAll $dependencies $mounts_nova_libvirt_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+        {{- if .Values.ceph.enabled }}
+        - name: ceph-keyring-placement
+          image: {{ .Values.images.libvirt }}
+          imagePullPolicy: {{ .Values.images.pull_policy }}
+          command:
+            - /tmp/ceph-keyring.sh
+          volumeMounts:
+            - name: etcceph
+              mountPath: /etc/ceph
+            - name: nova-bin
+              mountPath: /tmp/ceph-keyring.sh
+              subPath: ceph-keyring.sh
+            - name: ceph-keyring
+              mountPath: /tmp/client-keyring
+              subPath: key
+              readOnly: true
+        {{ end }}
       containers:
         - name: nova-libvirt
           image: {{ .Values.images.libvirt }}
@@ -101,13 +118,15 @@ spec:
             - name: cgroup
               mountPath: /sys/fs/cgroup
             {{- if .Values.ceph.enabled }}
-            - name: nova-etc
+            - name: etcceph
+              mountPath: /etc/ceph
+            - name: ceph-etc
               mountPath: /etc/ceph/ceph.conf
               subPath: ceph.conf
               readOnly: true
-            - name: nova-etc
+            - name: ceph-keyring
-              mountPath: /etc/ceph/ceph.client.keyring
+              mountPath: /tmp/client-keyring
-              subPath: ceph.client.keyring
+              subPath: key
               readOnly: true
             - name: nova-bin
               mountPath: /tmp/ceph-secret-define.sh
@@ -123,6 +142,16 @@ spec:
         - name: nova-etc
           configMap:
             name: nova-etc
+        {{- if .Values.ceph.enabled }}
+        - name: etcceph
+          emptyDir: {}
+        - name: ceph-etc
+          configMap:
+            name: ceph-etc
+        - name: ceph-keyring
+          secret:
+            secretName: pvc-ceph-client-key
+        {{ end }}
         - name: libmodules
           hostPath:
             path: /lib/modules

nova/templates/etc/_ceph.conf.tpl

@@ -1,32 +0,0 @@
-# Copyright 2017 The Openstack-Helm Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-[global]
-rgw_thread_pool_size = 1024
-rgw_num_rados_handles = 100
-{{- if .Values.ceph.enabled }}
-[mon]
-{{- if .Values.ceph.monitors }}
-{{ range .Values.ceph.monitors }}
-    [mon.{{ . }}]
-      host = {{ . }}
-      mon_addr = {{ . }}
-{{ end }}
-{{- else }}
-mon_host = ceph-mon.ceph
-{{- end }}
-{{- end }}
-[client]
-  rbd_cache_enabled = true
-  rbd_cache_writethrough_until_flush = true

nova/values.yaml

@@ -130,7 +130,6 @@ keystone:
   admin_region_name: "RegionOne"
   nova_user_role: "admin"
 
-
 ceph:
   enabled: false
   monitors: []

tests/pvc-test.yaml

@@ -12,15 +12,42 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  labels:
+    version: v0.1.0
+    test: ceph
+  name: ceph-test-job
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: test
+          image: docker.io/alpine:latest
+          imagePullPolicy: Always
+          command:
+            - /bin/sh
+            - -ec
+            - |
+              echo "Ceph PVC Mount Test Passed"
+          volumeMounts:
+            - name: ceph-mount
+              mountPath: /mnt/ceph
+      volumes:
+        - name: ceph-mount
+          persistentVolumeClaim:
+            claimName: ceph-test
 ---
 kind: PersistentVolumeClaim
 apiVersion: v1
 metadata:
-  name: pvc-test
+  name: ceph-test
 spec:
-  accessModes:
+  storageClassName: general
-    - ReadWriteOnce
+  accessModes: [ "ReadWriteOnce" ]
   resources:
     requests:
       storage: 1Gi
-  storageClassName: general

tools/gate/README.rst

@@ -17,6 +17,7 @@ integration test is below:
 
     export INTEGRATION=aio
     export INTEGRATION_TYPE=basic
+    export PVC_BACKEND=ceph
     ./tools/gate/setup_gate.sh
 
 Supported Platforms

tools/gate/basic_launch.sh

@@ -29,14 +29,53 @@ if [ "x$HOST_OS" == "xfedora" ]; then
   sudo modprobe ip6_tables
 fi
 
+if [ "x$PVC_BACKEND" == "xceph" ]; then
+  kubectl label nodes ceph-storage=enabled --all
+  CONTROLLER_MANAGER_POD=$(kubectl get -n kube-system pods -l component=kube-controller-manager --no-headers -o name | head -1 | awk -F '/' '{ print $NF }')
+  kubectl exec -n kube-system ${CONTROLLER_MANAGER_POD} -- sh -c "cat > /etc/resolv.conf <<EOF
+nameserver 10.96.0.10
+nameserver 8.8.8.8
+search cluster.local svc.cluster.local
+EOF"
+
+  export osd_cluster_network=192.168.0.0/16
+  export osd_public_network=192.168.0.0/16
+
+  helm install --namespace=ceph local/ceph --name=ceph2 \
+    --set manifests_enabled.client_secrets=false \
+    --set network.public=$osd_public_network \
+    --set network.cluster=$osd_cluster_network
+
+  kube_wait_for_pods ceph 420
+
+  kubectl exec -n ceph ceph-mon-0 -- ceph -s
+
+  helm install --namespace=openstack local/ceph --name=ceph-openstack-config \
+    --set manifests_enabled.storage_secrets=false \
+    --set manifests_enabled.deployment=false \
+    --set ceph.namespace=ceph \
+    --set network.public=$osd_public_network \
+    --set network.cluster=$osd_cluster_network
+
+  kube_wait_for_pods ceph 420
+
+  kubectl exec -n ceph ceph-mon-0 -- ceph osd pool create volumes 8
+  kubectl exec -n ceph ceph-mon-0 -- ceph osd pool create images 8
+  kubectl exec -n ceph ceph-mon-0 -- ceph osd pool create vms 8
+fi
+
 helm install --namespace=openstack local/mariadb --name=mariadb
 helm install --namespace=openstack local/memcached --name=memcached
 helm install --namespace=openstack local/etcd --name=etcd-rabbitmq
 helm install --namespace=openstack local/rabbitmq --name=rabbitmq
 kube_wait_for_pods openstack 420
 helm install --namespace=openstack local/keystone --name=keystone
-helm install --namespace=openstack local/glance --name=glance \
+if [ "x$PVC_BACKEND" == "xceph" ]; then
-    --values=${WORK_DIR}/tools/overrides/mvp/glance.yaml
+  helm install --namespace=openstack local/glance --name=glance
+else
+  helm install --namespace=openstack local/glance --name=glance \
+      --values=${WORK_DIR}/tools/overrides/mvp/glance.yaml
+fi
 kube_wait_for_pods openstack 420
 helm install --namespace=openstack local/nova --name=nova \
     --values=${WORK_DIR}/tools/overrides/mvp/nova.yaml \
@@ -44,7 +83,12 @@ helm install --namespace=openstack local/nova --name=nova \
 helm install --namespace=openstack local/neutron --name=neutron \
     --values=${WORK_DIR}/tools/overrides/mvp/neutron.yaml
 kube_wait_for_pods openstack 420
-helm install --namespace=openstack local/cinder --name=cinder
+if [ "x$PVC_BACKEND" == "xceph" ]; then
+  helm install --namespace=openstack local/cinder --name=cinder
+else
+  helm install --namespace=openstack local/cinder --name=cinder \
+      --values=${WORK_DIR}/tools/overrides/mvp/cinder.yaml
+fi
 helm install --namespace=openstack local/heat --name=heat
 helm install --namespace=openstack local/horizon --name=horizon
 kube_wait_for_pods openstack 420

tools/gate/dump_logs.sh

@@ -53,6 +53,15 @@ kubectl get svc -o json --all-namespaces | jq -r \
     ${LOGS_DIR}/k8s/svc/$NAMESPACE-$NAME.txt
 done
 
+mkdir -p ${LOGS_DIR}/k8s/pvc
+kubectl get pvc -o json --all-namespaces | jq -r \
+  '.items[].metadata | .namespace + " " + .name' | while read line; do
+  NAMESPACE=$(echo $line | awk '{print $1}')
+  NAME=$(echo $line | awk '{print $2}')
+  kubectl describe pvc $NAME --namespace $NAMESPACE > \
+    ${LOGS_DIR}/k8s/pvc/$NAMESPACE-$NAME.txt
+done
+
 mkdir -p ${LOGS_DIR}/k8s/rbac
 for OBJECT_TYPE in clusterroles \
                    roles \
@@ -76,5 +85,6 @@ sudo iptables-save > ${LOGS_DIR}/nodes/$(hostname)/iptables.txt
 sudo ip a > ${LOGS_DIR}/nodes/$(hostname)/ip.txt
 sudo route -n > ${LOGS_DIR}/nodes/$(hostname)/routes.txt
 arp -a > ${LOGS_DIR}/nodes/$(hostname)/arp.txt
+cat /etc/resolv.conf > ${LOGS_DIR}/nodes/$(hostname)/resolv.conf
 
 exit $1

tools/gate/funcs/common.sh

@@ -29,3 +29,17 @@ function base_install {
       iptables
   fi
 }
+
+function ceph_support_install {
+  if [ "x$HOST_OS" == "xubuntu" ]; then
+    sudo apt-get update -y
+    sudo apt-get install -y --no-install-recommends -qq \
+      ceph-common
+  elif [ "x$HOST_OS" == "xcentos" ]; then
+    sudo yum install -y \
+      ceph
+  elif [ "x$HOST_OS" == "xfedora" ]; then
+    sudo dnf install -y \
+      ceph
+  fi
+}

tools/gate/funcs/kube.sh

@@ -130,3 +130,9 @@ function kubeadm_aio_launch {
   kube_wait_for_pods kube-system 240
   kube_wait_for_pods default 240
 }
+
+function ceph_kube_controller_manager_replace {
+  sudo docker pull ${CEPH_KUBE_CONTROLLER_MANAGER_IMAGE}
+  IMAGE_ID=$(sudo docker images ${CEPH_KUBE_CONTROLLER_MANAGER_IMAGE} -q)
+  sudo docker tag ${IMAGE_ID} ${BASE_KUBE_CONTROLLER_MANAGER_IMAGE}
+}

tools/gate/funcs/network.sh

@@ -42,3 +42,4 @@ function net_hosts_pre_kube {
 function net_hosts_post_kube {
   sudo cp -f /etc/hosts-pre-kube /etc/hosts
 }
+

tools/gate/kubeadm_aio.sh

@@ -18,5 +18,11 @@ source ${WORK_DIR}/tools/gate/funcs/kube.sh
 
 kubeadm_aio_reqs_install
 sudo docker pull ${KUBEADM_IMAGE} || kubeadm_aio_build
+
+if [ "x$PVC_BACKEND" == "xceph" ]; then
+  ceph_kube_controller_manager_replace
+  sudo modprobe rbd
+fi
+
 kubeadm_aio_launch
 net_resolv_kube

tools/gate/provision_gate_worker_node.sh

@@ -37,8 +37,15 @@ sudo rm -rfv \
     /var/lib/etcd \
     /var/etcd \
     /var/lib/kubelet/* \
+    /var/lib/nova \
+    /var/lib/openstack-helm \
     /run/openvswitch || true
 
+# Load ceph kernel module if required
+if [ "x$PVC_BACKEND" == "xceph" ]; then
+  sudo modprobe rbd
+fi
+
 # Launch Container
 sudo docker run \
     -dt \

tools/gate/setup_gate.sh

@@ -13,10 +13,12 @@
 # limitations under the License.
 set -ex
 
-export HELM_VERSION=${2:-v2.3.1}
+export HELM_VERSION=${2:-v2.4.1}
 export KUBE_VERSION=${3:-v1.6.5}
 export KUBECONFIG=${HOME}/.kubeadm-aio/admin.conf
-export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:${KUBE_VERSION}
+export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:${KUBE_VERSION}-ceph
+export BASE_KUBE_CONTROLLER_MANAGER_IMAGE=gcr.io/google_containers/kube-controller-manager-amd64:${KUBE_VERSION}
+export CEPH_KUBE_CONTROLLER_MANAGER_IMAGE=quay.io/attcomdev/kube-controller-manager:${KUBE_VERSION}
 
 export WORK_DIR=$(pwd)
 source /etc/os-release
@@ -24,6 +26,7 @@ export HOST_OS=${ID}
 source ${WORK_DIR}/tools/gate/funcs/common.sh
 source ${WORK_DIR}/tools/gate/funcs/network.sh
 source ${WORK_DIR}/tools/gate/funcs/helm.sh
+export PVC_BACKEND=ceph
 
 # Setup the logging location: by default use the working dir as the root.
 export LOGS_DIR=${LOGS_DIR:-"${WORK_DIR}/logs"}
@@ -34,14 +37,17 @@ function dump_logs () {
 }
 trap 'dump_logs "$?"' ERR
 
-# Install base requirements
-base_install
-
 # Moving the ws-linter here to avoid it blocking all the jobs just for ws
 if [ "x$INTEGRATION_TYPE" == "xlinter" ]; then
   bash ${WORK_DIR}/tools/gate/whitespace.sh
 fi
 
+# Install base requirements
+base_install
+if [ "x$PVC_BACKEND" == "xceph" ]; then
+  ceph_support_install
+fi
+
 # We setup the network for pre kube here, to enable cluster restarts on
 # development machines
 net_resolv_pre_kube

tools/gate/setup_gate_worker_nodes.sh

@@ -31,6 +31,7 @@ cat /etc/nodepool/sub_nodes_private | while read SUB_NODE; do
     export KUBEADM_TOKEN=${KUBEADM_TOKEN}; \
     export PRIMARY_NODE_IP=${PRIMARY_NODE_IP}; \
     export KUBEADM_IMAGE=${KUBEADM_IMAGE}; \
+    export PVC_BACKEND=${PVC_BACKEND}; \
     bash ${WORK_DIR}/tools/gate/provision_gate_worker_node.sh"
 EOS
 done

tools/images/ceph-config-helper/Dockerfile

@@ -0,0 +1,20 @@
+FROM ubuntu:16.04
+MAINTAINER pete.birley@att.com
+
+ARG KUBE_VERSION=v1.6.5
+
+RUN set -x \
+    && TMP_DIR=$(mktemp --directory) \
+    && cd ${TMP_DIR} \
+    && apt-get update \
+    && apt-get install -y \
+        apt-transport-https \
+        ca-certificates \
+        curl \
+        python \
+        jq \
+# Install kubectl:
+    && curl -sSL https://dl.k8s.io/${KUBE_VERSION}/kubernetes-client-linux-amd64.tar.gz | tar -zxv --strip-components=1 \
+    && mv ${TMP_DIR}/client/bin/kubectl /usr/bin/kubectl \
+    && chmod +x /usr/bin/kubectl \
+    && rm -rf ${TMP_DIR}

tools/images/ceph-config-helper/README.rst

@@ -0,0 +1,39 @@
+Ceph Config Helper Container
+=====================
+
+This container builds a small image with kubectl and some other utilites for
+use in the ceph-config chart.
+
+Instructions
+------------
+
+OS Specific Host setup:
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Ubuntu:
+^^^^^^^aa
+
+From a freshly provisioned Ubuntu 16.04 LTS host run:
+
+.. code:: bash
+
+    sudo apt-get update -y
+    sudo apt-get install -y \
+            docker.io \
+            git
+
+Build the Ceph-Helper Image environment (optional)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A known good image is published to dockerhub on a fairly regular basis, but if
+you wish to build your own image, from the root directory of the OpenStack-Helm
+repo run:
+
+.. code:: bash
+
+    export KUBE_VERSION=v1.6.5
+    sudo docker build \
+      --build-arg KUBE_VERSION=${KUBE_VERSION} \
+      -t docker.io/port/ceph-config-helper:${KUBE_VERSION} \
+      tools/images/ceph-config-helper
+    sudo docker push docker.io/port/ceph-config-helper:${KUBE_VERSION}

tools/kubeadm-aio/Dockerfile

@@ -61,8 +61,9 @@ RUN set -x \
         git \
         vim \
         jq \
-# Install nfs utils for development PVC provisioner
+# Install utils for PVC provisioners
         nfs-common \
+        ceph-common \
 # Tweak Systemd units and targets for running in a container
     && find /lib/systemd/system/sysinit.target.wants/ ! -name 'systemd-tmpfiles-setup.service' -type l -exec rm -fv {} + \
     && rm -fv \

tools/kubeadm-aio/README.rst

@@ -57,7 +57,7 @@ repo run:
 
 .. code:: bash
 
-    export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:v1.6.4
+    export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:v1.6.5
     sudo docker build --pull -t ${KUBEADM_IMAGE} tools/kubeadm-aio
 
 Deploy the AIO environment

tools/kubeadm-aio/assets/usr/bin/kubelet

@@ -36,7 +36,8 @@ else
         --restart=always \
         --volume=/sys/fs/cgroup:/sys/fs/cgroup:ro \
         --volume=/:/rootfs:ro \
-        --volume=/dev/net:/dev/net:rw \
+        --volume=/dev:/dev:rshared \
+        --volume=/lib/modules:/lib/modules:ro \
         --volume=/var/run/netns:/var/run/netns:rw \
         --volume=/sys:/sys:ro \
         --volume=/etc/machine-id:/etc/machine-id:ro \

tools/kubeadm-aio/kubeadm-aio-launcher.sh

@@ -33,6 +33,7 @@ sudo rm -rfv \
     /run/openvswitch \
     /var/lib/nova \
     ${HOME}/.kubeadm-aio/admin.conf \
+    /var/lib/openstack-helm \
     /var/lib/nfs-provisioner || true
 
 # Launch Container

tools/overrides/mvp/cinder.yaml

@@ -1,5 +1,3 @@
-#!/bin/python
-
 # Copyright 2017 The Openstack-Helm Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,17 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
+# MVP values for glance.
-import struct
+# This file contains overrides to launch a MVP deployment of glance for the
-import time
+# OpenStack-Helm Single node gates, and local development use. It should be
-import base64
+# kept to the bare minimum required for this purpose.
+
+storage: "gate"
+
+conf:
+  cinder:
+    default:
+      cinder:
+        backup_driver: "cinder.backup.drivers.nfs"
+  backends:
+    rbd1:
+      volume_driver: "cinder.volume.drivers.nfs.NfsDriver"
 
-key = os.urandom(16)
-header = struct.pack(
-    '<hiih',
-    1,                 # le16 type: CEPH_CRYPTO_AES
-    int(time.time()),  # le32 created: seconds
-    0,                 # le32 created: nanoseconds,
-    len(key),          # le16: len(key)
-)
-print(base64.b64encode(header + key).decode('ascii'))

tools/overrides/mvp/nova.yaml

@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+ceph:
+  enabled: false
+
 conf:
   nova:
     default: