Refactor Ceph secret generation
This PS refactors the ceph chart and secret generation process. The updated chart replaces the existing "bootstrap" chart. Additionally, Ceph manifests and deployment guides were modified accordingly. Change-Id: I6f5bb88fc0f40cfee8865d9dab83859d765e7537 Co-Authored-By: Larry Rensing <lr699s@att.com>
This commit is contained in:
Pete Birley
parent
efa8293e54
commit
8ef5d94674
4
Makefile
4
Makefile
|
|
@ -15,8 +15,8 @@
|
|||
HELM = helm
|
||||
TASK = build
|
||||
|
||||
CHARTS = helm-toolkit bootstrap ceph mariadb etcd rabbitmq memcached
|
||||
CHARTS += keystone glance cinder horizon neutron nova heat
|
||||
CHARTS = helm-toolkit ceph mariadb etcd rabbitmq
|
||||
CHARTS += memcached keystone glance cinder horizon neutron nova heat
|
||||
CHARTS += barbican mistral senlin magnum ingress
|
||||
|
||||
all: $(CHARTS)
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
secrets/*
|
||||
!secrets/.gitkeep
|
||||
templates/_secrets.tpl
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
# Patterns to ignore when building packages.
|
||||
# This supports shell glob matching, relative path matching, and
|
||||
# negation (prefixed with !). Only one pattern per line.
|
||||
.DS_Store
|
||||
# Common VCS dirs
|
||||
.git/
|
||||
.gitignore
|
||||
.bzr/
|
||||
.bzrignore
|
||||
.hg/
|
||||
.hgignore
|
||||
.svn/
|
||||
# Common backup files
|
||||
*.swp
|
||||
*.bak
|
||||
*.tmp
|
||||
*~
|
||||
# Various IDEs
|
||||
.project
|
||||
.idea/
|
||||
*.tmproj
|
||||
|
||||
bin/
|
||||
etc/
|
||||
patches/
|
||||
*.py
|
||||
Makefile
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
# Copyright 2017 The Openstack-Helm Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
apiVersion: v1
|
||||
description: OpenStack-Helm namespace bootstrap
|
||||
name: bootstrap
|
||||
version: 0.1.0
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
# Copyright 2017 The Openstack-Helm Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
dependencies:
|
||||
- name: helm-toolkit
|
||||
repository: http://localhost:8879/charts
|
||||
version: 0.1.0
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
# Copyright 2017 The Openstack-Helm Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Default values for bootstrap.
|
||||
# This is a YAML-formatted file.
|
||||
# Declare name/value pairs to be passed into your templates.
|
||||
# name: value
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
#!/bin/python
|
||||
import os
|
||||
import struct
|
||||
import time
|
||||
import base64
|
||||
key = os.urandom(16)
|
||||
header = struct.pack(
|
||||
'<hiih',
|
||||
1, # le16 type: CEPH_CRYPTO_AES
|
||||
int(time.time()), # le32 created: seconds
|
||||
0, # le32 created: nanoseconds,
|
||||
len(key), # le16: len(key)
|
||||
)
|
||||
print(base64.b64encode(header + key).decode('ascii'))
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
#!/bin/bash
|
||||
set -ex
|
||||
|
||||
function ceph_gen_key () {
|
||||
python ${CEPH_GEN_DIR}/ceph-key.py
|
||||
}
|
||||
|
||||
function kube_ceph_keyring_gen () {
|
||||
CEPH_KEY=$1
|
||||
CEPH_KEY_TEMPLATE=$2
|
||||
sed "s|{{"{{"}} key {{"}}"}}|${CEPH_KEY}|" ${CEPH_TEMPLATES_DIR}/${CEPH_KEY_TEMPLATE} | base64 | tr -d '\n'
|
||||
}
|
||||
|
||||
function create_kube_key () {
|
||||
CEPH_KEYRING=$1
|
||||
CEPH_KEYRING_NAME=$2
|
||||
CEPH_KEYRING_TEMPLATE=$3
|
||||
KUBE_SECRET_NAME=$4
|
||||
if ! kubectl get --namespace ${DEPLOYMENT_NAMESPACE} secrets ${KUBE_SECRET_NAME}; then
|
||||
{
|
||||
cat <<EOF
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: ${KUBE_SECRET_NAME}
|
||||
type: Opaque
|
||||
data:
|
||||
${CEPH_KEYRING_NAME}: |
|
||||
$( kube_ceph_keyring_gen ${CEPH_KEYRING} ${CEPH_KEYRING_TEMPLATE} )
|
||||
EOF
|
||||
} | kubectl create --namespace ${DEPLOYMENT_NAMESPACE} -f -
|
||||
fi
|
||||
}
|
||||
|
||||
#create_kube_key <ceph_key> <ceph_keyring_name> <ceph_keyring_template> <kube_secret_name>
|
||||
create_kube_key $(ceph_gen_key) ${CEPH_KEYRING_NAME} ${CEPH_KEYRING_TEMPLATE} ${KUBE_SECRET_NAME}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
#!/bin/bash
|
||||
set -ex
|
||||
|
||||
ceph_activate_namespace() {
|
||||
kube_namespace=$1
|
||||
{
|
||||
cat <<EOF
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: "${PVC_CEPH_STORAGECLASS_USER_SECRET_NAME}"
|
||||
type: kubernetes.io/rbd
|
||||
data:
|
||||
key: |
|
||||
$(kubectl get secret ${PVC_CEPH_STORAGECLASS_ADMIN_SECRET_NAME} \
|
||||
--namespace=${PVC_CEPH_STORAGECLASS_DEPLOYED_NAMESPACE} \
|
||||
-o json | jq -r '.data | .[]')
|
||||
EOF
|
||||
} | kubectl create --namespace ${kube_namespace} -f -
|
||||
}
|
||||
|
||||
ceph_activate_namespace ${DEPLOYMENT_NAMESPACE}
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
#!/bin/bash
|
||||
set -ex
|
||||
|
||||
function ceph_gen_key () {
|
||||
python ${CEPH_GEN_DIR}/ceph-key.py
|
||||
}
|
||||
|
||||
function kube_ceph_keyring_gen () {
|
||||
CEPH_KEY=$1
|
||||
CEPH_KEY_TEMPLATE=$2
|
||||
sed "s|{{"{{"}} key {{"}}"}}|${CEPH_KEY}|" ${CEPH_TEMPLATES_DIR}/${CEPH_KEY_TEMPLATE} | base64 | tr -d '\n'
|
||||
}
|
||||
|
||||
CEPH_CLIENT_KEY=$(ceph_gen_key)
|
||||
|
||||
function create_kube_key () {
|
||||
CEPH_KEYRING=$1
|
||||
CEPH_KEYRING_NAME=$2
|
||||
CEPH_KEYRING_TEMPLATE=$3
|
||||
KUBE_SECRET_NAME=$4
|
||||
|
||||
if ! kubectl get --namespace ${DEPLOYMENT_NAMESPACE} secrets ${KUBE_SECRET_NAME}; then
|
||||
{
|
||||
cat <<EOF
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: ${KUBE_SECRET_NAME}
|
||||
type: Opaque
|
||||
data:
|
||||
${CEPH_KEYRING_NAME}: |
|
||||
$( kube_ceph_keyring_gen ${CEPH_KEYRING} ${CEPH_KEYRING_TEMPLATE} )
|
||||
EOF
|
||||
} | kubectl create --namespace ${DEPLOYMENT_NAMESPACE} -f -
|
||||
fi
|
||||
}
|
||||
#create_kube_key <ceph_key> <ceph_keyring_name> <ceph_keyring_template> <kube_secret_name>
|
||||
create_kube_key ${CEPH_CLIENT_KEY} ${CEPH_KEYRING_NAME} ${CEPH_KEYRING_TEMPLATE} ${CEPH_KEYRING_ADMIN_NAME}
|
||||
|
||||
function create_kube_storage_key () {
|
||||
CEPH_KEYRING=$1
|
||||
KUBE_SECRET_NAME=$2
|
||||
|
||||
if ! kubectl get --namespace ${DEPLOYMENT_NAMESPACE} secrets ${KUBE_SECRET_NAME}; then
|
||||
{
|
||||
cat <<EOF
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: ${KUBE_SECRET_NAME}
|
||||
type: kubernetes.io/rbd
|
||||
data:
|
||||
key: |
|
||||
$( echo ${CEPH_KEYRING} | base64 | tr -d '\n' )
|
||||
EOF
|
||||
} | kubectl create --namespace ${DEPLOYMENT_NAMESPACE} -f -
|
||||
fi
|
||||
}
|
||||
#create_kube_storage_key <ceph_key> <kube_secret_name>
|
||||
create_kube_storage_key ${CEPH_CLIENT_KEY} ${CEPH_STORAGECLASS_ADMIN_SECRET_NAME}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
# Copyright 2017 The Openstack-Helm Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: ceph-bin
|
||||
data:
|
||||
{{- if .Values.manifests_enabled.storage_secrets }}
|
||||
ceph-key.py: |+
|
||||
{{ tuple "bin/_ceph-key.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
ceph-key.sh: |+
|
||||
{{ tuple "bin/_ceph-key.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
ceph-storage-key.sh: |+
|
||||
{{ tuple "bin/_ceph-storage-key.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
{{- end }}
|
||||
{{- if .Values.manifests_enabled.client_secrets }}
|
||||
ceph-namespace-client-key.sh: |+
|
||||
{{ tuple "bin/_ceph-namespace-client-key.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
{{- end }}
|
||||
|
|
@ -12,21 +12,18 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
{{- if empty .Values.conf.ceph.config.global.mon_host -}}
|
||||
{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_endpoint_lookup" }}
|
||||
{{- $monHostDomain := default .Release.Namespace .Values.ceph.namespace }}
|
||||
{{- $monHostURI := cat $monHost "." $monHostDomain | nospace -}}
|
||||
{{- $monHostURI | set .Values.conf.ceph.config.global "mon_host" | quote | trunc 0 -}}
|
||||
{{- end -}}
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: "pvc-ceph-conf-combined-storageclass"
|
||||
type: kubernetes.io/rbd
|
||||
name: ceph-etc
|
||||
data:
|
||||
key: |
|
||||
{{ include "secrets/ceph-client-key" . | b64enc | indent 4 }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: "pvc-ceph-client-key"
|
||||
type: kubernetes.io/rbd
|
||||
data:
|
||||
key: |
|
||||
{{ include "secrets/ceph-client-key" . | b64enc | indent 4 }}
|
||||
ceph.conf: |+
|
||||
{{ tuple "etc/_ceph.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
# Copyright 2017 The Openstack-Helm Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
{{- if .Values.manifests_enabled.storage_secrets }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: ceph-templates
|
||||
data:
|
||||
admin.keyring: |+
|
||||
{{ tuple "templates/_admin.keyring.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
bootstrap.keyring.mds: |+
|
||||
{{ tuple "templates/_bootstrap.keyring.mds.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
bootstrap.keyring.osd: |+
|
||||
{{ tuple "templates/_bootstrap.keyring.osd.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
bootstrap.keyring.rgw: |+
|
||||
{{ tuple "templates/_bootstrap.keyring.rgw.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
mon.keyring: |+
|
||||
{{ tuple "templates/_mon.keyring.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
{{- end }}
|
||||
|
|
@ -12,6 +12,9 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
{{- if .Values.manifests_enabled.deployment }}
|
||||
{{- $envAll := . }}
|
||||
{{- $dependencies := .Values.dependencies.osd }}
|
||||
---
|
||||
kind: DaemonSet
|
||||
apiVersion: extensions/v1beta1
|
||||
|
|
@ -29,47 +32,21 @@ spec:
|
|||
spec:
|
||||
nodeSelector:
|
||||
{{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
|
||||
volumes:
|
||||
- name: devices
|
||||
hostPath:
|
||||
path: /dev
|
||||
- name: ceph
|
||||
hostPath:
|
||||
path: {{ .Values.storage.var_directory }}
|
||||
- name: ceph-conf
|
||||
secret:
|
||||
secretName: ceph-conf-combined
|
||||
- name: ceph-bootstrap-osd-keyring
|
||||
secret:
|
||||
secretName: ceph-bootstrap-osd-keyring
|
||||
- name: ceph-bootstrap-mds-keyring
|
||||
secret:
|
||||
secretName: ceph-bootstrap-mds-keyring
|
||||
- name: ceph-bootstrap-rgw-keyring
|
||||
secret:
|
||||
secretName: ceph-bootstrap-rgw-keyring
|
||||
- name: osd-directory
|
||||
hostPath:
|
||||
path: {{ .Values.storage.osd_directory }}
|
||||
initContainers:
|
||||
{{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
|
||||
containers:
|
||||
- name: osd-pod
|
||||
image: {{ .Values.images.daemon }}
|
||||
imagePullPolicy: {{ .Values.images.pull_policy }}
|
||||
volumeMounts:
|
||||
- name: devices
|
||||
mountPath: /dev
|
||||
- name: ceph
|
||||
mountPath: /var/lib/ceph
|
||||
- name: ceph-conf
|
||||
mountPath: /etc/ceph
|
||||
- name: ceph-bootstrap-osd-keyring
|
||||
mountPath: /var/lib/ceph/bootstrap-osd
|
||||
- name: ceph-bootstrap-mds-keyring
|
||||
mountPath: /var/lib/ceph/bootstrap-mds
|
||||
- name: ceph-bootstrap-rgw-keyring
|
||||
mountPath: /var/lib/ceph/bootstrap-rgw
|
||||
- name: osd-directory
|
||||
mountPath: /var/lib/ceph/osd
|
||||
{{- if .Values.resources.enabled }}
|
||||
resources:
|
||||
requests:
|
||||
memory: {{ .Values.resources.osd.requests.memory | quote }}
|
||||
cpu: {{ .Values.resources.osd.requests.cpu | quote }}
|
||||
limits:
|
||||
memory: {{ .Values.resources.osd.limits.memory | quote }}
|
||||
cpu: {{ .Values.resources.osd.limits.cpu | quote }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
privileged: true
|
||||
env:
|
||||
|
|
@ -81,6 +58,10 @@ spec:
|
|||
value: ceph
|
||||
- name: CEPH_GET_ADMIN_KEY
|
||||
value: "1"
|
||||
command:
|
||||
- /entrypoint.sh
|
||||
ports:
|
||||
- containerPort: 6800
|
||||
livenessProbe:
|
||||
tcpSocket:
|
||||
port: 6800
|
||||
|
|
@ -90,10 +71,65 @@ spec:
|
|||
tcpSocket:
|
||||
port: 6800
|
||||
timeoutSeconds: 5
|
||||
resources:
|
||||
requests:
|
||||
memory: {{ .Values.resources.osd.requests.memory | quote }}
|
||||
cpu: {{ .Values.resources.osd.requests.cpu | quote }}
|
||||
limits:
|
||||
memory: {{ .Values.resources.osd.limits.memory | quote }}
|
||||
cpu: {{ .Values.resources.osd.limits.cpu | quote }}
|
||||
volumeMounts:
|
||||
- name: devices
|
||||
mountPath: /dev
|
||||
readOnly: false
|
||||
- name: ceph
|
||||
mountPath: /var/lib/ceph
|
||||
readOnly: false
|
||||
- name: ceph-etc
|
||||
mountPath: /etc/ceph/ceph.conf
|
||||
subPath: ceph.conf
|
||||
readOnly: true
|
||||
- name: ceph-client-admin-keyring
|
||||
mountPath: /etc/ceph/ceph.client.admin.keyring
|
||||
subPath: ceph.client.admin.keyring
|
||||
readOnly: false
|
||||
- name: ceph-mon-keyring
|
||||
mountPath: /etc/ceph/ceph.mon.keyring
|
||||
subPath: ceph.mon.keyring
|
||||
readOnly: false
|
||||
- name: ceph-bootstrap-osd-keyring
|
||||
mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring
|
||||
subPath: ceph.keyring
|
||||
readOnly: false
|
||||
- name: ceph-bootstrap-mds-keyring
|
||||
mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring
|
||||
subPath: ceph.keyring
|
||||
readOnly: false
|
||||
- name: ceph-bootstrap-rgw-keyring
|
||||
mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring
|
||||
subPath: ceph.keyring
|
||||
readOnly: false
|
||||
- name: osd-directory
|
||||
mountPath: /var/lib/ceph/osd
|
||||
volumes:
|
||||
- name: devices
|
||||
hostPath:
|
||||
path: /dev
|
||||
- name: ceph
|
||||
hostPath:
|
||||
path: {{ .Values.ceph.storage.var_directory }}
|
||||
- name: ceph-etc
|
||||
configMap:
|
||||
name: ceph-etc
|
||||
- name: ceph-client-admin-keyring
|
||||
secret:
|
||||
secretName: {{ .Values.secrets.keyrings.admin }}
|
||||
- name: ceph-mon-keyring
|
||||
secret:
|
||||
secretName: {{ .Values.secrets.keyrings.mon }}
|
||||
- name: ceph-bootstrap-osd-keyring
|
||||
secret:
|
||||
secretName: {{ .Values.secrets.keyrings.osd }}
|
||||
- name: ceph-bootstrap-mds-keyring
|
||||
secret:
|
||||
secretName: {{ .Values.secrets.keyrings.mds }}
|
||||
- name: ceph-bootstrap-rgw-keyring
|
||||
secret:
|
||||
secretName: {{ .Values.secrets.keyrings.rgw }}
|
||||
- name: osd-directory
|
||||
hostPath:
|
||||
path: {{ .Values.ceph.storage.osd_directory }}
|
||||
{{- end }}
|
||||
|
|
|
|||
|
|
@ -12,7 +12,10 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
---
|
||||
{{- if .Values.manifests_enabled.deployment }}
|
||||
{{- if .Values.ceph.enabled.mds }}
|
||||
{{- $envAll := . }}
|
||||
{{- $dependencies := .Values.dependencies.mds }}
|
||||
kind: Deployment
|
||||
apiVersion: apps/v1beta1
|
||||
metadata:
|
||||
|
|
@ -31,24 +34,22 @@ spec:
|
|||
spec:
|
||||
nodeSelector:
|
||||
{{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
|
||||
initContainers:
|
||||
{{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
|
||||
serviceAccount: default
|
||||
volumes:
|
||||
- name: ceph-conf
|
||||
secret:
|
||||
secretName: ceph-conf-combined
|
||||
- name: ceph-bootstrap-osd-keyring
|
||||
secret:
|
||||
secretName: ceph-bootstrap-osd-keyring
|
||||
- name: ceph-bootstrap-mds-keyring
|
||||
secret:
|
||||
secretName: ceph-bootstrap-mds-keyring
|
||||
- name: ceph-bootstrap-rgw-keyring
|
||||
secret:
|
||||
secretName: ceph-bootstrap-rgw-keyring
|
||||
containers:
|
||||
- name: ceph-mds
|
||||
image: {{ .Values.images.daemon }}
|
||||
imagePullPolicy: {{ .Values.images.pull_policy }}
|
||||
{{- if .Values.resources.enabled }}
|
||||
resources:
|
||||
requests:
|
||||
memory: {{ .Values.resources.mds.requests.memory | quote }}
|
||||
cpu: {{ .Values.resources.mds.requests.cpu | quote }}
|
||||
limits:
|
||||
memory: {{ .Values.resources.mds.limits.memory | quote }}
|
||||
cpu: {{ .Values.resources.mds.limits.cpu | quote }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- containerPort: 6800
|
||||
env:
|
||||
|
|
@ -60,15 +61,33 @@ spec:
|
|||
value: k8s
|
||||
- name: CLUSTER
|
||||
value: ceph
|
||||
command:
|
||||
- /entrypoint.sh
|
||||
volumeMounts:
|
||||
- name: ceph-conf
|
||||
mountPath: /etc/ceph
|
||||
- name: ceph-etc
|
||||
mountPath: /etc/ceph/ceph.conf
|
||||
subPath: ceph.conf
|
||||
readOnly: true
|
||||
- name: ceph-client-admin-keyring
|
||||
mountPath: /etc/ceph/ceph.client.admin.keyring
|
||||
subPath: ceph.client.admin.keyring
|
||||
readOnly: true
|
||||
- name: ceph-mon-keyring
|
||||
mountPath: /etc/ceph/ceph.mon.keyring
|
||||
subPath: ceph.mon.keyring
|
||||
readOnly: true
|
||||
- name: ceph-bootstrap-osd-keyring
|
||||
mountPath: /var/lib/ceph/bootstrap-osd
|
||||
mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring
|
||||
subPath: ceph.keyring
|
||||
readOnly: false
|
||||
- name: ceph-bootstrap-mds-keyring
|
||||
mountPath: /var/lib/ceph/bootstrap-mds
|
||||
mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring
|
||||
subPath: ceph.keyring
|
||||
readOnly: false
|
||||
- name: ceph-bootstrap-rgw-keyring
|
||||
mountPath: /var/lib/ceph/bootstrap-rgw
|
||||
mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring
|
||||
subPath: ceph.keyring
|
||||
readOnly: false
|
||||
livenessProbe:
|
||||
tcpSocket:
|
||||
port: 6800
|
||||
|
|
@ -78,10 +97,24 @@ spec:
|
|||
tcpSocket:
|
||||
port: 6800
|
||||
timeoutSeconds: 5
|
||||
resources:
|
||||
requests:
|
||||
memory: {{ .Values.resources.mds.requests.memory | quote }}
|
||||
cpu: {{ .Values.resources.mds.requests.cpu | quote }}
|
||||
limits:
|
||||
memory: {{ .Values.resources.mds.limits.memory | quote }}
|
||||
cpu: {{ .Values.resources.mds.limits.cpu | quote }}
|
||||
volumes:
|
||||
- name: ceph-etc
|
||||
configMap:
|
||||
name: ceph-etc
|
||||
- name: ceph-client-admin-keyring
|
||||
secret:
|
||||
secretName: {{ .Values.secrets.keyrings.admin }}
|
||||
- name: ceph-mon-keyring
|
||||
secret:
|
||||
secretName: {{ .Values.secrets.keyrings.mon }}
|
||||
- name: ceph-bootstrap-osd-keyring
|
||||
secret:
|
||||
secretName: {{ .Values.secrets.keyrings.osd }}
|
||||
- name: ceph-bootstrap-mds-keyring
|
||||
secret:
|
||||
secretName: {{ .Values.secrets.keyrings.mds }}
|
||||
- name: ceph-bootstrap-rgw-keyring
|
||||
secret:
|
||||
secretName: {{ .Values.secrets.keyrings.rgw }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,9 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
{{- if .Values.manifests_enabled.deployment }}
|
||||
{{- $envAll := . }}
|
||||
{{- $dependencies := .Values.dependencies.moncheck }}
|
||||
---
|
||||
kind: Deployment
|
||||
apiVersion: apps/v1beta1
|
||||
|
|
@ -31,24 +34,22 @@ spec:
|
|||
spec:
|
||||
nodeSelector:
|
||||
{{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
|
||||
initContainers:
|
||||
{{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
|
||||
serviceAccount: default
|
||||
volumes:
|
||||
- name: ceph-conf
|
||||
secret:
|
||||
secretName: ceph-conf-combined
|
||||
- name: ceph-bootstrap-osd-keyring
|
||||
secret:
|
||||
secretName: ceph-bootstrap-osd-keyring
|
||||
- name: ceph-bootstrap-mds-keyring
|
||||
secret:
|
||||
secretName: ceph-bootstrap-mds-keyring
|
||||
- name: ceph-bootstrap-rgw-keyring
|
||||
secret:
|
||||
secretName: ceph-bootstrap-rgw-keyring
|
||||
containers:
|
||||
- name: ceph-mon
|
||||
image: {{ .Values.images.daemon }}
|
||||
imagePullPolicy: {{ .Values.images.pull_policy }}
|
||||
{{- if .Values.resources.enabled }}
|
||||
resources:
|
||||
requests:
|
||||
memory: {{ .Values.resources.mon_check.requests.memory | quote }}
|
||||
cpu: {{ .Values.resources.mon_check.requests.cpu | quote }}
|
||||
limits:
|
||||
memory: {{ .Values.resources.mon_check.limits.memory | quote }}
|
||||
cpu: {{ .Values.resources.mon_check.limits.cpu | quote }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- containerPort: 6789
|
||||
env:
|
||||
|
|
@ -60,19 +61,50 @@ spec:
|
|||
value: "1"
|
||||
- name: CLUSTER
|
||||
value: ceph
|
||||
command:
|
||||
- /entrypoint.sh
|
||||
volumeMounts:
|
||||
- name: ceph-conf
|
||||
mountPath: /etc/ceph
|
||||
- name: ceph-etc
|
||||
mountPath: /etc/ceph/ceph.conf
|
||||
subPath: ceph.conf
|
||||
readOnly: true
|
||||
- name: ceph-client-admin-keyring
|
||||
mountPath: /etc/ceph/ceph.client.admin.keyring
|
||||
subPath: ceph.client.admin.keyring
|
||||
readOnly: true
|
||||
- name: ceph-mon-keyring
|
||||
mountPath: /etc/ceph/ceph.mon.keyring
|
||||
subPath: ceph.mon.keyring
|
||||
readOnly: true
|
||||
- name: ceph-bootstrap-osd-keyring
|
||||
mountPath: /var/lib/ceph/bootstrap-osd
|
||||
mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring
|
||||
subPath: ceph.keyring
|
||||
readOnly: false
|
||||
- name: ceph-bootstrap-mds-keyring
|
||||
mountPath: /var/lib/ceph/bootstrap-mds
|
||||
mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring
|
||||
subPath: ceph.keyring
|
||||
readOnly: false
|
||||
- name: ceph-bootstrap-rgw-keyring
|
||||
mountPath: /var/lib/ceph/bootstrap-rgw
|
||||
resources:
|
||||
requests:
|
||||
memory: {{ .Values.resources.mon_check.requests.memory | quote }}
|
||||
cpu: {{ .Values.resources.mon_check.requests.cpu | quote }}
|
||||
limits:
|
||||
memory: {{ .Values.resources.mon_check.limits.memory | quote }}
|
||||
cpu: {{ .Values.resources.mon_check.limits.cpu | quote }}
|
||||
mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring
|
||||
subPath: ceph.keyring
|
||||
readOnly: false
|
||||
volumes:
|
||||
- name: ceph-etc
|
||||
configMap:
|
||||
name: ceph-etc
|
||||
- name: ceph-client-admin-keyring
|
||||
secret:
|
||||
secretName: {{ .Values.secrets.keyrings.admin }}
|
||||
- name: ceph-mon-keyring
|
||||
secret:
|
||||
secretName: {{ .Values.secrets.keyrings.mon }}
|
||||
- name: ceph-bootstrap-osd-keyring
|
||||
secret:
|
||||
secretName: {{ .Values.secrets.keyrings.osd }}
|
||||
- name: ceph-bootstrap-mds-keyring
|
||||
secret:
|
||||
secretName: {{ .Values.secrets.keyrings.mds }}
|
||||
- name: ceph-bootstrap-rgw-keyring
|
||||
secret:
|
||||
secretName: {{ .Values.secrets.keyrings.rgw }}
|
||||
{{- end }}
|
||||
|
|
|
|||
|
|
@ -12,7 +12,10 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
{{- if .Values.rgw.enabled }}
|
||||
{{- if .Values.manifests_enabled.deployment }}
|
||||
{{- if .Values.ceph.enabled.rgw }}
|
||||
{{- $envAll := . }}
|
||||
{{- $dependencies := .Values.dependencies.rgw }}
|
||||
---
|
||||
kind: Deployment
|
||||
apiVersion: apps/v1beta1
|
||||
|
|
@ -32,24 +35,22 @@ spec:
|
|||
spec:
|
||||
nodeSelector:
|
||||
{{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
|
||||
initContainers:
|
||||
{{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
|
||||
serviceAccount: default
|
||||
volumes:
|
||||
- name: ceph-conf
|
||||
secret:
|
||||
secretName: ceph-conf-combined
|
||||
- name: ceph-bootstrap-osd-keyring
|
||||
secret:
|
||||
secretName: ceph-bootstrap-osd-keyring
|
||||
- name: ceph-bootstrap-mds-keyring
|
||||
secret:
|
||||
secretName: ceph-bootstrap-mds-keyring
|
||||
- name: ceph-bootstrap-rgw-keyring
|
||||
secret:
|
||||
secretName: ceph-bootstrap-rgw-keyring
|
||||
containers:
|
||||
- name: ceph-rgw
|
||||
image: {{ .Values.images.daemon }}
|
||||
imagePullPolicy: {{ .Values.images.pull_policy }}
|
||||
{{- if .Values.resources.enabled }}
|
||||
resources:
|
||||
requests:
|
||||
memory: {{ .Values.resources.rgw.requests.memory | quote }}
|
||||
cpu: {{ .Values.resources.rgw.requests.cpu | quote }}
|
||||
limits:
|
||||
memory: {{ .Values.resources.rgw.limits.memory | quote }}
|
||||
cpu: {{ .Values.resources.rgw.limits.cpu | quote }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- containerPort: {{ .Values.network.port.rgw_target }}
|
||||
env:
|
||||
|
|
@ -61,15 +62,33 @@ spec:
|
|||
value: k8s
|
||||
- name: CLUSTER
|
||||
value: ceph
|
||||
command:
|
||||
- /entrypoint.sh
|
||||
volumeMounts:
|
||||
- name: ceph-conf
|
||||
mountPath: /etc/ceph
|
||||
- name: ceph-etc
|
||||
mountPath: /etc/ceph/ceph.conf
|
||||
subPath: ceph.conf
|
||||
readOnly: true
|
||||
- name: ceph-client-admin-keyring
|
||||
mountPath: /etc/ceph/ceph.client.admin.keyring
|
||||
subPath: ceph.client.admin.keyring
|
||||
readOnly: true
|
||||
- name: ceph-mon-keyring
|
||||
mountPath: /etc/ceph/ceph.mon.keyring
|
||||
subPath: ceph.mon.keyring
|
||||
readOnly: true
|
||||
- name: ceph-bootstrap-osd-keyring
|
||||
mountPath: /var/lib/ceph/bootstrap-osd
|
||||
mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring
|
||||
subPath: ceph.keyring
|
||||
readOnly: false
|
||||
- name: ceph-bootstrap-mds-keyring
|
||||
mountPath: /var/lib/ceph/bootstrap-mds
|
||||
mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring
|
||||
subPath: ceph.keyring
|
||||
readOnly: false
|
||||
- name: ceph-bootstrap-rgw-keyring
|
||||
mountPath: /var/lib/ceph/bootstrap-rgw
|
||||
mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring
|
||||
subPath: ceph.keyring
|
||||
readOnly: false
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
|
|
@ -81,11 +100,24 @@ spec:
|
|||
path: /
|
||||
port: {{ .Values.network.port.rgw_target }}
|
||||
timeoutSeconds: 5
|
||||
resources:
|
||||
requests:
|
||||
memory: {{ .Values.resources.rgw.requests.memory | quote }}
|
||||
cpu: {{ .Values.resources.rgw.requests.cpu | quote }}
|
||||
limits:
|
||||
memory: {{ .Values.resources.rgw.limits.memory | quote }}
|
||||
cpu: {{ .Values.resources.rgw.limits.cpu | quote }}
|
||||
volumes:
|
||||
- name: ceph-etc
|
||||
configMap:
|
||||
name: ceph-etc
|
||||
- name: ceph-client-admin-keyring
|
||||
secret:
|
||||
secretName: {{ .Values.secrets.keyrings.admin }}
|
||||
- name: ceph-mon-keyring
|
||||
secret:
|
||||
secretName: {{ .Values.secrets.keyrings.mon }}
|
||||
- name: ceph-bootstrap-osd-keyring
|
||||
secret:
|
||||
secretName: {{ .Values.secrets.keyrings.osd }}
|
||||
- name: ceph-bootstrap-mds-keyring
|
||||
secret:
|
||||
secretName: {{ .Values.secrets.keyrings.mds }}
|
||||
- name: ceph-bootstrap-rgw-keyring
|
||||
secret:
|
||||
secretName: {{ .Values.secrets.keyrings.rgw }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,77 @@
|
|||
[global]
|
||||
fsid = {{ uuidv4 | default .Values.conf.ceph.config.global.uuid | quote }}
|
||||
cephx = {{ .Values.conf.ceph.config.global.cephx | default "true" | quote }}
|
||||
cephx_require_signatures = {{ .Values.conf.ceph.config.global.cephx_require_signatures | default "false" | quote }}
|
||||
cephx_cluster_require_signatures = {{ .Values.conf.ceph.config.global.cephx_cluster_require_signatures | default "true" | quote }}
|
||||
cephx_service_require_signatures = {{ .Values.conf.ceph.config.global.cephx_service_require_signatures | default "false" | quote }}
|
||||
|
||||
# auth
|
||||
max_open_files = {{ .Values.conf.ceph.config.global.max_open_files | default "131072" | quote }}
|
||||
|
||||
osd_pool_default_pg_num = {{ .Values.conf.ceph.config.global.osd_pool_default_pg_num | default "128" | quote }}
|
||||
osd_pool_default_pgp_num = {{ .Values.conf.ceph.config.global.osd_pool_default_pgp_num | default "128" | quote }}
|
||||
osd_pool_default_size = {{ .Values.conf.ceph.config.global.osd_pool_default_size | default "3" | quote }}
|
||||
osd_pool_default_min_size = {{ .Values.conf.ceph.config.global.osd_pool_default_min_size | default "1" | quote }}
|
||||
|
||||
mon_osd_full_ratio = {{ .Values.conf.ceph.config.global.mon_osd_full_ratio | default ".95" | quote }}
|
||||
mon_osd_nearfull_ratio = {{ .Values.conf.ceph.config.global.mon_osd_nearfull_ratio | default ".85" | quote }}
|
||||
mon_host = {{ .Values.conf.ceph.config.global.mon_host | quote }}
|
||||
|
||||
rgw_thread_pool_size = {{ .Values.conf.ceph.config.global.rgw_thread_pool_size | default "1024" | quote }}
|
||||
rgw_num_rados_handles = {{ .Values.conf.ceph.config.global.rgw_num_rados_handles | default "100" | quote }}
|
||||
|
||||
[mon]
|
||||
mon_osd_down_out_interval = {{ .Values.conf.ceph.config.mon.mon_osd_down_out_interval | default "600" | quote }}
|
||||
mon_osd_min_down_reporters = {{ .Values.conf.ceph.config.mon.mon_osd_min_down_reporters | default "4" | quote }}
|
||||
mon_clock_drift_allowed = {{ .Values.conf.ceph.config.mon.mon_clock_drift_allowed | default "0.15" | quote }}
|
||||
mon_clock_drift_warn_backoff = {{ .Values.conf.ceph.config.mon.mon_clock_drift_warn_backoff | default "30" | quote }}
|
||||
mon_osd_report_timeout = {{ .Values.conf.ceph.config.mon.mon_osd_report_timeout | default "300" | quote }}
|
||||
|
||||
[osd]
|
||||
# network
|
||||
cluster_network = {{ .Values.network.cluster | default "192.168.0.0/16" | quote }}
|
||||
public_network = {{ .Values.network.public | default "192.168.0.0/16" | quote }}
|
||||
osd_mon_heartbeat_interval = {{ .Values.conf.ceph.config.osd.osd_mon_heartbeat_interval | default "30" | quote }}
|
||||
|
||||
# ports
|
||||
ms_bind_port_min = {{ .Values.conf.ceph.config.osd.ms_bind_port_min | default "6800" | quote }}
|
||||
ms_bind_port_max = {{ .Values.conf.ceph.config.osd.ms_bind_port_max | default "7100" | quote }}
|
||||
|
||||
# journal
|
||||
journal_size = {{ .Values.conf.ceph.config.osd.journal_size | default "100" | quote }}
|
||||
|
||||
# filesystem
|
||||
osd_mkfs_type = {{ .Values.conf.ceph.config.osd.osd_mkfs_type | default "xfs" | quote }}
|
||||
osd_mkfs_options_xfs = {{ .Values.conf.ceph.config.osd.osd_mkfs_options_xfs | default "-f -i size=2048" | quote }}
|
||||
osd_max_object_name_len = {{ .Values.conf.ceph.config.osd.osd_max_object_name_len | default "256" | quote }}
|
||||
|
||||
# crush
|
||||
osd_pool_default_crush_rule = {{ .Values.conf.ceph.config.osd.osd_pool_default_crush_rule | default "0" | quote }}
|
||||
osd_crush_update_on_start = {{ .Values.conf.ceph.config.osd.osd_crush_update_on_start | default "true" | quote }}
|
||||
|
||||
# backend
|
||||
osd_objectstore = {{ .Values.conf.ceph.config.osd.osd_objectstore | default "filestore" | quote }}
|
||||
|
||||
# performance tuning
|
||||
filestore_merge_threshold = {{ .Values.conf.ceph.config.osd.filestore_merge_threshold | default "40" | quote }}
|
||||
filestore_split_multiple = {{ .Values.conf.ceph.config.osd.filestore_split_multiple | default "8" | quote }}
|
||||
osd_op_threads = {{ .Values.conf.ceph.config.osd.osd_op_threads | default "8" | quote }}
|
||||
filestore_op_threads = {{ .Values.conf.ceph.config.osd.filestore_op_threads | default "8" | quote }}
|
||||
filestore_max_sync_interval = {{ .Values.conf.ceph.config.osd.filestore_max_sync_interval | default "5" | quote }}
|
||||
osd_max_scrubs = {{ .Values.conf.ceph.config.osd.osd_max_scrubs | default "1" | quote }}
|
||||
|
||||
# recovery tuning
|
||||
osd_recovery_max_active = {{ .Values.conf.ceph.config.osd.osd_recovery_max_active | default "5" | quote }}
|
||||
osd_max_backfills = {{ .Values.conf.ceph.config.osd.osd_max_backfills | default "2" | quote }}
|
||||
osd_recovery_op_priority = {{ .Values.conf.ceph.config.osd.osd_recovery_op_priority | default "2" | quote }}
|
||||
osd_client_op_priority = {{ .Values.conf.ceph.config.osd.osd_client_op_priority | default "63" | quote }}
|
||||
osd_recovery_max_chunk = {{ .Values.conf.ceph.config.osd.osd_client_op_priority | default "osd_recovery_max_chunk" | quote }}
|
||||
osd_recovery_threads = {{ .Values.conf.ceph.config.osd.osd_recovery_threads | default "1" | quote }}
|
||||
|
||||
[client]
|
||||
rbd_cache_enabled = {{ .Values.conf.ceph.config.client.rbd_cache_enabled | default "true" | quote }}
|
||||
rbd_cache_writethrough_until_flush = {{ .Values.conf.ceph.config.client.rbd_cache_writethrough_until_flush | default "true" | quote }}
|
||||
rbd_default_features = {{ .Values.conf.ceph.config.client.rbd_default_features | default "1" | quote }}
|
||||
|
||||
[mds]
|
||||
mds_cache_size = {{ .Values.conf.ceph.config.client.mds_mds_cache_size | default "100000" | quote }}
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
# Copyright 2017 The Openstack-Helm Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
{{- if .Values.manifests_enabled.storage_secrets }}
|
||||
{{- $envAll := . }}
|
||||
{{- range $key1, $cephBootstrapKey := tuple "mds" "osd" "rgw" "mon" }}
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: ceph-{{ $cephBootstrapKey }}-keyring-generator
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: ceph-secret-generator
|
||||
image: {{ $envAll.Values.images.ceph_config_helper }}
|
||||
imagePullPolicy: {{ $envAll.Values.images.pull_policy }}
|
||||
{{- if $envAll.Values.resources.enabled }}
|
||||
resources:
|
||||
requests:
|
||||
memory: {{ .Values.resources.jobs.secret_provisioning.requests.memory | quote }}
|
||||
cpu: {{ .Values.resources.jobs.secret_provisioning.requests.cpu | quote }}
|
||||
limits:
|
||||
memory: {{ .Values.resources.jobs.secret_provisioning.limits.memory | quote }}
|
||||
cpu: {{ .Values.resources.jobs.secret_provisioning.limits.cpu | quote }}
|
||||
{{- end }}
|
||||
env:
|
||||
- name: DEPLOYMENT_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: CEPH_GEN_DIR
|
||||
value: /opt/ceph
|
||||
- name: CEPH_TEMPLATES_DIR
|
||||
value: /opt/ceph/templates
|
||||
{{- if eq $cephBootstrapKey "mon"}}
|
||||
- name: CEPH_KEYRING_NAME
|
||||
value: ceph.mon.keyring
|
||||
- name: CEPH_KEYRING_TEMPLATE
|
||||
value: mon.keyring
|
||||
{{- else }}
|
||||
- name: CEPH_KEYRING_NAME
|
||||
value: ceph.keyring
|
||||
- name: CEPH_KEYRING_TEMPLATE
|
||||
value: bootstrap.keyring.{{ $cephBootstrapKey }}
|
||||
{{- end }}
|
||||
- name: KUBE_SECRET_NAME
|
||||
value: {{ index $envAll.Values.secrets.keyrings $cephBootstrapKey }}
|
||||
command:
|
||||
- /opt/ceph/ceph-key.sh
|
||||
volumeMounts:
|
||||
- name: ceph-bin
|
||||
mountPath: /opt/ceph/ceph-key.sh
|
||||
subPath: ceph-key.sh
|
||||
readOnly: true
|
||||
- name: ceph-bin
|
||||
mountPath: /opt/ceph/ceph-key.py
|
||||
subPath: ceph-key.py
|
||||
readOnly: true
|
||||
- name: ceph-templates
|
||||
mountPath: /opt/ceph/templates
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: ceph-bin
|
||||
configMap:
|
||||
name: ceph-bin
|
||||
defaultMode: 0555
|
||||
- name: ceph-templates
|
||||
configMap:
|
||||
name: ceph-templates
|
||||
{{ end }}
|
||||
{{ end }}
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
# Copyright 2017 The Openstack-Helm Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
{{- if .Values.manifests_enabled.client_secrets }}
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: ceph-namespace-client-key-generator
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: ceph-storage-keys-generator
|
||||
image: {{ .Values.images.ceph_config_helper }}
|
||||
imagePullPolicy: {{ .Values.images.pull_policy }}
|
||||
{{- if .Values.resources.enabled }}
|
||||
resources:
|
||||
requests:
|
||||
memory: {{ .Values.resources.jobs.secret_provisioning.requests.memory | quote }}
|
||||
cpu: {{ .Values.resources.jobs.secret_provisioning.requests.cpu | quote }}
|
||||
limits:
|
||||
memory: {{ .Values.resources.jobs.secret_provisioning.limits.memory | quote }}
|
||||
cpu: {{ .Values.resources.jobs.secret_provisioning.limits.cpu | quote }}
|
||||
{{- end }}
|
||||
env:
|
||||
- name: DEPLOYMENT_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: PVC_CEPH_STORAGECLASS_USER_SECRET_NAME
|
||||
value: {{ .Values.storageclass.user_secret_name }}
|
||||
- name: PVC_CEPH_STORAGECLASS_ADMIN_SECRET_NAME
|
||||
value: {{ .Values.storageclass.admin_secret_name }}
|
||||
- name: PVC_CEPH_STORAGECLASS_DEPLOYED_NAMESPACE
|
||||
value: {{ .Values.storageclass.admin_secret_namespace }}
|
||||
command:
|
||||
- /opt/ceph/ceph-namespace-client-key.sh
|
||||
volumeMounts:
|
||||
- name: ceph-bin
|
||||
mountPath: /opt/ceph/ceph-namespace-client-key.sh
|
||||
subPath: ceph-namespace-client-key.sh
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: ceph-bin
|
||||
configMap:
|
||||
name: ceph-bin
|
||||
defaultMode: 0555
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
# Copyright 2017 The Openstack-Helm Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
{{- if .Values.manifests_enabled.storage_secrets }}
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: ceph-storage-keys-generator
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: ceph-storage-keys-generator
|
||||
image: {{ .Values.images.ceph_config_helper }}
|
||||
imagePullPolicy: {{ .Values.images.pull_policy }}
|
||||
{{- if .Values.resources.enabled }}
|
||||
resources:
|
||||
requests:
|
||||
memory: {{ .Values.resources.jobs.secret_provisioning.requests.memory | quote }}
|
||||
cpu: {{ .Values.resources.jobs.secret_provisioning.requests.cpu | quote }}
|
||||
limits:
|
||||
memory: {{ .Values.resources.jobs.secret_provisioning.limits.memory | quote }}
|
||||
cpu: {{ .Values.resources.jobs.secret_provisioning.limits.cpu | quote }}
|
||||
{{- end }}
|
||||
env:
|
||||
- name: DEPLOYMENT_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: CEPH_GEN_DIR
|
||||
value: /opt/ceph
|
||||
- name: CEPH_TEMPLATES_DIR
|
||||
value: /opt/ceph/templates
|
||||
- name: CEPH_KEYRING_NAME
|
||||
value: ceph.client.admin.keyring
|
||||
- name: CEPH_KEYRING_TEMPLATE
|
||||
value: admin.keyring
|
||||
- name: CEPH_KEYRING_ADMIN_NAME
|
||||
value: {{ .Values.secrets.keyrings.admin }}
|
||||
- name: CEPH_STORAGECLASS_ADMIN_SECRET_NAME
|
||||
value: {{ .Values.storageclass.admin_secret_name }}
|
||||
command:
|
||||
- /opt/ceph/ceph-storage-key.sh
|
||||
volumeMounts:
|
||||
- name: ceph-bin
|
||||
mountPath: /opt/ceph/ceph-storage-key.sh
|
||||
subPath: ceph-storage-key.sh
|
||||
readOnly: true
|
||||
- name: ceph-bin
|
||||
mountPath: /opt/ceph/ceph-key.py
|
||||
subPath: ceph-key.py
|
||||
readOnly: true
|
||||
- name: ceph-templates
|
||||
mountPath: /opt/ceph/templates
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: ceph-bin
|
||||
configMap:
|
||||
name: ceph-bin
|
||||
defaultMode: 0555
|
||||
- name: ceph-templates
|
||||
configMap:
|
||||
name: ceph-templates
|
||||
{{- end }}
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
{{- if .Values.manifests_enabled.deployment }}
|
||||
apiVersion: policy/v1beta1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
|
|
@ -7,4 +8,5 @@ spec:
|
|||
selector:
|
||||
matchLabels:
|
||||
app: ceph
|
||||
daemon: mon
|
||||
daemon: mon
|
||||
{{- end }}
|
||||
|
|
|
|||
|
|
@ -1,65 +0,0 @@
|
|||
# Copyright 2017 The Openstack-Helm Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
{{- if .Values.secrets.use_common_secrets -}}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: "ceph-conf-combined"
|
||||
type: Opaque
|
||||
data:
|
||||
ceph.conf: |
|
||||
{{ include "secrets/ceph.conf" . | b64enc | indent 4 }}
|
||||
ceph.client.admin.keyring: |
|
||||
{{ include "secrets/ceph.client.admin.keyring" . | b64enc | indent 4 }}
|
||||
ceph.mon.keyring: |
|
||||
{{ include "secrets/ceph.mon.keyring" . | b64enc | indent 4 }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: "ceph-bootstrap-rgw-keyring"
|
||||
type: Opaque
|
||||
data:
|
||||
ceph.keyring: |
|
||||
{{ include "secrets/ceph.rgw.keyring" . | b64enc | indent 4 }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: "ceph-bootstrap-mds-keyring"
|
||||
type: Opaque
|
||||
data:
|
||||
ceph.keyring: |
|
||||
{{ include "secrets/ceph.mds.keyring" . | b64enc | indent 4 }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: "ceph-bootstrap-osd-keyring"
|
||||
type: Opaque
|
||||
data:
|
||||
ceph.keyring: |
|
||||
{{ include "secrets/ceph.osd.keyring" . | b64enc | indent 4 }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: "ceph-client-key"
|
||||
type: Opaque
|
||||
data:
|
||||
ceph-client-key: |
|
||||
{{ include "secrets/ceph-client-key" . | b64enc | indent 4 }}
|
||||
{{- end -}}
|
||||
|
|
@ -12,7 +12,8 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
{{- if .Values.rgw.enabled }}
|
||||
{{- if .Values.manifests_enabled.deployment }}
|
||||
{{- if .Values.ceph.enabled.rgw }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
|
|
@ -30,3 +31,4 @@ spec:
|
|||
app: ceph
|
||||
daemon: rgw
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
|
|||
|
|
@ -12,11 +12,12 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
{{- if .Values.manifests_enabled.deployment }}
|
||||
---
|
||||
kind: Service
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: ceph-mon
|
||||
name: {{ .Values.endpoints.ceph_mon.hosts.default }}
|
||||
labels:
|
||||
app: ceph
|
||||
daemon: mon
|
||||
|
|
@ -37,3 +38,4 @@ spec:
|
|||
app: ceph
|
||||
daemon: mon
|
||||
clusterIP: None
|
||||
{{- end }}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,9 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
{{- if .Values.manifests_enabled.deployment }}
|
||||
{{- $envAll := . }}
|
||||
{{- $dependencies := .Values.dependencies.mon }}
|
||||
---
|
||||
apiVersion: apps/v1beta1
|
||||
kind: StatefulSet
|
||||
|
|
@ -21,7 +24,7 @@ metadata:
|
|||
daemon: mon
|
||||
name: ceph-mon
|
||||
spec:
|
||||
serviceName: {{ .Values.service.mon.name | quote }}
|
||||
serviceName: {{ tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_endpoint_lookup" }}
|
||||
replicas: {{ .Values.replicas.mon }}
|
||||
template:
|
||||
metadata:
|
||||
|
|
@ -46,33 +49,22 @@ spec:
|
|||
weight: 10
|
||||
nodeSelector:
|
||||
{{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
|
||||
initContainers:
|
||||
{{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
|
||||
serviceAccount: default
|
||||
volumes:
|
||||
- name: ceph-conf
|
||||
secret:
|
||||
secretName: ceph-conf-combined
|
||||
- name: ceph-bootstrap-osd-keyring
|
||||
secret:
|
||||
secretName: ceph-bootstrap-osd-keyring
|
||||
- name: ceph-bootstrap-mds-keyring
|
||||
secret:
|
||||
secretName: ceph-bootstrap-mds-keyring
|
||||
- name: ceph-bootstrap-rgw-keyring
|
||||
secret:
|
||||
secretName: ceph-bootstrap-rgw-keyring
|
||||
- name: ceph-monfs
|
||||
hostPath:
|
||||
path: {{ .Values.storage.mon_directory }}
|
||||
containers:
|
||||
- name: ceph-mon
|
||||
image: {{ .Values.images.daemon }}
|
||||
imagePullPolicy: {{ .Values.images.pull_policy }}
|
||||
lifecycle:
|
||||
preStop:
|
||||
exec:
|
||||
# remove the mon on Pod stop.
|
||||
command:
|
||||
- "/remove-mon.sh"
|
||||
{{- if .Values.resources.enabled }}
|
||||
resources:
|
||||
requests:
|
||||
memory: {{ .Values.resources.mon.requests.memory | quote }}
|
||||
cpu: {{ .Values.resources.mon.requests.cpu | quote }}
|
||||
limits:
|
||||
memory: {{ .Values.resources.mon.limits.memory | quote }}
|
||||
cpu: {{ .Values.resources.mon.limits.cpu | quote }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- containerPort: 6789
|
||||
env:
|
||||
|
|
@ -90,17 +82,39 @@ spec:
|
|||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: status.podIP
|
||||
command:
|
||||
- /entrypoint.sh
|
||||
lifecycle:
|
||||
preStop:
|
||||
exec:
|
||||
# remove the mon on Pod stop.
|
||||
command:
|
||||
- "/remove-mon.sh"
|
||||
volumeMounts:
|
||||
- name: ceph-conf
|
||||
mountPath: /etc/ceph
|
||||
- name: ceph-etc
|
||||
mountPath: /etc/ceph/ceph.conf
|
||||
subPath: ceph.conf
|
||||
readOnly: true
|
||||
- name: ceph-client-admin-keyring
|
||||
mountPath: /etc/ceph/ceph.client.admin.keyring
|
||||
subPath: ceph.client.admin.keyring
|
||||
readOnly: true
|
||||
- name: ceph-mon-keyring
|
||||
mountPath: /etc/ceph/ceph.mon.keyring
|
||||
subPath: ceph.mon.keyring
|
||||
readOnly: false
|
||||
- name: ceph-bootstrap-osd-keyring
|
||||
mountPath: /var/lib/ceph/bootstrap-osd
|
||||
mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring
|
||||
subPath: ceph.keyring
|
||||
readOnly: false
|
||||
- name: ceph-bootstrap-mds-keyring
|
||||
mountPath: /var/lib/ceph/bootstrap-mds
|
||||
mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring
|
||||
subPath: ceph.keyring
|
||||
readOnly: false
|
||||
- name: ceph-bootstrap-rgw-keyring
|
||||
mountPath: /var/lib/ceph/bootstrap-rgw
|
||||
- name: ceph-monfs
|
||||
mountPath: /var/lib/ceph/mon
|
||||
mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring
|
||||
subPath: ceph.keyring
|
||||
readOnly: false
|
||||
livenessProbe:
|
||||
tcpSocket:
|
||||
port: 6789
|
||||
|
|
@ -110,10 +124,23 @@ spec:
|
|||
tcpSocket:
|
||||
port: 6789
|
||||
timeoutSeconds: 5
|
||||
resources:
|
||||
requests:
|
||||
memory: {{ .Values.resources.mon.requests.memory | quote }}
|
||||
cpu: {{ .Values.resources.mon.requests.cpu | quote }}
|
||||
limits:
|
||||
memory: {{ .Values.resources.mon.limits.memory | quote }}
|
||||
cpu: {{ .Values.resources.mon.limits.cpu | quote }}
|
||||
volumes:
|
||||
- name: ceph-etc
|
||||
configMap:
|
||||
name: ceph-etc
|
||||
- name: ceph-client-admin-keyring
|
||||
secret:
|
||||
secretName: {{ .Values.secrets.keyrings.admin }}
|
||||
- name: ceph-mon-keyring
|
||||
secret:
|
||||
secretName: {{ .Values.secrets.keyrings.mon }}
|
||||
- name: ceph-bootstrap-osd-keyring
|
||||
secret:
|
||||
secretName: {{ .Values.secrets.keyrings.osd }}
|
||||
- name: ceph-bootstrap-mds-keyring
|
||||
secret:
|
||||
secretName: {{ .Values.secrets.keyrings.mds }}
|
||||
- name: ceph-bootstrap-rgw-keyring
|
||||
secret:
|
||||
secretName: {{ .Values.secrets.keyrings.rgw }}
|
||||
{{- end }}
|
||||
|
|
|
|||
|
|
@ -12,8 +12,7 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# note that these secrets are handled by the common chart, not the ceph
|
||||
# chart, as we likely want them "everywhere"
|
||||
{{- if .Values.manifests_enabled.deployment }}
|
||||
---
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
|
|
@ -21,12 +20,11 @@ metadata:
|
|||
name: {{ .Values.storageclass.name }}
|
||||
provisioner: kubernetes.io/rbd
|
||||
parameters:
|
||||
monitors: {{ .Values.storageclass.monitors | default "ceph-mon.ceph:6789" }}
|
||||
monitors: {{ tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.hostname_endpoint_uri_lookup" }}
|
||||
adminId: {{ .Values.storageclass.admin_id }}
|
||||
adminSecretName: {{ .Values.storageclass.admin_secret_name }}
|
||||
# forcing namespace due to issue with default pipeline of "{{ .Release.Namespace }}" }}
|
||||
# during helm lint
|
||||
adminSecretNamespace: {{ .Values.storageclass.admin_secret_namespace | default "ceph" }}
|
||||
adminSecretNamespace: {{ .Values.storageclass.admin_secret_namespace }}
|
||||
pool: {{ .Values.storageclass.pool }}
|
||||
userId: {{ .Values.storageclass.user_id }}
|
||||
userSecretName: {{ .Values.storageclass.user_secret_name }}
|
||||
{{- end }}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,6 @@
|
|||
[client.admin]
|
||||
key = {{"{{"}} key {{"}}"}}
|
||||
auid = 0
|
||||
caps mds = "allow"
|
||||
caps mon = "allow *"
|
||||
caps osd = "allow *"
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
[client.bootstrap-mds]
|
||||
key = {{"{{"}} key {{"}}"}}
|
||||
caps mon = "allow profile bootstrap-mds"
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
[client.bootstrap-osd]
|
||||
key = {{"{{"}} key {{"}}"}}
|
||||
caps mon = "allow profile bootstrap-osd"
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
[client.bootstrap-rgw]
|
||||
key = {{"{{"}} key {{"}}"}}
|
||||
caps mon = "allow profile bootstrap-rgw"
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
[mon.]
|
||||
key = {{"{{"}} key {{"}}"}}
|
||||
caps mon = "allow *"
|
||||
170
ceph/values.yaml
170
ceph/values.yaml
|
|
@ -12,6 +12,11 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
manifests_enabled:
|
||||
storage_secrets: true
|
||||
client_secrets: true
|
||||
deployment: true
|
||||
|
||||
replicas:
|
||||
mon: 3
|
||||
rgw: 3
|
||||
|
|
@ -22,8 +27,10 @@ service:
|
|||
name: ceph-mon
|
||||
|
||||
images:
|
||||
daemon: docker.io/library/ceph/daemon:tag-build-master-jewel-ubuntu-16.04
|
||||
pull_policy: IfNotPresent
|
||||
dep_check: docker.io/kolla/ubuntu-source-kubernetes-entrypoint:4.0.0
|
||||
daemon: quay.io/attcomdev/ceph-daemon:tag-build-master-jewel-ubuntu-16.04
|
||||
ceph_config_helper: docker.io/port/ceph-config-helper:v1.6.5
|
||||
pull_policy: Always
|
||||
|
||||
labels:
|
||||
node_selector_key: ceph-storage
|
||||
|
|
@ -33,23 +40,125 @@ pod_disruption_budget:
|
|||
mon:
|
||||
min_available: 0
|
||||
|
||||
secrets:
|
||||
keyrings:
|
||||
mon: ceph-mon-keyring
|
||||
mds: ceph-bootstrap-mds-keyring
|
||||
osd: ceph-bootstrap-osd-keyring
|
||||
rgw: ceph-bootstrap-rgw-keyring
|
||||
admin: ceph-client-admin-keyring
|
||||
|
||||
network:
|
||||
public: "10.25.0.0/16"
|
||||
public: "192.168.0.0/16"
|
||||
cluster: "192.168.0.0/16"
|
||||
port:
|
||||
mon: 6789
|
||||
rgw_ingress: 80
|
||||
rgw_target: 8088
|
||||
|
||||
storage:
|
||||
osd_directory: /var/lib/openstack-helm/ceph/osd
|
||||
var_directory: /var/lib/openstack-helm/ceph/ceph
|
||||
mon_directory: /var/lib/openstack-helm/ceph/mon
|
||||
conf:
|
||||
ceph:
|
||||
override:
|
||||
append:
|
||||
config:
|
||||
global:
|
||||
# auth
|
||||
cephx: true
|
||||
cephx_require_signatures: false
|
||||
cephx_cluster_require_signatures: true
|
||||
cephx_service_require_signatures: false
|
||||
|
||||
max_open_files: 131072
|
||||
osd_pool_default_pg_num: 128
|
||||
osd_pool_default_pgp_num: 128
|
||||
osd_pool_default_size: 3
|
||||
osd_pool_default_min_size: 1
|
||||
mon_osd_full_ratio: .95
|
||||
mon_osd_nearfull_ratio: .85
|
||||
mon_host: null
|
||||
mon:
|
||||
mon_osd_down_out_interval: 600
|
||||
mon_osd_min_down_reporters: 4
|
||||
mon_clock_drift_allowed: .15
|
||||
mon_clock_drift_warn_backoff: 30
|
||||
mon_osd_report_timeout: 300
|
||||
osd:
|
||||
journal_size: 100
|
||||
osd_mkfs_type: xfs
|
||||
osd_mkfs_options_xfs: -f -i size=2048
|
||||
osd_mon_heartbeat_interval: 30
|
||||
osd_max_object_name_len: 256
|
||||
#crush
|
||||
osd_pool_default_crush_rule: 0
|
||||
osd_crush_update_on_start: true
|
||||
#backend
|
||||
osd_objectstore: filestore
|
||||
#performance tuning
|
||||
filestore_merge_threshold: 40
|
||||
filestore_split_multiple: 8
|
||||
osd_op_threads: 8
|
||||
filestore_op_threads: 8
|
||||
filestore_max_sync_interval: 5
|
||||
osd_max_scrubs: 1
|
||||
#recovery tuning
|
||||
osd_recovery_max_active: 5
|
||||
osd_max_backfills: 2
|
||||
osd_recovery_op_priority: 2
|
||||
osd_client_op_priority: 63
|
||||
osd_recovery_max_chunk: 1048576
|
||||
osd_recovery_threads: 1
|
||||
#ports
|
||||
ms_bind_port_min: 6800
|
||||
ms_bind_port_max: 7100
|
||||
client:
|
||||
rbd_cache_enabled: true
|
||||
rbd_cache_writethrough_until_flush: true
|
||||
rbd_default_features: "1"
|
||||
mds:
|
||||
mds_cache_size: 100000
|
||||
|
||||
|
||||
dependencies:
|
||||
mon:
|
||||
jobs:
|
||||
service:
|
||||
osd:
|
||||
jobs:
|
||||
services:
|
||||
- service: ceph_mon
|
||||
endpoint: internal
|
||||
moncheck:
|
||||
jobs:
|
||||
services:
|
||||
- service: ceph_mon
|
||||
endpoint: internal
|
||||
rgw:
|
||||
jobs:
|
||||
services:
|
||||
- service: ceph_mon
|
||||
endpoint: internal
|
||||
mds:
|
||||
jobs:
|
||||
services:
|
||||
- service: ceph_mon
|
||||
endpoint: internal
|
||||
|
||||
|
||||
ceph:
|
||||
enabled:
|
||||
mds: true
|
||||
rgw: false
|
||||
storage:
|
||||
osd_directory: /var/lib/openstack-helm/ceph/osd
|
||||
var_directory: /var/lib/openstack-helm/ceph/ceph
|
||||
mon_directory: /var/lib/openstack-helm/ceph/mon
|
||||
|
||||
# rgw is optionally disabled
|
||||
rgw:
|
||||
enabled: false
|
||||
|
||||
resources:
|
||||
enabled: false
|
||||
osd:
|
||||
requests:
|
||||
memory: "512Mi"
|
||||
|
|
@ -85,36 +194,14 @@ resources:
|
|||
limits:
|
||||
memory: "50Mi"
|
||||
cpu: "500m"
|
||||
|
||||
# Setting this to false will assume you will
|
||||
# setup and orchestrate your own secrets and
|
||||
# configmaps outside of this helm chart
|
||||
#
|
||||
# The list below is in the format of
|
||||
#
|
||||
# configMapName:
|
||||
# elementKeyName
|
||||
#
|
||||
# ceph.conf:
|
||||
# ceph.conf
|
||||
# ceph.client.admin.keyring
|
||||
# ceph.client.admin.keyring
|
||||
# ceph.mon.keyring:
|
||||
# ceph.mon.keyring
|
||||
# ceph-bootstrap-rgw-keyring:
|
||||
# ceph.keyring
|
||||
# ceph.rgw.keyring
|
||||
# ceph-bootstrap-mds-keyring:
|
||||
# ceph.keyring
|
||||
# ceph.mds.keyring
|
||||
# ceph-bootstrap-osd-keyring:
|
||||
# ceph.keyring
|
||||
# ceph.osd.keyring
|
||||
# ceph-client-key:
|
||||
# ceph-client-key
|
||||
secrets:
|
||||
use_common_secrets: true
|
||||
|
||||
jobs:
|
||||
secret_provisioning:
|
||||
limits:
|
||||
memory: "1024Mi"
|
||||
cpu: "2000m"
|
||||
requests:
|
||||
memory: "128Mi"
|
||||
cpu: "500m"
|
||||
# if you change provision_storage_class to false
|
||||
# it is presumed you manage your own storage
|
||||
# class definition externally
|
||||
|
|
@ -125,6 +212,13 @@ storageclass:
|
|||
pool: rbd
|
||||
admin_id: admin
|
||||
admin_secret_name: pvc-ceph-conf-combined-storageclass
|
||||
admin_secret_namespace: null
|
||||
admin_secret_namespace: ceph
|
||||
user_id: admin
|
||||
user_secret_name: pvc-ceph-client-key
|
||||
|
||||
endpoints:
|
||||
ceph_mon:
|
||||
hosts:
|
||||
default: ceph-mon
|
||||
port:
|
||||
mon: 6789
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
{{- define "cinder.is_ceph_configured" -}}
|
||||
{{- define "cinder.is_ceph_volume_configured" -}}
|
||||
{{- range $section, $values := .Values.conf.backends -}}
|
||||
{{- if kindIs "map" $values -}}
|
||||
{{- if eq $values.volume_driver "cinder.volume.drivers.rbd.RBDDriver" -}}
|
||||
|
|
@ -21,3 +21,10 @@ true
|
|||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "cinder.is_ceph_backup_configured" -}}
|
||||
{{- $values := .Values.conf.cinder.default.cinder -}}
|
||||
{{- if eq $values.backup_driver "cinder.backup.drivers.ceph" -}}
|
||||
true
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2017 The Openstack-Helm Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
|
|
@ -12,9 +14,16 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -ex
|
||||
export HOME=/tmp
|
||||
|
||||
cat <<EOF > /etc/ceph/ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring
|
||||
[client.{{ .Values.conf.backends.rbd1.rbd_user }}]
|
||||
{{- if .Values.conf.ceph.cinder_keyring }}
|
||||
key = {{ .Values.conf.ceph.cinder_keyring }}
|
||||
{{- else }}
|
||||
key = {{- include "secrets/ceph-client-key" . -}}
|
||||
key = $(cat /tmp/client-keyring)
|
||||
{{- end }}
|
||||
EOF
|
||||
|
||||
exit 0
|
||||
|
|
@ -33,6 +33,8 @@ data:
|
|||
{{ tuple "bin/_cinder-scheduler.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
cinder-volume.sh: |
|
||||
{{ tuple "bin/_cinder-volume.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
ceph-keyring.sh: |+
|
||||
{{ tuple "bin/_ceph-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
{{- if .Values.bootstrap.enabled }}
|
||||
bootstrap.sh: |+
|
||||
{{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
|
|
|
|||
|
|
@ -97,17 +97,4 @@ data:
|
|||
{{ .Values.conf.policy.override | indent 4 }}
|
||||
{{- else -}}
|
||||
{{ tuple "etc/_policy.json.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
{{- end }}
|
||||
ceph.conf: |+
|
||||
{{- if or (include "cinder.is_ceph_configured" .) (eq .Values.conf.cinder.default.cinder.backup_driver "cinder.backup.drivers.ceph") }}
|
||||
{{ if .Values.conf.ceph.override -}}
|
||||
{{ .Values.conf.ceph.override | indent 4 }}
|
||||
{{- else -}}
|
||||
{{ tuple "etc/_ceph.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
{{- end }}
|
||||
{{- if .Values.conf.ceph.append -}}
|
||||
{{ .Values.conf.ceph.append | indent 4 }}
|
||||
{{- end }}
|
||||
ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring: |+
|
||||
{{ tuple "etc/_ceph-cinder.keyring.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
{{- end }}
|
||||
|
|
|
|||
|
|
@ -42,6 +42,23 @@ spec:
|
|||
{{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
|
||||
initContainers:
|
||||
{{ tuple $envAll $dependencies $mounts_cinder_backup_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
|
||||
{{- if include "cinder.is_ceph_backup_configured" . }}
|
||||
- name: ceph-keyring-placement
|
||||
image: {{ .Values.images.backup }}
|
||||
imagePullPolicy: {{ .Values.images.pull_policy }}
|
||||
command:
|
||||
- /tmp/ceph-keyring.sh
|
||||
volumeMounts:
|
||||
- name: etcceph
|
||||
mountPath: /etc/ceph
|
||||
- name: cinder-bin
|
||||
mountPath: /tmp/ceph-keyring.sh
|
||||
subPath: ceph-keyring.sh
|
||||
- name: ceph-keyring
|
||||
mountPath: /tmp/client-keyring
|
||||
subPath: key
|
||||
readOnly: true
|
||||
{{ end }}
|
||||
containers:
|
||||
- name: cinder-backup
|
||||
image: {{ .Values.images.backup }}
|
||||
|
|
@ -67,13 +84,15 @@ spec:
|
|||
subPath: cinder.conf
|
||||
readOnly: true
|
||||
{{- if eq .Values.conf.cinder.default.cinder.backup_driver "cinder.backup.drivers.ceph" }}
|
||||
- name: cinder-etc
|
||||
- name: etcceph
|
||||
mountPath: /etc/ceph
|
||||
- name: ceph-etc
|
||||
mountPath: /etc/ceph/ceph.conf
|
||||
subPath: ceph.conf
|
||||
readOnly: true
|
||||
- name: cinder-etc
|
||||
mountPath: /etc/ceph/ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring
|
||||
subPath: ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring
|
||||
- name: ceph-keyring
|
||||
mountPath: /tmp/client-keyring
|
||||
subPath: key
|
||||
readOnly: true
|
||||
{{- end -}}
|
||||
{{ if $mounts_cinder_backup.volumeMounts }}{{ toYaml $mounts_cinder_backup.volumeMounts | indent 12 }}{{ end }}
|
||||
|
|
@ -83,4 +102,18 @@ spec:
|
|||
- name: cinder-etc
|
||||
configMap:
|
||||
name: cinder-etc
|
||||
- name: cinder-bin
|
||||
configMap:
|
||||
name: cinder-bin
|
||||
defaultMode: 0555
|
||||
{{- if include "cinder.is_ceph_backup_configured" . }}
|
||||
- name: etcceph
|
||||
emptyDir: {}
|
||||
- name: ceph-etc
|
||||
configMap:
|
||||
name: ceph-etc
|
||||
- name: ceph-keyring
|
||||
secret:
|
||||
secretName: pvc-ceph-client-key
|
||||
{{ end }}
|
||||
{{ if $mounts_cinder_backup.volumes }}{{ toYaml $mounts_cinder_backup.volumes | indent 8 }}{{ end }}
|
||||
|
|
|
|||
|
|
@ -42,6 +42,23 @@ spec:
|
|||
{{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
|
||||
initContainers:
|
||||
{{ tuple $envAll $dependencies $mounts_cinder_volume_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
|
||||
{{- if include "cinder.is_ceph_volume_configured" . }}
|
||||
- name: ceph-keyring-placement
|
||||
image: {{ .Values.images.volume }}
|
||||
imagePullPolicy: {{ .Values.images.pull_policy }}
|
||||
command:
|
||||
- /tmp/ceph-keyring.sh
|
||||
volumeMounts:
|
||||
- name: etcceph
|
||||
mountPath: /etc/ceph
|
||||
- name: cinder-bin
|
||||
mountPath: /tmp/ceph-keyring.sh
|
||||
subPath: ceph-keyring.sh
|
||||
- name: ceph-keyring
|
||||
mountPath: /tmp/client-keyring
|
||||
subPath: key
|
||||
readOnly: true
|
||||
{{ end }}
|
||||
containers:
|
||||
- name: cinder-volume
|
||||
image: {{ .Values.images.volume }}
|
||||
|
|
@ -72,14 +89,16 @@ spec:
|
|||
mountPath: /etc/cinder/conf/backends.conf
|
||||
subPath: backends.conf
|
||||
readOnly: true
|
||||
{{- if include "cinder.is_ceph_configured" . }}
|
||||
- name: cinder-etc
|
||||
{{- if include "cinder.is_ceph_volume_configured" . }}
|
||||
- name: etcceph
|
||||
mountPath: /etc/ceph
|
||||
- name: ceph-etc
|
||||
mountPath: /etc/ceph/ceph.conf
|
||||
subPath: ceph.conf
|
||||
readOnly: true
|
||||
- name: cinder-etc
|
||||
mountPath: /etc/ceph/ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring
|
||||
subPath: ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring
|
||||
- name: ceph-keyring
|
||||
mountPath: /tmp/client-keyring
|
||||
subPath: key
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{ if $mounts_cinder_volume.volumeMounts }}{{ toYaml $mounts_cinder_volume.volumeMounts | indent 12 }}{{ end }}
|
||||
|
|
@ -93,4 +112,14 @@ spec:
|
|||
- name: cinder-etc
|
||||
configMap:
|
||||
name: cinder-etc
|
||||
{{- if include "cinder.is_ceph_volume_configured" . }}
|
||||
- name: etcceph
|
||||
emptyDir: {}
|
||||
- name: ceph-etc
|
||||
configMap:
|
||||
name: ceph-etc
|
||||
- name: ceph-keyring
|
||||
secret:
|
||||
secretName: pvc-ceph-client-key
|
||||
{{ end }}
|
||||
{{ if $mounts_cinder_volume.volumes }}{{ toYaml $mounts_cinder_volume.volumes | indent 8 }}{{ end }}
|
||||
|
|
|
|||
|
|
@ -1,30 +0,0 @@
|
|||
# Copyright 2017 The Openstack-Helm Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
[global]
|
||||
rgw_thread_pool_size = 1024
|
||||
rgw_num_rados_handles = 100
|
||||
{{- if .Values.conf.ceph.monitors }}
|
||||
[mon]
|
||||
{{ range .Values.conf.ceph.monitors }}
|
||||
[mon.{{ . }}]
|
||||
host = {{ . }}
|
||||
mon_addr = {{ . }}
|
||||
{{ end }}
|
||||
{{- else }}
|
||||
mon_host = ceph-mon.ceph
|
||||
{{- end }}
|
||||
[client]
|
||||
rbd_cache_enabled = true
|
||||
rbd_cache_writethrough_until_flush = true
|
||||
|
|
@ -23,6 +23,8 @@ replicas:
|
|||
scheduler: 1
|
||||
backup: 1
|
||||
|
||||
storage: ceph
|
||||
|
||||
labels:
|
||||
node_selector_key: openstack-control-plane
|
||||
node_selector_value: enabled
|
||||
|
|
|
|||
|
|
@ -141,32 +141,13 @@ completed.
|
|||
Installing Ceph Host Requirements
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
At some future point, we want to ensure that our solution is
|
||||
cloud-native, allowing installation on any host system without a package
|
||||
manager and only a container runtime (i.e. CoreOS). Until this happens,
|
||||
we will need to ensure that ``ceph-common`` is installed on each of our
|
||||
hosts. Using our Ubuntu example:
|
||||
You need to ensure that ``ceph-common`` or equivalent is
|
||||
installed on each of our hosts. Using our Ubuntu example:
|
||||
|
||||
::
|
||||
|
||||
sudo apt-get install ceph-common -y
|
||||
|
||||
We will always attempt to keep host-specific requirements to a minimum,
|
||||
and we are working with the Ceph team (Sébastien Han) to quickly address
|
||||
this Ceph requirement.
|
||||
|
||||
Ceph Secrets Generation
|
||||
~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Another thing of interest is that our deployment assumes that you can
|
||||
generate secrets at the time of the container deployment. We require the
|
||||
`sigil <https://github.com/gliderlabs/sigil/releases/download/v0.4.0/sigil_0.4.0_Linux_x86_64.tgz>`__
|
||||
binary on your deployment host in order to perform this action.
|
||||
|
||||
::
|
||||
|
||||
curl -L https://github.com/gliderlabs/sigil/releases/download/v0.4.0/sigil_0.4.0_Linux_x86_64.tgz | sudo tar -zxC /usr/local/bin
|
||||
|
||||
Kubernetes Controller Manager
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
|
@ -200,20 +181,18 @@ Kubernetes v1.6.5.
|
|||
export kube_version=v1.6.5
|
||||
sudo sed -i "s|gcr.io/google_containers/kube-controller-manager-amd64:$kube_version|quay.io/attcomdev/kube-controller-manager:$kube_version|g" /etc/kubernetes/manifests/kube-controller-manager.yaml
|
||||
|
||||
Now you will want to ``restart`` your Kubernetes master server to
|
||||
Now you will want to ``restart`` the Kubernetes master server to
|
||||
continue.
|
||||
|
||||
Kube Controller Manager DNS Resolution
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Until the following `Kubernetes Pull
|
||||
Request <https://github.com/kubernetes/kubernetes/issues/17406>`__ is
|
||||
merged, you will need to allow the Kubernetes Controller to use the
|
||||
internal container ``skydns`` endpoint as a DNS server, and add the
|
||||
Kubernetes search suffix into the controller's resolv.conf. As of now,
|
||||
the Kubernetes controller only mirrors the host's ``resolv.conf``. This
|
||||
is not sufficient if you want the controller to know how to correctly
|
||||
resolve container service endpoints (in the case of DaemonSets).
|
||||
You will need to allow the Kubernetes Controller to use the
|
||||
Kubernetes service DNS server, and add the Kubernetes search suffix
|
||||
to the controller's resolv.conf. As of now, the Kubernetes controller
|
||||
only mirrors the host's ``resolv.conf``. This is not sufficient if you
|
||||
want the controller to know how to correctly resolve container service
|
||||
endpoints.
|
||||
|
||||
First, find out what the IP Address of your ``kube-dns`` deployment is:
|
||||
|
||||
|
|
@ -224,82 +203,16 @@ First, find out what the IP Address of your ``kube-dns`` deployment is:
|
|||
kube-dns 10.96.0.10 <none> 53/UDP,53/TCP 1d
|
||||
admin@kubenode01:~$
|
||||
|
||||
As you can see by this example, ``10.96.0.10`` is the
|
||||
``CLUSTER-IP``\ IP. Now, have a look at the current
|
||||
``kube-controller-manager-kubenode01`` ``/etc/resolv.conf``:
|
||||
Then update the controller manager configuration to match:
|
||||
|
||||
::
|
||||
|
||||
admin@kubenode01:~$ kubectl exec kube-controller-manager-kubenode01 -n kube-system -- cat /etc/resolv.conf
|
||||
# Dynamic resolv.conf(5) file for glibc resolver(3) generated by resolvconf(8)
|
||||
# DO NOT EDIT THIS FILE BY HAND -- YOUR CHANGES WILL BE OVERWRITTEN
|
||||
nameserver 192.168.1.70
|
||||
nameserver 8.8.8.8
|
||||
search jinkit.com
|
||||
admin@kubenode01:~$
|
||||
|
||||
What we need is for ``kube-controller-manager-kubenode01``
|
||||
``/etc/resolv.conf`` to look like this:
|
||||
|
||||
::
|
||||
|
||||
admin@kubenode01:~$ kubectl exec kube-controller-manager-kubenode01 -n kube-system -- cat /etc/resolv.conf
|
||||
admin@kubenode01:~$ CONTROLLER_MANAGER_POD=$(kubectl get -n kube-system pods -l component=kube-controller-manager --no-headers -o name | head -1 | awk -F '/' '{ print $NF }')
|
||||
admin@kubenode01:~$ kubectl exec -n kube-system ${CONTROLLER_MANAGER_POD} -- sh -c "cat > /etc/resolv.conf <<EOF
|
||||
nameserver 10.96.0.10
|
||||
nameserver 192.168.1.70
|
||||
nameserver 8.8.8.8
|
||||
search svc.cluster.local jinkit.com
|
||||
admin@kubenode01:~$
|
||||
|
||||
You can change this by doing the following:
|
||||
|
||||
::
|
||||
|
||||
admin@kubenode01:~$ kubectl exec kube-controller-manager-kubenode01 -it -n kube-system -- /bin/bash
|
||||
root@kubenode01:/# cat <<EOF > /etc/resolv.conf
|
||||
nameserver 10.96.0.10
|
||||
nameserver 192.168.1.70
|
||||
nameserver 8.8.8.8
|
||||
search svc.cluster.local jinkit.com
|
||||
EOF
|
||||
root@kubenode01:/#
|
||||
|
||||
Now you can test your changes by deploying a service to your cluster,
|
||||
and resolving this from the controller. As an example, lets deploy
|
||||
something useful, like `Kubernetes
|
||||
dashboard <https://github.com/kubernetes/dashboard>`__:
|
||||
|
||||
::
|
||||
|
||||
kubectl create -f https://rawgit.com/kubernetes/dashboard/master/src/deploy/kubernetes-dashboard.yaml
|
||||
|
||||
Note the ``IP`` field:
|
||||
|
||||
::
|
||||
|
||||
admin@kubenode01:~$ kubectl describe svc kubernetes-dashboard -n kube-system
|
||||
Name: kubernetes-dashboard
|
||||
Namespace: kube-system
|
||||
Labels: app=kubernetes-dashboard
|
||||
Selector: app=kubernetes-dashboard
|
||||
Type: NodePort
|
||||
IP: 10.110.207.144
|
||||
Port: <unset> 80/TCP
|
||||
NodePort: <unset> 32739/TCP
|
||||
Endpoints: 10.25.178.65:9090
|
||||
Session Affinity: None
|
||||
No events.
|
||||
admin@kubenode01:~$
|
||||
|
||||
Now you should be able to resolve the host
|
||||
``kubernetes-dashboard.kube-system.svc.cluster.local``:
|
||||
|
||||
::
|
||||
|
||||
admin@kubenode01:~$ kubectl exec kube-controller-manager-kubenode01 -it -n kube-system -- ping kubernetes-dashboard.kube-system.svc.cluster.local
|
||||
PING kubernetes-dashboard.kube-system.svc.cluster.local (10.110.207.144) 56(84) bytes of data.
|
||||
|
||||
.. note::
|
||||
This host example above has ``iputils-ping`` installed.
|
||||
search cluster.local svc.cluster.local
|
||||
EOF"
|
||||
|
||||
Kubernetes Node DNS Resolution
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
|
@ -358,42 +271,15 @@ Download the latest copy of Openstack-Helm:
|
|||
Ceph Preparation and Installation
|
||||
---------------------------------
|
||||
|
||||
Ceph must be aware of the OSX cluster and public networks. These CIDR
|
||||
Ceph must be aware of the OSD cluster and public networks. These CIDR
|
||||
ranges are the exact same ranges you used earlier in your Calico
|
||||
deployment yaml (our example was 10.25.0.0/16 due to our 192.168.0.0/16
|
||||
overlap). Explore this variable to your deployment environment by
|
||||
deployment yaml. Export this variable to your deployment environment by
|
||||
issuing the following commands:
|
||||
|
||||
::
|
||||
|
||||
export osd_cluster_network=10.25.0.0/16
|
||||
export osd_public_network=10.25.0.0/16
|
||||
|
||||
Ceph Storage Volumes
|
||||
--------------------
|
||||
|
||||
Ceph must also have volumes to mount on each host labeled for
|
||||
``ceph-storage``. On each host that you labeled, create the following
|
||||
directory (can be overriden):
|
||||
|
||||
::
|
||||
|
||||
mkdir -p /var/lib/openstack-helm/ceph
|
||||
|
||||
*Repeat this step for each node labeled: ``ceph-storage``*
|
||||
|
||||
Ceph Secrets Generation
|
||||
-----------------------
|
||||
|
||||
Although you can bring your own secrets, we have conveniently created a
|
||||
secret generation tool for you (for greenfield deployments). You can
|
||||
create secrets for your project by issuing the following:
|
||||
|
||||
::
|
||||
|
||||
cd helm-toolkit/utils/secret-generator
|
||||
./generate_secrets.sh all `./generate_secrets.sh fsid`
|
||||
cd ../../..
|
||||
export osd_cluster_network=192.168.0.0/16
|
||||
export osd_public_network=192.168.0.0/16
|
||||
|
||||
Nova Compute Instance Storage
|
||||
-----------------------------
|
||||
|
|
@ -468,30 +354,55 @@ the following command to install Ceph:
|
|||
|
||||
::
|
||||
|
||||
helm install --set network.public=$osd_public_network --name=ceph local/ceph --namespace=ceph
|
||||
helm install --namespace=ceph local/ceph --name=ceph \
|
||||
--set manifests_enabled.client_secrets=false \
|
||||
--set network.public=$osd_public_network \
|
||||
--set network.cluster=$osd_cluster_network
|
||||
|
||||
Bootstrap Installation
|
||||
----------------------
|
||||
Activating Control-Plane Namespace for Ceph
|
||||
-------------------------------------------
|
||||
|
||||
At this time (and before verification of Ceph) you'll need to install
|
||||
the ``bootstrap`` chart. The ``bootstrap`` chart will install secrets
|
||||
for both the ``ceph`` and ``openstack`` namespaces for the general
|
||||
StorageClass:
|
||||
In order for Ceph to fulfill PersistentVolumeClaims within Kubernetes namespaces
|
||||
outside of Ceph's namespace, a client keyring needs to be present within that
|
||||
namespace. For the rest of the OpenStack and supporting core services, this guide
|
||||
will be deploying the control plane to a seperate namespace ``openstack``. To
|
||||
deploy the aforementioned client keyring to the ``openstack`` namespace:
|
||||
|
||||
::
|
||||
|
||||
helm install --name=bootstrap-ceph local/bootstrap --namespace=ceph
|
||||
helm install --name=bootstrap-openstack local/bootstrap --namespace=openstack
|
||||
helm install --namespace=openstack local/ceph --name=ceph-openstack-config \
|
||||
--set manifests_enabled.storage_secrets=false \
|
||||
--set manifests_enabled.deployment=false \
|
||||
--set ceph.namespace=ceph \
|
||||
--set network.public=$osd_public_network \
|
||||
--set network.cluster=$osd_cluster_network
|
||||
|
||||
This will load the client keyring as well as the same ``ceph.conf`` into
|
||||
the specified namespace. Deploying ceph.conf into this namespace allows
|
||||
OpenStack services to consume this ConfigMap for their Ceph-specific
|
||||
configurations.
|
||||
|
||||
You may want to validate that Ceph is deployed successfully. For more
|
||||
information on this, please see the section entitled `Ceph
|
||||
Troubleshooting <../../operator/troubleshooting/persistent-storage.html>`__.
|
||||
|
||||
Ceph pool creation
|
||||
------------------
|
||||
|
||||
You should now be ready to create the pools for OpenStack services to consume,
|
||||
using the following commands:
|
||||
|
||||
::
|
||||
|
||||
kubectl exec -n ceph ceph-mon-0 -- ceph osd pool create volumes 8
|
||||
kubectl exec -n ceph ceph-mon-0 -- ceph osd pool create images 8
|
||||
kubectl exec -n ceph ceph-mon-0 -- ceph osd pool create vms 8
|
||||
|
||||
MariaDB Installation and Verification
|
||||
-------------------------------------
|
||||
|
||||
We are using Galera to cluster MariaDB and establish a quorum. To
|
||||
install the MariaDB, issue the following command:
|
||||
We are using Galera to cluster MariaDB. To install MariaDB, issue the following
|
||||
command:
|
||||
|
||||
::
|
||||
|
||||
|
|
@ -514,43 +425,49 @@ Now you can easily install the other services simply by going in order:
|
|||
|
||||
::
|
||||
|
||||
helm install --name=keystone local/keystone --set replicas=2 --namespace=openstack
|
||||
helm install --namespace=openstack --name=keystone local/keystone \
|
||||
--set replicas=2
|
||||
|
||||
**Install Horizon:**
|
||||
|
||||
::
|
||||
|
||||
helm install --name=horizon local/horizon --set network.enable_node_port=true --namespace=openstack
|
||||
helm install --namespace=openstack --name=horizon local/horizon \
|
||||
--set network.enable_node_port=true
|
||||
|
||||
**Install Glance:**
|
||||
|
||||
::
|
||||
|
||||
helm install --name=glance local/glance --set replicas.api=2,replicas.registry=2 --namespace=openstack
|
||||
helm install --namespace=openstack --name=glance local/glance \
|
||||
--set replicas.api=2,replicas.registry=2
|
||||
|
||||
**Install Heat:**
|
||||
|
||||
::
|
||||
|
||||
helm install --name=heat local/heat --namespace=openstack
|
||||
helm install --namespace=openstack --name=heat local/heat
|
||||
|
||||
**Install Neutron:**
|
||||
|
||||
::
|
||||
|
||||
helm install --name=neutron local/neutron --set replicas.server=2 --namespace=openstack
|
||||
helm install --namespace=openstack --name=neutron local/neutron \
|
||||
--set replicas.server=2
|
||||
|
||||
**Install Nova:**
|
||||
|
||||
::
|
||||
|
||||
helm install --name=nova local/nova --set control_replicas=2 --namespace=openstack
|
||||
helm install --namespace=openstack --name=nova local/nova \
|
||||
--set control_replicas=2
|
||||
|
||||
**Install Cinder:**
|
||||
|
||||
::
|
||||
|
||||
helm install --name=cinder local/cinder --set replicas.api=2 --namespace=openstack
|
||||
helm install --namespace=openstack --name=cinder local/cinder \
|
||||
--set replicas.api=2
|
||||
|
||||
Final Checks
|
||||
------------
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2017 The Openstack-Helm Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
|
|
@ -12,9 +14,16 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -ex
|
||||
export HOME=/tmp
|
||||
|
||||
cat <<EOF > /etc/ceph/ceph.client.{{ .Values.conf.glance.glance_store.glance.store.rbd_store_user }}.keyring
|
||||
[client.{{ .Values.conf.glance.glance_store.glance.store.rbd_store_user }}]
|
||||
{{- if .Values.conf.ceph.keyring }}
|
||||
key = {{ .Values.conf.ceph.keyring }}
|
||||
{{- else }}
|
||||
key = {{- include "secrets/ceph-client-key" . -}}
|
||||
key = $(cat /tmp/client-keyring)
|
||||
{{- end }}
|
||||
EOF
|
||||
|
||||
exit 0
|
||||
|
|
@ -35,3 +35,5 @@ data:
|
|||
{{ tuple "bin/_glance-registry.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
bootstrap.sh: |+
|
||||
{{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
ceph-keyring.sh: |+
|
||||
{{ tuple "bin/_ceph-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
|
|
|
|||
|
|
@ -78,30 +78,6 @@ data:
|
|||
{{- end }}
|
||||
{{- if .Values.conf.rally_tests.append -}}
|
||||
{{ .Values.conf.rally_tests.append | indent 4 }}
|
||||
{{- end }}
|
||||
ceph.conf: |+
|
||||
{{ if .Values.conf.ceph.override -}}
|
||||
{{ .Values.conf.ceph.override | indent 4 }}
|
||||
{{- else -}}
|
||||
{{- if .Values.conf.ceph.prefix -}}
|
||||
{{ .Values.conf.ceph.prefix | indent 4 }}
|
||||
{{- end }}
|
||||
{{ tuple "etc/_ceph.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
{{- end }}
|
||||
{{- if .Values.conf.ceph.append -}}
|
||||
{{ .Values.conf.ceph.append | indent 4 }}
|
||||
{{- end }}
|
||||
ceph.client.{{ .Values.conf.glance.glance_store.glance.store.rbd_store_user }}.keyring: |+
|
||||
{{ if .Values.conf.ceph_client.override -}}
|
||||
{{ .Values.conf.ceph_client.override | indent 4 }}
|
||||
{{- else -}}
|
||||
{{- if .Values.conf.ceph_client.prefix -}}
|
||||
{{ .Values.conf.ceph_client.prefix | indent 4 }}
|
||||
{{- end }}
|
||||
{{ tuple "etc/_ceph.client.glance.keyring.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
{{- end }}
|
||||
{{- if .Values.conf.ceph_client.append -}}
|
||||
{{ .Values.conf.ceph_client.append | indent 4 }}
|
||||
{{- end }}
|
||||
glance-api.conf: |+
|
||||
{{ if .Values.conf.glance.override -}}
|
||||
|
|
|
|||
|
|
@ -43,6 +43,47 @@ spec:
|
|||
terminationGracePeriodSeconds: {{ .Values.termination_grace_period.api.timeout | default "600" }}
|
||||
initContainers:
|
||||
{{ tuple $envAll $dependencies $mounts_glance_api_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
|
||||
{{- if eq .Values.storage "pvc" }}
|
||||
- name: glance-perms
|
||||
image: {{ .Values.images.api }}
|
||||
imagePullPolicy: {{ .Values.images.pull_policy }}
|
||||
securityContext:
|
||||
runAsUser: 0
|
||||
{{- if .Values.resources.enabled }}
|
||||
resources:
|
||||
limits:
|
||||
cpu: {{ .Values.resources.api.limits.cpu | quote }}
|
||||
memory: {{ .Values.resources.api.limits.memory | quote }}
|
||||
requests:
|
||||
cpu: {{ .Values.resources.api.requests.cpu | quote }}
|
||||
memory: {{ .Values.resources.api.requests.memory | quote }}
|
||||
{{- end }}
|
||||
command:
|
||||
- chown
|
||||
- -R
|
||||
- "glance:"
|
||||
- {{ .Values.conf.glance.glance_store.glance.store.filesystem_store_datadir }}
|
||||
volumeMounts:
|
||||
- name: glance-images
|
||||
mountPath: {{ .Values.conf.glance.glance_store.glance.store.filesystem_store_datadir }}
|
||||
{{- end }}
|
||||
{{ if eq .Values.storage "ceph" }}
|
||||
- name: ceph-keyring-placement
|
||||
image: {{ .Values.images.api }}
|
||||
imagePullPolicy: {{ .Values.images.pull_policy }}
|
||||
command:
|
||||
- /tmp/ceph-keyring.sh
|
||||
volumeMounts:
|
||||
- name: etcceph
|
||||
mountPath: /etc/ceph
|
||||
- name: glance-bin
|
||||
mountPath: /tmp/ceph-keyring.sh
|
||||
subPath: ceph-keyring.sh
|
||||
- name: ceph-keyring
|
||||
mountPath: /tmp/client-keyring
|
||||
subPath: key
|
||||
readOnly: true
|
||||
{{ end }}
|
||||
containers:
|
||||
- name: glance-api
|
||||
image: {{ .Values.images.api }}
|
||||
|
|
@ -93,13 +134,15 @@ spec:
|
|||
- name: glance-images
|
||||
mountPath: {{ .Values.conf.glance.glance_store.glance.store.filesystem_store_datadir }}
|
||||
{{- else }}
|
||||
- name: glance-etc
|
||||
- name: etcceph
|
||||
mountPath: /etc/ceph
|
||||
- name: ceph-etc
|
||||
mountPath: /etc/ceph/ceph.conf
|
||||
subPath: ceph.conf
|
||||
readOnly: true
|
||||
- name: glance-etc
|
||||
mountPath: /etc/ceph/ceph.client.{{ .Values.conf.glance.glance_store.glance.store.rbd_store_user }}.keyring
|
||||
subPath: ceph.client.{{ .Values.conf.glance.glance_store.glance.store.rbd_store_user }}.keyring
|
||||
- name: ceph-keyring
|
||||
mountPath: /tmp/client-keyring
|
||||
subPath: key
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{ if $mounts_glance_api.volumeMounts }}{{ toYaml $mounts_glance_api.volumeMounts | indent 12 }}{{ end }}
|
||||
|
|
@ -117,5 +160,14 @@ spec:
|
|||
- name: glance-images
|
||||
persistentVolumeClaim:
|
||||
claimName: glance-images
|
||||
{{ else }}
|
||||
- name: etcceph
|
||||
emptyDir: {}
|
||||
- name: ceph-etc
|
||||
configMap:
|
||||
name: ceph-etc
|
||||
- name: ceph-keyring
|
||||
secret:
|
||||
secretName: pvc-ceph-client-key
|
||||
{{- end }}
|
||||
{{ if $mounts_glance_api.volumes }}{{ toYaml $mounts_glance_api.volumes | indent 8 }}{{ end }}
|
||||
|
|
|
|||
|
|
@ -1,30 +0,0 @@
|
|||
# Copyright 2017 The Openstack-Helm Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
[global]
|
||||
rgw_thread_pool_size = 1024
|
||||
rgw_num_rados_handles = 100
|
||||
{{- if .Values.conf.ceph.monitors }}
|
||||
[mon]
|
||||
{{ range .Values.conf.ceph.monitors }}
|
||||
[mon.{{ . }}]
|
||||
host = {{ . }}
|
||||
mon_addr = {{ . }}
|
||||
{{ end }}
|
||||
{{- else }}
|
||||
mon_host = ceph-mon.ceph
|
||||
{{- end }}
|
||||
[client]
|
||||
rbd_cache_enabled = true
|
||||
rbd_cache_writethrough_until_flush = true
|
||||
|
|
@ -1,78 +0,0 @@
|
|||
Ceph Kubernetes Secret Generation
|
||||
=================================
|
||||
|
||||
This script will generate ceph keyrings and configs as Kubernetes
|
||||
secrets.
|
||||
|
||||
Sigil is required for template handling and must be installed in system
|
||||
``PATH``. Instructions can be found`here
|
||||
<https://github.com/gliderlabs/sigil>`__
|
||||
|
||||
The following functions are provided:
|
||||
|
||||
Generate raw FSID (can be used for other functions)
|
||||
---------------------------------------------------
|
||||
|
||||
.. code:: bash
|
||||
|
||||
./generate_secrets.sh fsid
|
||||
|
||||
Generate raw ceph.conf (For verification)
|
||||
-----------------------------------------
|
||||
|
||||
.. code:: bash
|
||||
|
||||
./generate_secrets.sh ceph-conf-raw <fsid> "overridekey=value"
|
||||
|
||||
Take a look at ``ceph/ceph.conf.tmpl`` for the default values
|
||||
|
||||
Generate encoded ceph.conf secret
|
||||
---------------------------------
|
||||
|
||||
.. code:: bash
|
||||
|
||||
./generate_secrets.sh ceph-conf <fsid> "overridekey=value"
|
||||
|
||||
Generate encoded admin keyring secret
|
||||
-------------------------------------
|
||||
|
||||
.. code:: bash
|
||||
|
||||
./generate_secrets.sh admin-keyring
|
||||
|
||||
Generate encoded mon keyring secret
|
||||
-----------------------------------
|
||||
|
||||
.. code:: bash
|
||||
|
||||
./generate_secrets.sh mon-keyring
|
||||
|
||||
Generate a combined secret
|
||||
--------------------------
|
||||
|
||||
Contains ceph.conf, admin keyring and mon keyring. Useful for generating
|
||||
the ``/etc/ceph`` directory
|
||||
|
||||
.. code:: bash
|
||||
|
||||
./generate_secrets.sh combined-conf
|
||||
|
||||
Generate encoded boostrap keyring secret
|
||||
----------------------------------------
|
||||
|
||||
.. code:: bash
|
||||
|
||||
./generate_secrets.sh bootstrap-keyring <osd|mds|rgw>
|
||||
|
||||
Kubernetes workflow
|
||||
===================
|
||||
|
||||
.. code:: bash
|
||||
|
||||
./generator/generate_secrets.sh all `./generate_secrets.sh fsid`
|
||||
|
||||
kubectl create secret generic ceph-conf-combined --from-file=ceph.conf --from-file=ceph.client.admin.keyring --from-file=ceph.mon.keyring --namespace=ceph
|
||||
kubectl create secret generic ceph-bootstrap-rgw-keyring --from-file=ceph.keyring=ceph.rgw.keyring --namespace=ceph
|
||||
kubectl create secret generic ceph-bootstrap-mds-keyring --from-file=ceph.keyring=ceph.mds.keyring --namespace=ceph
|
||||
kubectl create secret generic ceph-bootstrap-osd-keyring --from-file=ceph.keyring=ceph.osd.keyring --namespace=ceph
|
||||
kubectl create secret generic ceph-client-key --from-file=ceph-client-key --namespace=ceph
|
||||
|
|
@ -1,96 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2017 The Openstack-Helm Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
gen-fsid() {
|
||||
echo "$(uuidgen)"
|
||||
}
|
||||
|
||||
gen-ceph-conf-raw() {
|
||||
fsid=${1:?}
|
||||
shift
|
||||
conf=$(sigil -p -f templates/ceph/ceph.conf.tmpl "fsid=${fsid}" $@)
|
||||
echo "${conf}"
|
||||
}
|
||||
|
||||
gen-ceph-conf() {
|
||||
fsid=${1:?}
|
||||
shift
|
||||
conf=$(sigil -p -f templates/ceph/ceph.conf.tmpl "fsid=${fsid}" $@)
|
||||
echo "${conf}"
|
||||
}
|
||||
|
||||
gen-admin-keyring() {
|
||||
key=$(python ceph-key.py)
|
||||
keyring=$(sigil -f templates/ceph/admin.keyring.tmpl "key=${key}")
|
||||
echo "${keyring}"
|
||||
}
|
||||
|
||||
gen-mon-keyring() {
|
||||
key=$(python ceph-key.py)
|
||||
keyring=$(sigil -f templates/ceph/mon.keyring.tmpl "key=${key}")
|
||||
echo "${keyring}"
|
||||
}
|
||||
|
||||
gen-combined-conf() {
|
||||
fsid=${1:?}
|
||||
shift
|
||||
conf=$(sigil -p -f templates/ceph/ceph.conf.tmpl "fsid=${fsid}" $@)
|
||||
echo "${conf}" > ../../secrets/ceph.conf
|
||||
|
||||
key=$(python ceph-key.py)
|
||||
keyring=$(sigil -f templates/ceph/admin.keyring.tmpl "key=${key}")
|
||||
echo "${key}" > ../../secrets/ceph-client-key
|
||||
echo "${keyring}" > ../../secrets/ceph.client.admin.keyring
|
||||
|
||||
key=$(python ceph-key.py)
|
||||
keyring=$(sigil -f templates/ceph/mon.keyring.tmpl "key=${key}")
|
||||
echo "${keyring}" > ../../secrets/ceph.mon.keyring
|
||||
}
|
||||
|
||||
gen-bootstrap-keyring() {
|
||||
service="${1:-osd}"
|
||||
key=$(python ceph-key.py)
|
||||
bootstrap=$(sigil -f templates/ceph/bootstrap.keyring.tmpl "key=${key}" "service=${service}")
|
||||
echo "${bootstrap}"
|
||||
}
|
||||
|
||||
gen-all-bootstrap-keyrings() {
|
||||
gen-bootstrap-keyring osd > ../../secrets/ceph.osd.keyring
|
||||
gen-bootstrap-keyring mds > ../../secrets/ceph.mds.keyring
|
||||
gen-bootstrap-keyring rgw > ../../secrets/ceph.rgw.keyring
|
||||
}
|
||||
|
||||
gen-all() {
|
||||
gen-combined-conf $@
|
||||
gen-all-bootstrap-keyrings
|
||||
}
|
||||
|
||||
|
||||
main() {
|
||||
set -eo pipefail
|
||||
case "$1" in
|
||||
fsid) shift; gen-fsid $@;;
|
||||
ceph-conf-raw) shift; gen-ceph-conf-raw $@;;
|
||||
ceph-conf) shift; gen-ceph-conf $@;;
|
||||
admin-keyring) shift; gen-admin-keyring $@;;
|
||||
mon-keyring) shift; gen-mon-keyring $@;;
|
||||
bootstrap-keyring) shift; gen-bootstrap-keyring $@;;
|
||||
combined-conf) shift; gen-combined-conf $@;;
|
||||
all) shift; gen-all $@;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
# Copyright 2017 The Openstack-Helm Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
[client.admin]
|
||||
key = {{ $key }}
|
||||
auid = 0
|
||||
caps mds = "allow"
|
||||
caps mon = "allow *"
|
||||
caps osd = "allow *"
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
# Copyright 2017 The Openstack-Helm Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
[client.bootstrap-{{ $service }}]
|
||||
key = {{ $key }}
|
||||
caps mon = "allow profile bootstrap-{{ $service }}"
|
||||
|
|
@ -1,85 +0,0 @@
|
|||
# Copyright 2017 The Openstack-Helm Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
[global]
|
||||
fsid = ${fsid:?}
|
||||
cephx = ${auth_cephx:-"true"}
|
||||
cephx_require_signatures = ${auth_cephx_require_signatures:-"false"}
|
||||
cephx_cluster_require_signatures = ${auth_cephx_cluster_require_signatures:-"true"}
|
||||
cephx_service_require_signatures = ${auth_cephx_service_require_signatures:-"false"}
|
||||
|
||||
# auth
|
||||
max_open_files = ${global_max_open_files:-"131072"}
|
||||
osd_pool_default_pg_num = ${global_osd_pool_default_pg_num:-"128"}
|
||||
osd_pool_default_pgp_num = ${global_osd_pool_default_pgp_num:-"128"}
|
||||
osd_pool_default_size = ${global_osd_pool_default_size:-"3"}
|
||||
osd_pool_default_min_size = ${global_osd_pool_default_min_size:-"1"}
|
||||
|
||||
mon_osd_full_ratio = ${global_mon_osd_full_ratio:-".95"}
|
||||
mon_osd_nearfull_ratio = ${global_mon_osd_nearfull_ratio:-".85"}
|
||||
|
||||
mon_host = ${global_mon_host:-'ceph-mon'}
|
||||
|
||||
[mon]
|
||||
mon_osd_down_out_interval = ${mon_mon_osd_down_out_interval:-"600"}
|
||||
mon_osd_min_down_reporters = ${mon_mon_osd_min_down_reporters:-"4"}
|
||||
mon_clock_drift_allowed = ${mon_mon_clock_drift_allowed:-".15"}
|
||||
mon_clock_drift_warn_backoff = ${mon_mon_clock_drift_warn_backoff:-"30"}
|
||||
mon_osd_report_timeout = ${mon_mon_osd_report_timeout:-"300"}
|
||||
|
||||
|
||||
[osd]
|
||||
journal_size = ${osd_journal_size:-"100"}
|
||||
cluster_network = ${osd_cluster_network:-'192.168.0.0/16'}
|
||||
public_network = ${osd_public_network:-'192.168.0.0/16'}
|
||||
osd_mkfs_type = ${osd_osd_mkfs_type:-"xfs"}
|
||||
osd_mkfs_options_xfs = ${osd_osd_mkfs_options_xfs:-"-f -i size=2048"}
|
||||
osd_mon_heartbeat_interval = ${osd_osd_mon_heartbeat_interval:-"30"}
|
||||
osd_max_object_name_len = ${osd_max_object_name_len:-"256"}
|
||||
|
||||
#crush
|
||||
osd_pool_default_crush_rule = ${osd_pool_default_crush_rule:-"0"}
|
||||
osd_crush_update_on_start = ${osd_osd_crush_update_on_start:-"true"}
|
||||
|
||||
#backend
|
||||
osd_objectstore = ${osd_osd_objectstore:-"filestore"}
|
||||
|
||||
#performance tuning
|
||||
filestore_merge_threshold = ${osd_filestore_merge_threshold:-"40"}
|
||||
filestore_split_multiple = ${osd_filestore_split_multiple:-"8"}
|
||||
osd_op_threads = ${osd_osd_op_threads:-"8"}
|
||||
filestore_op_threads = ${osd_filestore_op_threads:-"8"}
|
||||
filestore_max_sync_interval = ${osd_filestore_max_sync_interval:-"5"}
|
||||
osd_max_scrubs = ${osd_osd_max_scrubs:-"1"}
|
||||
|
||||
|
||||
#recovery tuning
|
||||
osd_recovery_max_active = ${osd_osd_recovery_max_active:-"5"}
|
||||
osd_max_backfills = ${osd_osd_max_backfills:-"2"}
|
||||
osd_recovery_op_priority = ${osd_osd_recovery_op_priority:-"2"}
|
||||
osd_client_op_priority = ${osd_osd_client_op_priority:-"63"}
|
||||
osd_recovery_max_chunk = ${osd_osd_recovery_max_chunk:-"1048576"}
|
||||
osd_recovery_threads = ${osd_osd_recovery_threads:-"1"}
|
||||
|
||||
#ports
|
||||
ms_bind_port_min = ${osd_ms_bind_port_min:-"6800"}
|
||||
ms_bind_port_max = ${osd_ms_bind_port_max:-"7100"}
|
||||
|
||||
[client]
|
||||
rbd_cache_enabled = ${client_rbd_cache_enabled:-"true"}
|
||||
rbd_cache_writethrough_until_flush = ${client_rbd_cache_writethrough_until_flush:-"true"}
|
||||
rbd_default_features = ${client_rbd_default_features:-"1"}
|
||||
|
||||
[mds]
|
||||
mds_cache_size = ${mds_mds_cache_size:-"100000"}
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
# Copyright 2017 The Openstack-Helm Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
[mon.]
|
||||
key = {{ $key }}
|
||||
caps mon = "allow *"
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
# Copyright 2017 The Openstack-Helm Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: ceph-rbd-test
|
||||
spec:
|
||||
containers:
|
||||
- name: cephrbd-rw
|
||||
image: busybox
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- while true; do sleep 1; done
|
||||
volumeMounts:
|
||||
- mountPath: "/mnt/cephrbd"
|
||||
name: cephrbd
|
||||
volumes:
|
||||
- name: cephrbd
|
||||
rbd:
|
||||
monitors:
|
||||
#This only works if you have skyDNS resolveable from the kubernetes node. Otherwise you must manually put in one or more mon pod ips.
|
||||
- ceph-mon.ceph:6789
|
||||
user: admin
|
||||
image: ceph-rbd-test
|
||||
pool: rbd
|
||||
secretRef:
|
||||
name: ceph-client-key
|
||||
|
|
@ -1,3 +1,5 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2017 The Openstack-Helm Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
|
|
@ -12,9 +14,16 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -ex
|
||||
export HOME=/tmp
|
||||
|
||||
cat <<EOF > /etc/ceph/ceph.client.keyring
|
||||
[client.{{ .Values.ceph.cinder_user }}]
|
||||
{{- if .Values.ceph.cinder_keyring }}
|
||||
key = {{ .Values.ceph.cinder_keyring }}
|
||||
{{- else }}
|
||||
key = {{- include "secrets/ceph-client-key" . -}}
|
||||
key = $(cat /tmp/client-keyring)
|
||||
{{- end }}
|
||||
EOF
|
||||
|
||||
exit 0
|
||||
|
|
@ -33,7 +33,7 @@ data:
|
|||
{{ tuple "bin/_libvirt.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
{{- if .Values.ceph.enabled }}
|
||||
ceph-secret-define.sh: |
|
||||
{{ tuple "bin/_ceph-secret-define.sh.tpl" . | include "helm-toolkit.template" | indent 4 }}
|
||||
{{ tuple "bin/_ceph-secret-define.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
{{- end }}
|
||||
bootstrap.sh: |
|
||||
{{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
|
|
@ -51,3 +51,5 @@ data:
|
|||
{{ tuple "bin/_nova-scheduler.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
fake-iptables.sh: |
|
||||
{{ tuple "bin/_fake-iptables.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
ceph-keyring.sh: |+
|
||||
{{ tuple "bin/_ceph-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
|
|
|
|||
|
|
@ -92,8 +92,6 @@ data:
|
|||
{{- else -}}
|
||||
{{ tuple "etc/_policy.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
{{- end }}
|
||||
ceph.client.cinder.keyring.yaml: |+
|
||||
{{ tuple "etc/_ceph.client.cinder.keyring.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
resolv.conf: |+
|
||||
{{ tuple "etc/_resolv.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
libvirtd.conf: |+
|
||||
|
|
|
|||
|
|
@ -39,6 +39,23 @@ spec:
|
|||
dnsPolicy: ClusterFirst
|
||||
initContainers:
|
||||
{{ tuple $envAll $dependencies $mounts_nova_compute_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
|
||||
{{- if .Values.ceph.enabled }}
|
||||
- name: ceph-keyring-placement
|
||||
image: {{ .Values.images.compute }}
|
||||
imagePullPolicy: {{ .Values.images.pull_policy }}
|
||||
command:
|
||||
- /tmp/ceph-keyring.sh
|
||||
volumeMounts:
|
||||
- name: etcceph
|
||||
mountPath: /etc/ceph
|
||||
- name: nova-bin
|
||||
mountPath: /tmp/ceph-keyring.sh
|
||||
subPath: ceph-keyring.sh
|
||||
- name: ceph-keyring
|
||||
mountPath: /tmp/client-keyring
|
||||
subPath: key
|
||||
readOnly: true
|
||||
{{ end }}
|
||||
containers:
|
||||
- name: nova-compute
|
||||
image: {{ .Values.images.compute }}
|
||||
|
|
@ -77,14 +94,18 @@ spec:
|
|||
mountPath: /etc/resolv.conf
|
||||
subPath: resolv.conf
|
||||
readOnly: true
|
||||
- name: nova-etc
|
||||
{{- if .Values.ceph.enabled }}
|
||||
- name: etcceph
|
||||
mountPath: /etc/ceph
|
||||
- name: ceph-etc
|
||||
mountPath: /etc/ceph/ceph.conf
|
||||
subPath: ceph.conf
|
||||
readOnly: true
|
||||
- name: nova-etc
|
||||
mountPath: /etc/ceph/ceph.client.keyring
|
||||
subPath: ceph.client.keyring
|
||||
- name: ceph-keyring
|
||||
mountPath: /tmp/client-keyring
|
||||
subPath: key
|
||||
readOnly: true
|
||||
{{ end }}
|
||||
- mountPath: /lib/modules
|
||||
name: libmodules
|
||||
readOnly: true
|
||||
|
|
@ -105,6 +126,16 @@ spec:
|
|||
- name: nova-etc
|
||||
configMap:
|
||||
name: nova-etc
|
||||
{{- if .Values.ceph.enabled }}
|
||||
- name: etcceph
|
||||
emptyDir: {}
|
||||
- name: ceph-etc
|
||||
configMap:
|
||||
name: ceph-etc
|
||||
- name: ceph-keyring
|
||||
secret:
|
||||
secretName: pvc-ceph-client-key
|
||||
{{ end }}
|
||||
- name: libmodules
|
||||
hostPath:
|
||||
path: /lib/modules
|
||||
|
|
|
|||
|
|
@ -38,6 +38,23 @@ spec:
|
|||
dnsPolicy: ClusterFirst
|
||||
initContainers:
|
||||
{{ tuple $envAll $dependencies $mounts_nova_libvirt_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
|
||||
{{- if .Values.ceph.enabled }}
|
||||
- name: ceph-keyring-placement
|
||||
image: {{ .Values.images.libvirt }}
|
||||
imagePullPolicy: {{ .Values.images.pull_policy }}
|
||||
command:
|
||||
- /tmp/ceph-keyring.sh
|
||||
volumeMounts:
|
||||
- name: etcceph
|
||||
mountPath: /etc/ceph
|
||||
- name: nova-bin
|
||||
mountPath: /tmp/ceph-keyring.sh
|
||||
subPath: ceph-keyring.sh
|
||||
- name: ceph-keyring
|
||||
mountPath: /tmp/client-keyring
|
||||
subPath: key
|
||||
readOnly: true
|
||||
{{ end }}
|
||||
containers:
|
||||
- name: nova-libvirt
|
||||
image: {{ .Values.images.libvirt }}
|
||||
|
|
@ -101,13 +118,15 @@ spec:
|
|||
- name: cgroup
|
||||
mountPath: /sys/fs/cgroup
|
||||
{{- if .Values.ceph.enabled }}
|
||||
- name: nova-etc
|
||||
- name: etcceph
|
||||
mountPath: /etc/ceph
|
||||
- name: ceph-etc
|
||||
mountPath: /etc/ceph/ceph.conf
|
||||
subPath: ceph.conf
|
||||
readOnly: true
|
||||
- name: nova-etc
|
||||
mountPath: /etc/ceph/ceph.client.keyring
|
||||
subPath: ceph.client.keyring
|
||||
- name: ceph-keyring
|
||||
mountPath: /tmp/client-keyring
|
||||
subPath: key
|
||||
readOnly: true
|
||||
- name: nova-bin
|
||||
mountPath: /tmp/ceph-secret-define.sh
|
||||
|
|
@ -123,6 +142,16 @@ spec:
|
|||
- name: nova-etc
|
||||
configMap:
|
||||
name: nova-etc
|
||||
{{- if .Values.ceph.enabled }}
|
||||
- name: etcceph
|
||||
emptyDir: {}
|
||||
- name: ceph-etc
|
||||
configMap:
|
||||
name: ceph-etc
|
||||
- name: ceph-keyring
|
||||
secret:
|
||||
secretName: pvc-ceph-client-key
|
||||
{{ end }}
|
||||
- name: libmodules
|
||||
hostPath:
|
||||
path: /lib/modules
|
||||
|
|
|
|||
|
|
@ -1,32 +0,0 @@
|
|||
# Copyright 2017 The Openstack-Helm Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
[global]
|
||||
rgw_thread_pool_size = 1024
|
||||
rgw_num_rados_handles = 100
|
||||
{{- if .Values.ceph.enabled }}
|
||||
[mon]
|
||||
{{- if .Values.ceph.monitors }}
|
||||
{{ range .Values.ceph.monitors }}
|
||||
[mon.{{ . }}]
|
||||
host = {{ . }}
|
||||
mon_addr = {{ . }}
|
||||
{{ end }}
|
||||
{{- else }}
|
||||
mon_host = ceph-mon.ceph
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
[client]
|
||||
rbd_cache_enabled = true
|
||||
rbd_cache_writethrough_until_flush = true
|
||||
|
|
@ -130,7 +130,6 @@ keystone:
|
|||
admin_region_name: "RegionOne"
|
||||
nova_user_role: "admin"
|
||||
|
||||
|
||||
ceph:
|
||||
enabled: false
|
||||
monitors: []
|
||||
|
|
|
|||
|
|
@ -12,15 +12,42 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
labels:
|
||||
version: v0.1.0
|
||||
test: ceph
|
||||
name: ceph-test-job
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: test
|
||||
image: docker.io/alpine:latest
|
||||
imagePullPolicy: Always
|
||||
command:
|
||||
- /bin/sh
|
||||
- -ec
|
||||
- |
|
||||
echo "Ceph PVC Mount Test Passed"
|
||||
volumeMounts:
|
||||
- name: ceph-mount
|
||||
mountPath: /mnt/ceph
|
||||
volumes:
|
||||
- name: ceph-mount
|
||||
persistentVolumeClaim:
|
||||
claimName: ceph-test
|
||||
---
|
||||
kind: PersistentVolumeClaim
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: pvc-test
|
||||
name: ceph-test
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: general
|
||||
accessModes: [ "ReadWriteOnce" ]
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
storageClassName: general
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ integration test is below:
|
|||
|
||||
export INTEGRATION=aio
|
||||
export INTEGRATION_TYPE=basic
|
||||
export PVC_BACKEND=ceph
|
||||
./tools/gate/setup_gate.sh
|
||||
|
||||
Supported Platforms
|
||||
|
|
|
|||
|
|
@ -29,14 +29,53 @@ if [ "x$HOST_OS" == "xfedora" ]; then
|
|||
sudo modprobe ip6_tables
|
||||
fi
|
||||
|
||||
if [ "x$PVC_BACKEND" == "xceph" ]; then
|
||||
kubectl label nodes ceph-storage=enabled --all
|
||||
CONTROLLER_MANAGER_POD=$(kubectl get -n kube-system pods -l component=kube-controller-manager --no-headers -o name | head -1 | awk -F '/' '{ print $NF }')
|
||||
kubectl exec -n kube-system ${CONTROLLER_MANAGER_POD} -- sh -c "cat > /etc/resolv.conf <<EOF
|
||||
nameserver 10.96.0.10
|
||||
nameserver 8.8.8.8
|
||||
search cluster.local svc.cluster.local
|
||||
EOF"
|
||||
|
||||
export osd_cluster_network=192.168.0.0/16
|
||||
export osd_public_network=192.168.0.0/16
|
||||
|
||||
helm install --namespace=ceph local/ceph --name=ceph2 \
|
||||
--set manifests_enabled.client_secrets=false \
|
||||
--set network.public=$osd_public_network \
|
||||
--set network.cluster=$osd_cluster_network
|
||||
|
||||
kube_wait_for_pods ceph 420
|
||||
|
||||
kubectl exec -n ceph ceph-mon-0 -- ceph -s
|
||||
|
||||
helm install --namespace=openstack local/ceph --name=ceph-openstack-config \
|
||||
--set manifests_enabled.storage_secrets=false \
|
||||
--set manifests_enabled.deployment=false \
|
||||
--set ceph.namespace=ceph \
|
||||
--set network.public=$osd_public_network \
|
||||
--set network.cluster=$osd_cluster_network
|
||||
|
||||
kube_wait_for_pods ceph 420
|
||||
|
||||
kubectl exec -n ceph ceph-mon-0 -- ceph osd pool create volumes 8
|
||||
kubectl exec -n ceph ceph-mon-0 -- ceph osd pool create images 8
|
||||
kubectl exec -n ceph ceph-mon-0 -- ceph osd pool create vms 8
|
||||
fi
|
||||
|
||||
helm install --namespace=openstack local/mariadb --name=mariadb
|
||||
helm install --namespace=openstack local/memcached --name=memcached
|
||||
helm install --namespace=openstack local/etcd --name=etcd-rabbitmq
|
||||
helm install --namespace=openstack local/rabbitmq --name=rabbitmq
|
||||
kube_wait_for_pods openstack 420
|
||||
helm install --namespace=openstack local/keystone --name=keystone
|
||||
helm install --namespace=openstack local/glance --name=glance \
|
||||
--values=${WORK_DIR}/tools/overrides/mvp/glance.yaml
|
||||
if [ "x$PVC_BACKEND" == "xceph" ]; then
|
||||
helm install --namespace=openstack local/glance --name=glance
|
||||
else
|
||||
helm install --namespace=openstack local/glance --name=glance \
|
||||
--values=${WORK_DIR}/tools/overrides/mvp/glance.yaml
|
||||
fi
|
||||
kube_wait_for_pods openstack 420
|
||||
helm install --namespace=openstack local/nova --name=nova \
|
||||
--values=${WORK_DIR}/tools/overrides/mvp/nova.yaml \
|
||||
|
|
@ -44,7 +83,12 @@ helm install --namespace=openstack local/nova --name=nova \
|
|||
helm install --namespace=openstack local/neutron --name=neutron \
|
||||
--values=${WORK_DIR}/tools/overrides/mvp/neutron.yaml
|
||||
kube_wait_for_pods openstack 420
|
||||
helm install --namespace=openstack local/cinder --name=cinder
|
||||
if [ "x$PVC_BACKEND" == "xceph" ]; then
|
||||
helm install --namespace=openstack local/cinder --name=cinder
|
||||
else
|
||||
helm install --namespace=openstack local/cinder --name=cinder \
|
||||
--values=${WORK_DIR}/tools/overrides/mvp/cinder.yaml
|
||||
fi
|
||||
helm install --namespace=openstack local/heat --name=heat
|
||||
helm install --namespace=openstack local/horizon --name=horizon
|
||||
kube_wait_for_pods openstack 420
|
||||
|
|
|
|||
|
|
@ -53,6 +53,15 @@ kubectl get svc -o json --all-namespaces | jq -r \
|
|||
${LOGS_DIR}/k8s/svc/$NAMESPACE-$NAME.txt
|
||||
done
|
||||
|
||||
mkdir -p ${LOGS_DIR}/k8s/pvc
|
||||
kubectl get pvc -o json --all-namespaces | jq -r \
|
||||
'.items[].metadata | .namespace + " " + .name' | while read line; do
|
||||
NAMESPACE=$(echo $line | awk '{print $1}')
|
||||
NAME=$(echo $line | awk '{print $2}')
|
||||
kubectl describe pvc $NAME --namespace $NAMESPACE > \
|
||||
${LOGS_DIR}/k8s/pvc/$NAMESPACE-$NAME.txt
|
||||
done
|
||||
|
||||
mkdir -p ${LOGS_DIR}/k8s/rbac
|
||||
for OBJECT_TYPE in clusterroles \
|
||||
roles \
|
||||
|
|
@ -76,5 +85,6 @@ sudo iptables-save > ${LOGS_DIR}/nodes/$(hostname)/iptables.txt
|
|||
sudo ip a > ${LOGS_DIR}/nodes/$(hostname)/ip.txt
|
||||
sudo route -n > ${LOGS_DIR}/nodes/$(hostname)/routes.txt
|
||||
arp -a > ${LOGS_DIR}/nodes/$(hostname)/arp.txt
|
||||
cat /etc/resolv.conf > ${LOGS_DIR}/nodes/$(hostname)/resolv.conf
|
||||
|
||||
exit $1
|
||||
|
|
|
|||
|
|
@ -29,3 +29,17 @@ function base_install {
|
|||
iptables
|
||||
fi
|
||||
}
|
||||
|
||||
function ceph_support_install {
|
||||
if [ "x$HOST_OS" == "xubuntu" ]; then
|
||||
sudo apt-get update -y
|
||||
sudo apt-get install -y --no-install-recommends -qq \
|
||||
ceph-common
|
||||
elif [ "x$HOST_OS" == "xcentos" ]; then
|
||||
sudo yum install -y \
|
||||
ceph
|
||||
elif [ "x$HOST_OS" == "xfedora" ]; then
|
||||
sudo dnf install -y \
|
||||
ceph
|
||||
fi
|
||||
}
|
||||
|
|
|
|||
|
|
@ -130,3 +130,9 @@ function kubeadm_aio_launch {
|
|||
kube_wait_for_pods kube-system 240
|
||||
kube_wait_for_pods default 240
|
||||
}
|
||||
|
||||
function ceph_kube_controller_manager_replace {
|
||||
sudo docker pull ${CEPH_KUBE_CONTROLLER_MANAGER_IMAGE}
|
||||
IMAGE_ID=$(sudo docker images ${CEPH_KUBE_CONTROLLER_MANAGER_IMAGE} -q)
|
||||
sudo docker tag ${IMAGE_ID} ${BASE_KUBE_CONTROLLER_MANAGER_IMAGE}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,3 +42,4 @@ function net_hosts_pre_kube {
|
|||
function net_hosts_post_kube {
|
||||
sudo cp -f /etc/hosts-pre-kube /etc/hosts
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -18,5 +18,11 @@ source ${WORK_DIR}/tools/gate/funcs/kube.sh
|
|||
|
||||
kubeadm_aio_reqs_install
|
||||
sudo docker pull ${KUBEADM_IMAGE} || kubeadm_aio_build
|
||||
|
||||
if [ "x$PVC_BACKEND" == "xceph" ]; then
|
||||
ceph_kube_controller_manager_replace
|
||||
sudo modprobe rbd
|
||||
fi
|
||||
|
||||
kubeadm_aio_launch
|
||||
net_resolv_kube
|
||||
|
|
|
|||
|
|
@ -37,8 +37,15 @@ sudo rm -rfv \
|
|||
/var/lib/etcd \
|
||||
/var/etcd \
|
||||
/var/lib/kubelet/* \
|
||||
/var/lib/nova \
|
||||
/var/lib/openstack-helm \
|
||||
/run/openvswitch || true
|
||||
|
||||
# Load ceph kernel module if required
|
||||
if [ "x$PVC_BACKEND" == "xceph" ]; then
|
||||
sudo modprobe rbd
|
||||
fi
|
||||
|
||||
# Launch Container
|
||||
sudo docker run \
|
||||
-dt \
|
||||
|
|
|
|||
|
|
@ -13,10 +13,12 @@
|
|||
# limitations under the License.
|
||||
set -ex
|
||||
|
||||
export HELM_VERSION=${2:-v2.3.1}
|
||||
export HELM_VERSION=${2:-v2.4.1}
|
||||
export KUBE_VERSION=${3:-v1.6.5}
|
||||
export KUBECONFIG=${HOME}/.kubeadm-aio/admin.conf
|
||||
export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:${KUBE_VERSION}
|
||||
export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:${KUBE_VERSION}-ceph
|
||||
export BASE_KUBE_CONTROLLER_MANAGER_IMAGE=gcr.io/google_containers/kube-controller-manager-amd64:${KUBE_VERSION}
|
||||
export CEPH_KUBE_CONTROLLER_MANAGER_IMAGE=quay.io/attcomdev/kube-controller-manager:${KUBE_VERSION}
|
||||
|
||||
export WORK_DIR=$(pwd)
|
||||
source /etc/os-release
|
||||
|
|
@ -24,6 +26,7 @@ export HOST_OS=${ID}
|
|||
source ${WORK_DIR}/tools/gate/funcs/common.sh
|
||||
source ${WORK_DIR}/tools/gate/funcs/network.sh
|
||||
source ${WORK_DIR}/tools/gate/funcs/helm.sh
|
||||
export PVC_BACKEND=ceph
|
||||
|
||||
# Setup the logging location: by default use the working dir as the root.
|
||||
export LOGS_DIR=${LOGS_DIR:-"${WORK_DIR}/logs"}
|
||||
|
|
@ -34,14 +37,17 @@ function dump_logs () {
|
|||
}
|
||||
trap 'dump_logs "$?"' ERR
|
||||
|
||||
# Install base requirements
|
||||
base_install
|
||||
|
||||
# Moving the ws-linter here to avoid it blocking all the jobs just for ws
|
||||
if [ "x$INTEGRATION_TYPE" == "xlinter" ]; then
|
||||
bash ${WORK_DIR}/tools/gate/whitespace.sh
|
||||
fi
|
||||
|
||||
# Install base requirements
|
||||
base_install
|
||||
if [ "x$PVC_BACKEND" == "xceph" ]; then
|
||||
ceph_support_install
|
||||
fi
|
||||
|
||||
# We setup the network for pre kube here, to enable cluster restarts on
|
||||
# development machines
|
||||
net_resolv_pre_kube
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ cat /etc/nodepool/sub_nodes_private | while read SUB_NODE; do
|
|||
export KUBEADM_TOKEN=${KUBEADM_TOKEN}; \
|
||||
export PRIMARY_NODE_IP=${PRIMARY_NODE_IP}; \
|
||||
export KUBEADM_IMAGE=${KUBEADM_IMAGE}; \
|
||||
export PVC_BACKEND=${PVC_BACKEND}; \
|
||||
bash ${WORK_DIR}/tools/gate/provision_gate_worker_node.sh"
|
||||
EOS
|
||||
done
|
||||
|
|
|
|||
|
|
@ -0,0 +1,20 @@
|
|||
FROM ubuntu:16.04
|
||||
MAINTAINER pete.birley@att.com
|
||||
|
||||
ARG KUBE_VERSION=v1.6.5
|
||||
|
||||
RUN set -x \
|
||||
&& TMP_DIR=$(mktemp --directory) \
|
||||
&& cd ${TMP_DIR} \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y \
|
||||
apt-transport-https \
|
||||
ca-certificates \
|
||||
curl \
|
||||
python \
|
||||
jq \
|
||||
# Install kubectl:
|
||||
&& curl -sSL https://dl.k8s.io/${KUBE_VERSION}/kubernetes-client-linux-amd64.tar.gz | tar -zxv --strip-components=1 \
|
||||
&& mv ${TMP_DIR}/client/bin/kubectl /usr/bin/kubectl \
|
||||
&& chmod +x /usr/bin/kubectl \
|
||||
&& rm -rf ${TMP_DIR}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
Ceph Config Helper Container
|
||||
=====================
|
||||
|
||||
This container builds a small image with kubectl and some other utilites for
|
||||
use in the ceph-config chart.
|
||||
|
||||
Instructions
|
||||
------------
|
||||
|
||||
OS Specific Host setup:
|
||||
~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Ubuntu:
|
||||
^^^^^^^aa
|
||||
|
||||
From a freshly provisioned Ubuntu 16.04 LTS host run:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
sudo apt-get update -y
|
||||
sudo apt-get install -y \
|
||||
docker.io \
|
||||
git
|
||||
|
||||
Build the Ceph-Helper Image environment (optional)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
A known good image is published to dockerhub on a fairly regular basis, but if
|
||||
you wish to build your own image, from the root directory of the OpenStack-Helm
|
||||
repo run:
|
||||
|
||||
.. code:: bash
|
||||
|
||||
export KUBE_VERSION=v1.6.5
|
||||
sudo docker build \
|
||||
--build-arg KUBE_VERSION=${KUBE_VERSION} \
|
||||
-t docker.io/port/ceph-config-helper:${KUBE_VERSION} \
|
||||
tools/images/ceph-config-helper
|
||||
sudo docker push docker.io/port/ceph-config-helper:${KUBE_VERSION}
|
||||
|
|
@ -61,8 +61,9 @@ RUN set -x \
|
|||
git \
|
||||
vim \
|
||||
jq \
|
||||
# Install nfs utils for development PVC provisioner
|
||||
# Install utils for PVC provisioners
|
||||
nfs-common \
|
||||
ceph-common \
|
||||
# Tweak Systemd units and targets for running in a container
|
||||
&& find /lib/systemd/system/sysinit.target.wants/ ! -name 'systemd-tmpfiles-setup.service' -type l -exec rm -fv {} + \
|
||||
&& rm -fv \
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ repo run:
|
|||
|
||||
.. code:: bash
|
||||
|
||||
export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:v1.6.4
|
||||
export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:v1.6.5
|
||||
sudo docker build --pull -t ${KUBEADM_IMAGE} tools/kubeadm-aio
|
||||
|
||||
Deploy the AIO environment
|
||||
|
|
|
|||
|
|
@ -36,7 +36,8 @@ else
|
|||
--restart=always \
|
||||
--volume=/sys/fs/cgroup:/sys/fs/cgroup:ro \
|
||||
--volume=/:/rootfs:ro \
|
||||
--volume=/dev/net:/dev/net:rw \
|
||||
--volume=/dev:/dev:rshared \
|
||||
--volume=/lib/modules:/lib/modules:ro \
|
||||
--volume=/var/run/netns:/var/run/netns:rw \
|
||||
--volume=/sys:/sys:ro \
|
||||
--volume=/etc/machine-id:/etc/machine-id:ro \
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ sudo rm -rfv \
|
|||
/run/openvswitch \
|
||||
/var/lib/nova \
|
||||
${HOME}/.kubeadm-aio/admin.conf \
|
||||
/var/lib/openstack-helm \
|
||||
/var/lib/nfs-provisioner || true
|
||||
|
||||
# Launch Container
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
#!/bin/python
|
||||
|
||||
# Copyright 2017 The Openstack-Helm Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
|
|
@ -14,17 +12,19 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import struct
|
||||
import time
|
||||
import base64
|
||||
# MVP values for glance.
|
||||
# This file contains overrides to launch a MVP deployment of glance for the
|
||||
# OpenStack-Helm Single node gates, and local development use. It should be
|
||||
# kept to the bare minimum required for this purpose.
|
||||
|
||||
storage: "gate"
|
||||
|
||||
conf:
|
||||
cinder:
|
||||
default:
|
||||
cinder:
|
||||
backup_driver: "cinder.backup.drivers.nfs"
|
||||
backends:
|
||||
rbd1:
|
||||
volume_driver: "cinder.volume.drivers.nfs.NfsDriver"
|
||||
|
||||
key = os.urandom(16)
|
||||
header = struct.pack(
|
||||
'<hiih',
|
||||
1, # le16 type: CEPH_CRYPTO_AES
|
||||
int(time.time()), # le32 created: seconds
|
||||
0, # le32 created: nanoseconds,
|
||||
len(key), # le16: len(key)
|
||||
)
|
||||
print(base64.b64encode(header + key).decode('ascii'))
|
||||
|
|
@ -12,6 +12,9 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
ceph:
|
||||
enabled: false
|
||||
|
||||
conf:
|
||||
nova:
|
||||
default:
|
||||
|
|
|
|||
Loading…
Reference in New Issue