openstack
/
openstack-helm
Code Issues Proposed changes

Make Ceph Mon port configurable 

- make mon port overridable through env variable
- update ceph.conf to include fqdn with port number
- add mon_add: to ceph global conf
- updating ClusterRoleBinding name for cephfs and rbd
 provisioner.
- remove unused network.port section as we specify ports
 through endpoints

With these changes, ceph mon can run on different port

Change-Id: I8af2bbe3ee84fd3692bdff92d821b9621a6d53c9
This commit is contained in:
Renis 2018-08-08 21:22:52 -07:00
parent fbfcb51c31
commit f3b7c7fc82
12 changed files with 54 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
ceph-client/templates/configmap-etc-client.yaml
View File

@ -22,10 +22,15 @@ limitations under the License.
{{- if or (.Values.deployment.ceph) (.Values.deployment.client_secrets) }}
{{- if empty .Values.conf.ceph.global.mon_host -}}
{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
{{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
{{- end -}}
{{- if empty .Values.conf.ceph.global.mon_addr -}}
{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}}
{{- end -}}
{{- if empty .Values.conf.ceph.osd.cluster_network -}}
{{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}}
{{- end -}}

10
ceph-mon/templates/bin/mon/_start.sh.tpl
View File

@ -39,9 +39,9 @@ function get_mon_config {
while [[ -z "${MONMAP_ADD// }" && "${timeout}" -gt 0 ]]; do
# Get the ceph mon pods (name and IP) from the Kubernetes API. Formatted as a set of monmap params
if [[ ${K8S_HOST_NETWORK} -eq 0 ]]; then
MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.metadata.name}}`}} {{`{{.status.podIP}}`}} {{`{{end}}`}} {{`{{end}}`}}")
MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.metadata.name}}`}} {{`{{.status.podIP}}`}}:${MON_PORT} {{`{{end}}`}} {{`{{end}}`}}")
else
MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.spec.nodeName}}`}} {{`{{.status.podIP}}`}} {{`{{end}}`}} {{`{{end}}`}}")
MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--add {{`{{.spec.nodeName}}`}} {{`{{.status.podIP}}`}}:${MON_PORT} {{`{{end}}`}} {{`{{end}}`}}")
fi
(( timeout-- ))
sleep 1
@ -53,7 +53,7 @@ function get_mon_config {
# if monmap exists and the mon is already there, don't overwrite monmap
if [ -f "${MONMAP}" ]; then
monmaptool --print "${MONMAP}" |grep -q "${MON_IP// }"":6789"
monmaptool --print "${MONMAP}" |grep -q "${MON_IP// }"":${MON_PORT}"
if [ $? -eq 0 ]; then
echo "${MON_IP} already exists in monmap ${MONMAP}"
return
@ -93,7 +93,7 @@ else
# no mons are up and running yet
timeout 5 ceph --cluster "${CLUSTER}" mon getmap -o ${MONMAP} || true
ceph-mon --setuser ceph --setgroup ceph --cluster "${CLUSTER}" -i ${MON_NAME} --inject-monmap ${MONMAP} --keyring ${MON_KEYRING} --mon-data "${MON_DATA_DIR}"
timeout 7 ceph --cluster "${CLUSTER}" mon add "${MON_NAME}" "${MON_IP}:6789" || true
timeout 7 ceph --cluster "${CLUSTER}" mon add "${MON_NAME}" "${MON_IP}:${MON_PORT}" || true
fi
# start MON
@ -104,4 +104,4 @@ exec /usr/bin/ceph-mon \
-d \
-i ${MON_NAME} \
--mon-data "${MON_DATA_DIR}" \
--public-addr "${MON_IP}:6789"
--public-addr "${MON_IP}:${MON_PORT}"

7
ceph-mon/templates/configmap-etc.yaml
View File

@ -22,10 +22,15 @@ limitations under the License.
{{- if .Values.deployment.ceph }}
{{- if empty .Values.conf.ceph.global.mon_host -}}
{{- $monHost := tuple "ceph_mon" "discovery" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
{{- $monHost := tuple "ceph_mon" "discovery" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
{{- end -}}
{{- if empty .Values.conf.ceph.global.mon_addr -}}
{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}}
{{- end -}}
{{- if empty .Values.conf.ceph.global.fsid -}}
{{- $_ := uuidv4 | set .Values.conf.ceph.global "fsid" -}}
{{- end -}}

4
ceph-mon/templates/daemonset-mon.yaml
View File

@ -106,6 +106,8 @@ spec:
value: {{ .Values.network.public | quote }}
- name: KUBECTL_PARAM
value: {{ tuple $envAll "ceph" "mon" | include "helm-toolkit.snippets.kubernetes_kubectl_params" | indent 10 }}
- name: MON_PORT
value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
- name: MON_IP
valueFrom:
fieldRef:
@ -126,7 +128,7 @@ spec:
command:
- /tmp/mon-stop.sh
ports:
- containerPort: 6789
- containerPort: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
livenessProbe:
exec:
command:

2
ceph-mon/templates/deployment-moncheck.yaml
View File

@ -60,7 +60,7 @@ spec:
command:
- /tmp/moncheck-start.sh
ports:
- containerPort: 6789
- containerPort: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
volumeMounts:
- name: ceph-mon-bin
mountPath: /tmp/moncheck-start.sh

7
ceph-osd/templates/configmap-etc.yaml
View File

@ -20,10 +20,15 @@ limitations under the License.
{{- with $envAll }}
{{- if empty .Values.conf.ceph.global.mon_host -}}
{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
{{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
{{- end -}}
{{- if empty .Values.conf.ceph.global.mon_addr -}}
{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}}
{{- end -}}
{{- if empty .Values.conf.ceph.global.fsid -}}
{{- $_ := uuidv4 | set .Values.conf.ceph.global "fsid" -}}
{{- end -}}

4
ceph-osd/values.yaml
View File

@ -79,10 +79,6 @@ secrets:
network:
public: 192.168.0.0/16
cluster: 192.168.0.0/16
port:
mon: 6789
rgw: 8088
mgr: 7000
conf:
ceph:

7
ceph-provisioners/templates/configmap-etc-client.yaml
View File

@ -22,10 +22,15 @@ limitations under the License.
{{- if or (.Values.deployment.ceph) (.Values.deployment.client_secrets) }}
{{- if empty .Values.conf.ceph.global.mon_host -}}
{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}
{{- $monHost := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}}
{{- end -}}
{{- if empty .Values.conf.ceph.global.mon_addr -}}
{{- $monPort := tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- $_ := printf ":%s" $monPort | set .Values.conf.ceph.global "mon_addr" -}}
{{- end -}}
{{- if empty .Values.conf.ceph.osd.cluster_network -}}
{{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}}
{{- end -}}

2
ceph-provisioners/templates/deployment-cephfs-provisioner.yaml
View File

@ -91,7 +91,7 @@ rules:
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: run-cephfs-provisioner
name: {{ $serviceAccountName }}-run-cephfs-provisioner
subjects:
- kind: ServiceAccount
name: {{ $serviceAccountName }}

2
ceph-provisioners/templates/deployment-rbd-provisioner.yaml
View File

@ -81,7 +81,7 @@ rules:
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: run-rbd-provisioner
name: {{ $serviceAccountName }}-run-rbd-provisioner
subjects:
- kind: ServiceAccount
name: {{ $serviceAccountName }}

3
ceph-provisioners/values.yaml
View File

@ -98,9 +98,6 @@ secrets:
network:
public: 192.168.0.0/16
cluster: 192.168.0.0/16
port:
mon: 6789
mgr: 7000
conf:
ceph:

19
tools/deployment/developer/ceph/040-ceph.sh
View File

@ -40,11 +40,29 @@ endpoints:
namespace: openstack
object_store:
namespace: ceph
port:
api:
default: 8088
public: 80
ceph_mon:
namespace: ceph
port:
mon:
default: 6789
ceph_mgr:
namespace: ceph
port:
mgr:
default: 7000
metrics:
default: 9283
network:
public: 172.17.0.1/16
cluster: 172.17.0.1/16
port:
mon: 6789
rgw: 8088
mgr: 7000
deployment:
storage_secrets: true
ceph: true
@ -60,6 +78,7 @@ conf:
ceph:
global:
fsid: ${CEPH_FS_ID}
mon_addr: :6789
osd_pool_default_size: 1
osd:
osd_crush_chooseleaf_type: 0