Ceph Nautilus compatibility
This change updates the Ceph charts to use Ceph Nautilus images built on Ubuntu Bionic instead of Xenial. The mirror that hosts Ceph packages only provides Nautilus packages for Bionic at present, so this is necessary for Nautilus deployment. There are also several configuration and scripting changes included to provide compatibility with Ceph Nautilus. Most of these simply allow existing logic to execute for Nautilus deployments, but some logical changes are required to support Nautilus as well. NOTE: The cephfs test has been disabled because it was failing the gate. This test has passed in multiple dev environments, and since cephfs isn't used by any openstack-helm-infra components we don't want this to block getting this change merged. The gate issue will be investigated and addressed in a subsequent patch set. Change-Id: Id2d9d7b35d4dc66e93a0aacc9ea514e85ae13467
This commit is contained in:
Stephen Taylor
Steve Wilkerson
parent
edd6ffd712
commit
016b56e586
|
|
@ -35,10 +35,17 @@ function wait_for_inactive_pgs () {
|
|||
echo "#### Start: Checking for inactive pgs ####"
|
||||
|
||||
# Loop until all pgs are active
|
||||
while [[ `ceph --cluster ${CLUSTER} pg ls | tail -n +2 | grep -v "active+"` ]]
|
||||
do
|
||||
sleep 3
|
||||
done
|
||||
if [[ $(ceph -v | egrep -q "nautilus"; echo $?) -eq 0 ]]; then
|
||||
while [[ `ceph --cluster ${CLUSTER} pg ls | tail -n +2 | head -n -2 | grep -v "active+"` ]]
|
||||
do
|
||||
sleep 3
|
||||
done
|
||||
else
|
||||
while [[ `ceph --cluster ${CLUSTER} pg ls | tail -n +2 | grep -v "active+"` ]]
|
||||
do
|
||||
sleep 3
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
function create_crushrule () {
|
||||
|
|
@ -51,6 +58,11 @@ function create_crushrule () {
|
|||
fi
|
||||
}
|
||||
|
||||
# Set mons to use the msgr2 protocol on nautilus
|
||||
if [[ $(ceph -v | egrep -q "nautilus"; echo $?) -eq 0 ]]; then
|
||||
ceph --cluster "${CLUSTER}" mon enable-msgr2
|
||||
fi
|
||||
|
||||
{{- range $crush_rule := .Values.conf.pool.crush_rules -}}
|
||||
{{- with $crush_rule }}
|
||||
create_crushrule {{ .name }} {{ .crush_rule }} {{ .failure_domain }} {{ .device_class }}
|
||||
|
|
@ -147,7 +159,12 @@ reweight_osds
|
|||
{{ $crushRuleDefault := .Values.conf.pool.default.crush_rule }}
|
||||
{{ $targetQuota := .Values.conf.pool.target.quota | default 100 }}
|
||||
{{ $targetProtection := .Values.conf.pool.target.protected | default "false" | quote | lower }}
|
||||
cluster_capacity=$(ceph --cluster "${CLUSTER}" df | head -n3 | tail -n1 | awk '{print $1 substr($2, 1, 1)}' | numfmt --from=iec)
|
||||
cluster_capacity=0
|
||||
if [[ $(ceph -v | egrep -q "nautilus"; echo $?) -eq 0 ]]; then
|
||||
cluster_capacity=$(ceph --cluster "${CLUSTER}" df | grep "TOTAL" | awk '{print $2 substr($3, 1, 1)}' | numfmt --from=iec)
|
||||
else
|
||||
cluster_capacity=$(ceph --cluster "${CLUSTER}" df | head -n3 | tail -n1 | awk '{print $1 substr($2, 1, 1)}' | numfmt --from=iec)
|
||||
fi
|
||||
{{- range $pool := .Values.conf.pool.spec -}}
|
||||
{{- with $pool }}
|
||||
{{- if .crush_rule }}
|
||||
|
|
|
|||
|
|
@ -25,11 +25,11 @@ release_group: null
|
|||
images:
|
||||
pull_policy: IfNotPresent
|
||||
tags:
|
||||
ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:ubuntu_xenial-20191119'
|
||||
ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_xenial-20191119'
|
||||
ceph_mds: 'docker.io/openstackhelm/ceph-daemon:ubuntu_xenial-20191119'
|
||||
ceph_mgr: 'docker.io/openstackhelm/ceph-daemon:ubuntu_xenial-20191119'
|
||||
ceph_rbd_pool: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_xenial-20191119'
|
||||
ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:ubuntu_bionic-20191216'
|
||||
ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216'
|
||||
ceph_mds: 'docker.io/openstackhelm/ceph-daemon:ubuntu_bionic-20191216'
|
||||
ceph_mgr: 'docker.io/openstackhelm/ceph-daemon:ubuntu_bionic-20191216'
|
||||
ceph_rbd_pool: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216'
|
||||
dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0'
|
||||
image_repo_sync: 'docker.io/docker:17.07.0'
|
||||
local_registry:
|
||||
|
|
@ -451,7 +451,7 @@ bootstrap:
|
|||
ceph -s
|
||||
function ensure_pool () {
|
||||
ceph osd pool stats $1 || ceph osd pool create $1 $2
|
||||
local test_version=$(ceph tell osd.* version | egrep -c "mimic|luminous" | xargs echo)
|
||||
local test_version=$(ceph tell osd.* version | egrep -c "nautilus|mimic|luminous" | xargs echo)
|
||||
if [[ ${test_version} -gt 0 ]]; then
|
||||
ceph osd pool application enable $1 $3
|
||||
fi
|
||||
|
|
@ -475,11 +475,11 @@ ceph_mgr_enabled_modules:
|
|||
# of key/value. Refer to the doc
|
||||
# above for more info. For example:
|
||||
ceph_mgr_modules_config:
|
||||
balancer:
|
||||
active: 1
|
||||
prometheus:
|
||||
# balancer:
|
||||
# active: 1
|
||||
# prometheus:
|
||||
# server_port: 9283
|
||||
server_addr: 0.0.0.0
|
||||
# server_addr: 0.0.0.0
|
||||
# dashboard:
|
||||
# port: 7000
|
||||
# localpool:
|
||||
|
|
|
|||
|
|
@ -24,10 +24,10 @@ deployment:
|
|||
images:
|
||||
pull_policy: IfNotPresent
|
||||
tags:
|
||||
ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:ubuntu_xenial-20191119'
|
||||
ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_xenial-20191119'
|
||||
ceph_mon: 'docker.io/openstackhelm/ceph-daemon:ubuntu_xenial-20191119'
|
||||
ceph_mon_check: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_xenial-20191119'
|
||||
ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:ubuntu_bionic-20191216'
|
||||
ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216'
|
||||
ceph_mon: 'docker.io/openstackhelm/ceph-daemon:ubuntu_bionic-20191216'
|
||||
ceph_mon_check: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216'
|
||||
dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0'
|
||||
image_repo_sync: 'docker.io/docker:17.07.0'
|
||||
local_registry:
|
||||
|
|
@ -260,7 +260,7 @@ bootstrap:
|
|||
ceph -s
|
||||
function ensure_pool () {
|
||||
ceph osd pool stats $1 || ceph osd pool create $1 $2
|
||||
local test_version=$(ceph tell osd.* version | egrep -c "mimic|luminous" | xargs echo)
|
||||
local test_version=$(ceph tell osd.* version | egrep -c "nautilus|mimic|luminous" | xargs echo)
|
||||
if [[ ${test_version} -gt 0 ]]; then
|
||||
ceph osd pool application enable $1 $3
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -30,8 +30,8 @@ eval CRUSH_FAILURE_DOMAIN_TYPE=$(cat /etc/ceph/storage.json | python -c 'import
|
|||
eval CRUSH_FAILURE_DOMAIN_NAME=$(cat /etc/ceph/storage.json | python -c 'import sys, json; data = json.load(sys.stdin); print(json.dumps(data["failure_domain_name"]))')
|
||||
eval CRUSH_FAILURE_DOMAIN_BY_HOSTNAME=$(cat /etc/ceph/storage.json | python -c 'import sys, json; data = json.load(sys.stdin); print(json.dumps(data["failure_domain_by_hostname"]))')
|
||||
|
||||
if [[ $(ceph -v | egrep -q "mimic|luminous"; echo $?) -ne 0 ]]; then
|
||||
echo "ERROR- need Luminous/Mimic release"
|
||||
if [[ $(ceph -v | egrep -q "nautilus|mimic|luminous"; echo $?) -ne 0 ]]; then
|
||||
echo "ERROR- need Luminous/Mimic/Nautilus release"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
|
|
|||
|
|
@ -30,8 +30,8 @@ eval CRUSH_FAILURE_DOMAIN_TYPE=$(cat /etc/ceph/storage.json | python -c 'import
|
|||
eval CRUSH_FAILURE_DOMAIN_NAME=$(cat /etc/ceph/storage.json | python -c 'import sys, json; data = json.load(sys.stdin); print(json.dumps(data["failure_domain_name"]))')
|
||||
eval CRUSH_FAILURE_DOMAIN_BY_HOSTNAME=$(cat /etc/ceph/storage.json | python -c 'import sys, json; data = json.load(sys.stdin); print(json.dumps(data["failure_domain_by_hostname"]))')
|
||||
|
||||
if [[ $(ceph -v | egrep -q "mimic|luminous"; echo $?) -ne 0 ]]; then
|
||||
echo "ERROR- need Luminous/Mimic release"
|
||||
if [[ $(ceph -v | egrep -q "nautilus|mimic|luminous"; echo $?) -ne 0 ]]; then
|
||||
echo "ERROR- need Luminous/Mimic/Nautilus release"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
|
|
|||
|
|
@ -20,9 +20,9 @@
|
|||
images:
|
||||
pull_policy: IfNotPresent
|
||||
tags:
|
||||
ceph_osd: 'docker.io/openstackhelm/ceph-daemon:ubuntu_xenial-20191119'
|
||||
ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:ubuntu_xenial-20191119'
|
||||
ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_xenial-20191119'
|
||||
ceph_osd: 'docker.io/openstackhelm/ceph-daemon:ubuntu_bionic-20191216'
|
||||
ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:ubuntu_bionic-20191216'
|
||||
ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216'
|
||||
dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0'
|
||||
image_repo_sync: 'docker.io/docker:17.07.0'
|
||||
local_registry:
|
||||
|
|
@ -196,7 +196,7 @@ conf:
|
|||
# match the failure domain used on your CRUSH rules for pools. For example with a crush rule of
|
||||
# rack_replicated_rule you would specify "rack" as the `failure_domain` to use.
|
||||
# `failure_domain`: Set the CRUSH bucket type for your OSD to reside in. See the supported CRUSH configuration
|
||||
# as listed here: Supported CRUSH configuration is listed here: http://docs.ceph.com/docs/mimic/rados/operations/crush-map/
|
||||
# as listed here: Supported CRUSH configuration is listed here: http://docs.ceph.com/docs/nautilus/rados/operations/crush-map/
|
||||
# `failure_domain_by_hostname`: Specify the portion of the hostname to use for your failure domain bucket name.
|
||||
# `failure_domain_name`: Manually name the failure domain bucket name. This configuration option should only be used
|
||||
# when using host based overrides.
|
||||
|
|
|
|||
|
|
@ -28,10 +28,10 @@ release_group: null
|
|||
images:
|
||||
pull_policy: IfNotPresent
|
||||
tags:
|
||||
ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:ubuntu_xenial-20191119'
|
||||
ceph_cephfs_provisioner: 'docker.io/openstackhelm/ceph-cephfs-provisioner:ubuntu_xenial-20191119'
|
||||
ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_xenial-20191119'
|
||||
ceph_rbd_provisioner: 'docker.io/openstackhelm/ceph-rbd-provisioner:ubuntu_xenial-20191119'
|
||||
ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:ubuntu_bionic-20191216'
|
||||
ceph_cephfs_provisioner: 'docker.io/openstackhelm/ceph-cephfs-provisioner:ubuntu_bionic-20191216'
|
||||
ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216'
|
||||
ceph_rbd_provisioner: 'docker.io/openstackhelm/ceph-rbd-provisioner:ubuntu_bionic-20191216'
|
||||
dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0'
|
||||
image_repo_sync: 'docker.io/docker:17.07.0'
|
||||
local_registry:
|
||||
|
|
@ -224,7 +224,7 @@ bootstrap:
|
|||
ceph -s
|
||||
function ensure_pool () {
|
||||
ceph osd pool stats $1 || ceph osd pool create $1 $2
|
||||
local test_version=$(ceph tell osd.* version | egrep -c "mimic|luminous" | xargs echo)
|
||||
local test_version=$(ceph tell osd.* version | egrep -c "nautilus|mimic|luminous" | xargs echo)
|
||||
if [[ ${test_version} -gt 0 ]]; then
|
||||
ceph osd pool application enable $1 $3
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -25,11 +25,11 @@ release_group: null
|
|||
images:
|
||||
pull_policy: IfNotPresent
|
||||
tags:
|
||||
ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_xenial-20191119'
|
||||
ceph_rgw: 'docker.io/openstackhelm/ceph-daemon:ubuntu_xenial-20191119'
|
||||
ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216'
|
||||
ceph_rgw: 'docker.io/openstackhelm/ceph-daemon:ubuntu_bionic-20191216'
|
||||
dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0'
|
||||
image_repo_sync: 'docker.io/docker:17.07.0'
|
||||
rgw_s3_admin: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_xenial-20191119'
|
||||
rgw_s3_admin: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216'
|
||||
ks_endpoints: 'docker.io/openstackhelm/heat:newton-ubuntu_xenial'
|
||||
ks_service: 'docker.io/openstackhelm/heat:newton-ubuntu_xenial'
|
||||
ks_user: 'docker.io/openstackhelm/heat:newton-ubuntu_xenial'
|
||||
|
|
@ -420,7 +420,7 @@ bootstrap:
|
|||
ceph -s
|
||||
function ensure_pool () {
|
||||
ceph osd pool stats $1 || ceph osd pool create $1 $2
|
||||
local test_version=$(ceph tell osd.* version | egrep -c "mimic|luminous")
|
||||
local test_version=$(ceph tell osd.* version | egrep -c "nautilus|mimic|luminous")
|
||||
if [[ ${test_version} -gt 0 ]]; then
|
||||
ceph osd pool application enable $1 $3
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ set -ex
|
|||
ceph -s
|
||||
function ensure_pool () {
|
||||
ceph osd pool stats $1 || ceph osd pool create $1 $2
|
||||
local test_version=$(ceph tell osd.* version | egrep -c "mimic|luminous" | xargs echo)
|
||||
local test_version=$(ceph tell osd.* version | egrep -c "nautilus|mimic|luminous" | xargs echo)
|
||||
if [[ ${test_version} -gt 0 ]]; then
|
||||
ceph osd pool application enable $1 $3
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ release_group: null
|
|||
images:
|
||||
tags:
|
||||
dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
|
||||
gnocchi_storage_init: docker.io/openstackhelm/ceph-config-helper:ubuntu_xenial-20191119
|
||||
gnocchi_storage_init: docker.io/openstackhelm/ceph-config-helper:ubuntu_bionic-20191216
|
||||
db_init_indexer: docker.io/postgres:9.5
|
||||
# using non-kolla images until kolla supports postgres as
|
||||
# an indexer
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ sudo sed -i '/^::1/c\::1 localhost6 localhost6.localdomain6' /etc/hosts
|
|||
# Install required packages for K8s on host
|
||||
wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add -
|
||||
RELEASE_NAME=$(grep 'CODENAME' /etc/lsb-release | awk -F= '{print $2}')
|
||||
sudo add-apt-repository "deb https://download.ceph.com/debian-mimic/
|
||||
sudo add-apt-repository "deb https://download.ceph.com/debian-nautilus/
|
||||
${RELEASE_NAME} main"
|
||||
sudo -E apt-get update
|
||||
sudo -E apt-get install -y \
|
||||
|
|
|
|||
|
|
@ -40,6 +40,9 @@ bootstrap:
|
|||
conf:
|
||||
rgw_ks:
|
||||
enabled: false
|
||||
storageclass:
|
||||
cephfs:
|
||||
provision_storage_class: false
|
||||
EOF
|
||||
|
||||
: ${OSH_INFRA_EXTRA_HELM_ARGS_CEPH_NS_ACTIVATE:="$(./tools/deployment/common/get-values-overrides.sh ceph-provisioners)"}
|
||||
|
|
|
|||
|
|
@ -40,6 +40,9 @@ bootstrap:
|
|||
conf:
|
||||
rgw_ks:
|
||||
enabled: false
|
||||
storageclass:
|
||||
cephfs:
|
||||
provision_storage_class: false
|
||||
EOF
|
||||
|
||||
: ${OSH_EXTRA_HELM_ARGS_CEPH_NS_ACTIVATE:="$(./tools/deployment/common/get-values-overrides.sh ceph-provisioners)"}
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@
|
|||
- name: ubuntu | ensure community ceph repository exists
|
||||
when: ansible_distribution == 'Ubuntu'
|
||||
apt_repository:
|
||||
repo: "deb https://download.ceph.com/debian-mimic/ {{ ansible_lsb.codename }} main"
|
||||
repo: "deb https://download.ceph.com/debian-nautilus/ {{ ansible_lsb.codename }} main"
|
||||
state: present
|
||||
update_cache: yes
|
||||
|
||||
|
|
@ -30,7 +30,7 @@
|
|||
name: ceph
|
||||
description: "Ceph community packages for Redhat/Centos"
|
||||
gpgkey: "https://download.ceph.com/keys/release.asc"
|
||||
baseurl: "https://download.ceph.com/rpm-mimic/el7/$basearch"
|
||||
baseurl: "https://download.ceph.com/rpm-nautilus/el7/$basearch"
|
||||
gpgcheck: yes
|
||||
state: present
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue