From a53eaba74783eeb6fb91ef94ca2498a8c6c4fe0b Mon Sep 17 00:00:00 2001 From: Ramana Raja Date: Sat, 18 Mar 2017 17:19:48 +0530 Subject: [PATCH] Add support to setup CephFS NFS-Ganesha driver ... for Manila. Tested to work in Ubuntu Xenial and CentOS 7 environments. Co-Authored-By: Tom Barron Change-Id: Ib2e806fefdd0548b06ff6aadd4b6a8e62001ea33 --- README.md | 7 +++++ devstack/lib/ceph | 74 ++++++++++++++++++++++++++++++++++++++++++----- devstack/settings | 48 ++++++++++++++++++++---------- 3 files changed, 107 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index d7c54bd..cb00aa5 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,13 @@ This plugin also gets used to configure Ceph as the storage backend for the upst ENABLE_CEPH_MANILA=True ``` + CephFS Native driver that supports native Ceph protocol is used by default. + To use CephFS NFS-Ganesha driver that supports NFS protocol add the setting: + + ``` + MANILA_CEPH_DRIVER=cephfsnfs + ``` + Make sure that the manila plugin is enabled before devstack-plugin-ceph in the ```local.conf``` file. diff --git a/devstack/lib/ceph b/devstack/lib/ceph index ace5e25..f95ab9e 100644 --- a/devstack/lib/ceph +++ b/devstack/lib/ceph @@ -336,6 +336,9 @@ function cleanup_ceph_embedded { function cleanup_ceph_general { _undefine_virsh_secret + if is_ceph_enabled_for_service manila && [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then + cleanup_nfs_ganesha + fi } function cleanup_containerized_ceph { @@ -702,10 +705,32 @@ function configure_ceph_manila { client mount gid = 0 EOF + if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then + configure_nfs_ganesha + # NFS-Ganesha server cannot run alongwith with other kernel NFS server. + sudo systemctl stop nfs-server + sudo systemctl disable nfs-server + sudo systemctl enable nfs-ganesha + sudo systemctl start nfs-ganesha + fi + # RESTART DOCKER CONTAINER } +function configure_nfs_ganesha { + # Configure NFS-Ganesha to work with Manila's CephFS driver + sudo mkdir -p /etc/ganesha/export.d + sudo touch /etc/ganesha/export.d/INDEX.conf + echo "%include /etc/ganesha/export.d/INDEX.conf" | sudo tee /etc/ganesha/ganesha.conf +} + +function cleanup_nfs_ganesha { + sudo systemctl stop nfs-ganesha + sudo systemctl disable nfs-ganesha + sudo uninstall_package nfs-ganesha nfs-ganesha-ceph +} + function configure_ceph_embedded_manila { if [[ $CEPH_REPLICAS -ne 1 ]]; then sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool set ${CEPHFS_DATA_POOL} \ @@ -806,18 +831,29 @@ function install_ceph { install_package software-properties-common if is_ceph_enabled_for_service manila; then - if ! [[ $os_CODENAME =~ (xenial|trusty) ]]; then - die $LINENO "Need Ubuntu trusty or xenial to setup Manila with CephFS" - fi # The 'apt' package manager needs the following package to access # HTTPS enabled repositories such as the Ceph repos hosted by the # shaman/chacra system. install_package apt-transport-https - curl -L https://shaman.ceph.com/api/repos/ceph/jewel/latest/ubuntu/$os_CODENAME/repo | \ - sudo tee /etc/apt/sources.list.d/ceph.list - - CEPH_PACKAGES="${CEPH_PACKAGES} ceph-mds libcephfs1" + if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then + if [ $os_CODENAME != 'xenial' ]; then + die $LINENO "Need Ubuntu xenial to setup Manila with CephFS NFS-Ganesha driver" + fi + # TODO(rraja): Switch to LTS luminous after spring 2017 release. + curl -L https://shaman.ceph.com/api/repos/ceph/kraken/latest/ubuntu/$os_CODENAME/repo | \ + sudo tee /etc/apt/sources.list.d/ceph.list + curl -L https://shaman.ceph.com/api/repos/nfs-ganesha/next/latest/ubuntu/$os_CODENAME/flavors/ceph_kraken/repo | \ + sudo tee /etc/apt/sources.list.d/ext-nfs-ganesha.list + CEPH_PACKAGES="${CEPH_PACKAGES} ceph-mds libcephfs2 nfs-ganesha nfs-ganesha-ceph" + else + if ! [[ $os_CODENAME =~ (xenial|trusty) ]]; then + die $LINENO "Need Ubuntu trusty or xenial to setup Manila with CephFS native driver" + fi + curl -L https://shaman.ceph.com/api/repos/ceph/jewel/latest/ubuntu/$os_CODENAME/repo | \ + sudo tee /etc/apt/sources.list.d/ceph.list + CEPH_PACKAGES="${CEPH_PACKAGES} ceph-mds libcephfs1" + fi elif [ -f "$APT_REPOSITORY_FILE" ]; then # Opt into Openstack CI provided package repo mirror @@ -851,8 +887,29 @@ function install_ceph { REPOS_UPDATED=False install_package ${CEPH_PACKAGES} else + DISTRO_TYPE=${os_VENDOR,,} + RELEASE=$(echo $os_RELEASE | awk -F . '{print $1}') + CEPH_PACKAGES="ceph" + if is_ceph_enabled_for_service manila; then + if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then + if [ $DISTRO_TYPE == 'centos' ] && [ $RELEASE == 7 ]; then + # TODO(rraja): Switch to LTS luminous after spring 2017 release. + curl -L https://shaman.ceph.com/api/repos/ceph/kraken/latest/$DISTRO_TYPE/$RELEASE/repo | \ + sudo tee /etc/yum.repos.d/ext-ceph.repo + curl -L https://shaman.ceph.com/api/repos/nfs-ganesha/next/latest/$DISTRO_TYPE/$RELEASE/flavors/ceph_kraken/repo | \ + sudo tee /etc/yum.repos.d/ext-ganesha.repo + fi + CEPH_PACKAGES="${CEPH_PACKAGES} nfs-ganesha nfs-ganesha-ceph" + else + if [ $DISTRO_TYPE == 'centos' ] && [ $RELEASE == 7 ]; then + curl -L https://shaman.ceph.com/api/repos/ceph/jewel/latest/$DISTRO_TYPE/$RELEASE/repo | \ + sudo tee /etc/yum.repos.d/ext-ceph.repo + fi + fi + fi + if [ "$ENABLE_CEPH_RGW" = "True" ]; then install_package ceph-radosgw CEPH_PACKAGES="${CEPH_PACKAGES} ceph-radosgw" @@ -905,6 +962,9 @@ function stop_ceph { fi if is_ceph_enabled_for_service manila; then sudo systemctl stop ceph-mds@${MDS_ID} + if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then + sudo systemctl stop nfs-ganesha + fi fi # if mon is dead or unhealthy we won't get the list # of osds but should continue anyways. diff --git a/devstack/settings b/devstack/settings index 4f38a22..74c71e3 100644 --- a/devstack/settings +++ b/devstack/settings @@ -23,21 +23,39 @@ TEMPEST_VOLUME_MANAGE_SNAPSHOT=False source $CEPH_PLUGIN_DIR/lib/ceph # Set Manila related global variables used by Manila's DevStack plugin. -if (is_ceph_enabled_for_service manila) && \ -[[ $MANILA_CEPH_DRIVER == 'cephfsnative' ]]; then - MANILA_DHSS=$(trueorfalse False MANILA_DHSS) - MANILA_ENABLED_SHARE_PROTOCOLS=CEPHFS - MANILA_DEFAULT_SHARE_TYPE=cephfstype - # CephFSNative Driver does not yet support creation of shares from - # snapshot. - MANILA_DEFAULT_SHARE_TYPE_EXTRA_SPECS='snapshot_support=False' +if (is_ceph_enabled_for_service manila); then + if [[ $MANILA_CEPH_DRIVER == 'cephfsnative' ]]; then + MANILA_DHSS=$(trueorfalse False MANILA_DHSS) + MANILA_ENABLED_SHARE_PROTOCOLS=CEPHFS + MANILA_DEFAULT_SHARE_TYPE=cephfstype + # CephFSNative Driver does not yet support creation of shares from + # snapshot. + MANILA_DEFAULT_SHARE_TYPE_EXTRA_SPECS='snapshot_support=False' - MANILA_ENABLED_BACKENDS=cephfsnative1 - MANILA_CONFIGURE_GROUPS=cephfsnative1 + MANILA_ENABLED_BACKENDS=cephfsnative1 + MANILA_CONFIGURE_GROUPS=cephfsnative1 - MANILA_OPTGROUP_cephfsnative1_share_driver=manila.share.drivers.cephfs.cephfs_native.CephFSNativeDriver - MANILA_OPTGROUP_cephfsnative1_driver_handles_share_servers=$MANILA_DHSS - MANILA_OPTGROUP_cephfsnative1_share_backend_name=CEPHFSNATIVE1 - MANILA_OPTGROUP_cephfsnative1_cephfs_conf_path=${CEPH_CONF_FILE} - MANILA_OPTGROUP_cephfsnative1_cephfs_auth_id=${MANILA_CEPH_USER} + MANILA_OPTGROUP_cephfsnative1_share_driver=manila.share.drivers.cephfs.cephfs_native.CephFSNativeDriver + MANILA_OPTGROUP_cephfsnative1_driver_handles_share_servers=$MANILA_DHSS + MANILA_OPTGROUP_cephfsnative1_share_backend_name=CEPHFSNATIVE1 + MANILA_OPTGROUP_cephfsnative1_cephfs_conf_path=${CEPH_CONF_FILE} + MANILA_OPTGROUP_cephfsnative1_cephfs_auth_id=${MANILA_CEPH_USER} + elif [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then + MANILA_DHSS=$(trueorfalse False MANILA_DHSS) + MANILA_ENABLED_SHARE_PROTOCOLS=NFS + MANILA_DEFAULT_SHARE_TYPE=cephfsnfstype + # CephFS Driver does not yet support creation of shares from + # snapshot. + MANILA_DEFAULT_SHARE_TYPE_EXTRA_SPECS='snapshot_support=False' + + MANILA_ENABLED_BACKENDS=cephfsnfs1 + MANILA_CONFIGURE_GROUPS=cephfsnfs1 + + MANILA_OPTGROUP_cephfsnfs1_share_driver=manila.share.drivers.cephfs.driver.CephFSDriver + MANILA_OPTGROUP_cephfsnfs1_driver_handles_share_servers=$MANILA_DHSS + MANILA_OPTGROUP_cephfsnfs1_share_backend_name=CEPHFSNFS1 + MANILA_OPTGROUP_cephfsnfs1_cephfs_conf_path=${CEPH_CONF_FILE} + MANILA_OPTGROUP_cephfsnfs1_cephfs_auth_id=${MANILA_CEPH_USER} + MANILA_OPTGROUP_cephfsnfs1_cephfs_protocol_helper_type=NFS + fi fi