From 7b35ac05b3c30e680d8d48c7c473d60052ebf195 Mon Sep 17 00:00:00 2001 From: Jan Provaznik Date: Thu, 13 Jul 2017 09:39:47 +0200 Subject: [PATCH] Add support for cephfs+ganesha backend in manila Change-Id: I1262faef895df0023a225ab10dfe66dfb3d3efba Partially-Implements: bluprint nfs-ganesha --- manifests/profile/base/manila/api.pp | 2 +- manifests/profile/base/manila/share.pp | 52 ++++-- manifests/profile/pacemaker/ceph_nfs.pp | 124 +++++++++++++ .../profile/pacemaker/manila/share_bundle.pp | 170 ++++++++++++------ 4 files changed, 278 insertions(+), 70 deletions(-) create mode 100644 manifests/profile/pacemaker/ceph_nfs.pp diff --git a/manifests/profile/base/manila/api.pp b/manifests/profile/base/manila/api.pp index 374479c09..0816f5530 100644 --- a/manifests/profile/base/manila/api.pp +++ b/manifests/profile/base/manila/api.pp @@ -84,7 +84,7 @@ class tripleo::profile::base::manila::api ( $cifs_protocol = undef } if $backend_cephfs_enabled { - $cephfs_protocol = 'CEPHFS' + $cephfs_protocol = hiera('manila::backend::cephfs::cephfs_protocol_helper_type', 'CEPHFS') } else { $cephfs_protocol = undef } diff --git a/manifests/profile/base/manila/share.pp b/manifests/profile/base/manila/share.pp index 1c279993d..452ff1f50 100644 --- a/manifests/profile/base/manila/share.pp +++ b/manifests/profile/base/manila/share.pp @@ -94,21 +94,48 @@ class tripleo::profile::base::manila::share ( include ::manila::volume::cinder } - # manila cephfsnative: + # manila cephfs: if $backend_cephfs_enabled { - $manila_cephfsnative_backend = hiera('manila::backend::cephfsnative::title') - $cephfs_auth_id = hiera('manila::backend::cephfsnative::cephfs_auth_id') - $keyring_path = "/etc/ceph/ceph.client.${cephfs_auth_id}.keyring" - - manila::backend::cephfsnative { $manila_cephfsnative_backend : - driver_handles_share_servers => hiera('manila::backend::cephfsnative::driver_handles_share_servers', false), - share_backend_name => hiera('manila::backend::cephfsnative::share_backend_name'), - cephfs_conf_path => hiera('manila::backend::cephfsnative::cephfs_conf_path'), - cephfs_auth_id => $cephfs_auth_id, - cephfs_cluster_name => hiera('manila::backend::cephfsnative::cephfs_cluster_name'), - cephfs_enable_snapshots => hiera('manila::backend::cephfsnative::cephfs_enable_snapshots'), + $manila_cephfs_protocol_helper_type = hiera('manila::backend::cephfs::cephfs_protocol_helper_type', false) + if $manila_cephfs_protocol_helper_type { + # new manila ceph driver is renamed from 'cephfsnative' to 'cephfs' + # and supports both direct cephfs access or access through + # nfs-ganesha depending whether 'cephfs_protocol_helper_type' is + # set to 'CEPHFS' or 'NFS' + $manila_cephfs_backend = hiera('manila::backend::cephfs::title') + $cephfs_auth_id = hiera('manila::backend::cephfs::cephfs_auth_id') + manila::backend::cephfs { $manila_cephfs_backend : + driver_handles_share_servers => hiera('manila::backend::cephfs::driver_handles_share_servers', false), + share_backend_name => hiera('manila::backend::cephfs::share_backend_name'), + cephfs_conf_path => hiera('manila::backend::cephfs::cephfs_conf_path'), + cephfs_auth_id => $cephfs_auth_id, + cephfs_cluster_name => hiera('manila::backend::cephfs::cephfs_cluster_name'), + cephfs_enable_snapshots => hiera('manila::backend::cephfs::cephfs_enable_snapshots'), + cephfs_protocol_helper_type => $manila_cephfs_protocol_helper_type, + cephfs_ganesha_server_ip => hiera('ganesha_vip', undef), + } + if $manila_cephfs_protocol_helper_type == 'NFS' { + manila_config { + "${manila_cephfs_backend}/ganesha_rados_store_enable": value => true; + "${manila_cephfs_backend}/ganesha_rados_store_pool_name": value => 'manila_data'; + } + } + } else { + # for backward compatibility with older heat templates which used + # 'cephfsnative' driver + $manila_cephfsnative_backend = hiera('manila::backend::cephfsnative::title') + $cephfs_auth_id = hiera('manila::backend::cephfsnative::cephfs_auth_id') + manila::backend::cephfsnative { $manila_cephfsnative_backend : + driver_handles_share_servers => hiera('manila::backend::cephfsnative::driver_handles_share_servers', false), + share_backend_name => hiera('manila::backend::cephfsnative::share_backend_name'), + cephfs_conf_path => hiera('manila::backend::cephfsnative::cephfs_conf_path'), + cephfs_auth_id => $cephfs_auth_id, + cephfs_cluster_name => hiera('manila::backend::cephfsnative::cephfs_cluster_name'), + cephfs_enable_snapshots => hiera('manila::backend::cephfsnative::cephfs_enable_snapshots'), + } } + $keyring_path = "/etc/ceph/ceph.client.${cephfs_auth_id}.keyring" ceph_config { "client.${cephfs_auth_id}/keyring": value => $keyring_path; "client.${cephfs_auth_id}/client mount uid": value => 0; @@ -207,6 +234,7 @@ class tripleo::profile::base::manila::share ( [ $manila_generic_backend, $manila_cephfsnative_backend, + $manila_cephfs_backend, $manila_netapp_backend, $manila_vmax_backend, $manila_isilon_backend, diff --git a/manifests/profile/pacemaker/ceph_nfs.pp b/manifests/profile/pacemaker/ceph_nfs.pp new file mode 100644 index 000000000..7ee6f0af1 --- /dev/null +++ b/manifests/profile/pacemaker/ceph_nfs.pp @@ -0,0 +1,124 @@ +# Copyright 2016 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# == Class: tripleo::profile::pacemaker::ceph_nfs +# +# Ganesha Pacemaker HA profile for tripleo +# +# === Parameters +# +# [*bootstrap_node*] +# (Optional) The hostname of the node responsible for bootstrapping tasks +# Defaults to hiera('manila_share_short_bootstrap_node_name') +# +# [*step*] +# (Optional) The current step in deployment. See tripleo-heat-templates +# for more details. +# Defaults to hiera('step') +# +# [*pcs_tries*] +# (Optional) The number of times pcs commands should be retried. +# Defaults to hiera('pcs_tries', 20) +# +class tripleo::profile::pacemaker::ceph_nfs ( + $bootstrap_node = hiera('manila_share_short_bootstrap_node_name'), + $step = hiera('step'), + $pcs_tries = hiera('pcs_tries', 20), +) { + if $::hostname == downcase($bootstrap_node) { + $pacemaker_master = true + } else { + $pacemaker_master = false + } + + $ganesha_vip = hiera('ganesha_vip') + # NB: Until the IPaddr2 RA has a fix for https://bugzilla.redhat.com/show_bug.cgi?id=1445628 + # we need to specify the nic when creating the ipv6 vip. + if is_ipv6_address($ganesha_vip) { + $netmask = '128' + $nic = interface_for_ip($ganesha_vip) + $ipv6_addrlabel = '99' + } else { + $netmask = '32' + $nic = '' + $ipv6_addrlabel = '' + } + + + Service <| tag == 'ceph-nfs' |> { + hasrestart => true, + restart => '/bin/true', + start => '/bin/true', + stop => '/bin/true', + } + + if $step >= 2 { + pacemaker::property { 'ceph-nfs-role-node-property': + property => 'ceph-nfs-role', + value => true, + tries => $pcs_tries, + node => $::hostname, + } + if $pacemaker_master { + pacemaker::resource::ip { 'ganesha_vip': + ip_address => $ganesha_vip, + cidr_netmask => $netmask, + nic => $nic, + #ipv6_addrlabel => $ipv6_addrlabel, + #ipv6_addrlabel => '', + tries => $pcs_tries, + location_rule => { + resource_discovery => 'exclusive', + score => 0, + expression => ['ceph-nfs-role eq true'], + }, + } + } + } + + if $step >= 5 and $pacemaker_master { + pacemaker::resource::service { 'ceph-nfs' : + service_name => 'ceph-nfs@pacemaker', + op_params => 'start timeout=200s stop timeout=200s', + tries => $pcs_tries, + location_rule => { + resource_discovery => 'exclusive', + score => 0, + expression => ['ceph-nfs-role eq true'], + }, + } + + pacemaker::constraint::colocation { 'ganesha_vip-with-ganesha': + source => "ip-${ganesha_vip}", + target => 'ceph-nfs', + score => 'INFINITY', + tries => $pcs_tries, + } + + pacemaker::constraint::order { 'gansha_vip-then-ganesha': + first_resource => "ip-${ganesha_vip}", + second_resource => 'ceph-nfs', + first_action => 'start', + second_action => 'start', + constraint_params => 'kind=Optional', + tries => $pcs_tries, + tag => 'pacemaker_constraint', + } + + Pacemaker::Resource::Ip['ganesha_vip'] + -> Pacemaker::Resource::Service['ceph-nfs'] + -> Pacemaker::Constraint::Order['gansha_vip-then-ganesha'] + -> Pacemaker::Constraint::Colocation['ganesha_vip-with-ganesha'] + } +} diff --git a/manifests/profile/pacemaker/manila/share_bundle.pp b/manifests/profile/pacemaker/manila/share_bundle.pp index 0893bf070..d2ebd177f 100644 --- a/manifests/profile/pacemaker/manila/share_bundle.pp +++ b/manifests/profile/pacemaker/manila/share_bundle.pp @@ -18,6 +18,10 @@ # # === Parameters # +# [*backend_cephfs_enabled*] +# (Optional) Whether or not the cephfs backend is enabled +# Defaults to hiera('manila_backend_cephfs_enabled', false) +# # [*manila_share_docker_image*] # (Optional) The docker image to use for creating the pacemaker bundle # Defaults to hiera('tripleo::profile::pacemaker::manila::share_bundle::manila_docker_image', undef) @@ -39,6 +43,7 @@ class tripleo::profile::pacemaker::manila::share_bundle ( $bootstrap_node = hiera('manila_share_short_bootstrap_node_name'), $manila_share_docker_image = hiera('tripleo::profile::pacemaker::manila::share_bundle::manila_share_docker_image', undef), + $backend_cephfs_enabled = hiera('manila_backend_cephfs_enabled', false), $pcs_tries = hiera('pcs_tries', 20), $step = Integer(hiera('step')), ) { @@ -67,6 +72,89 @@ class tripleo::profile::pacemaker::manila::share_bundle ( if $pacemaker_master { $manila_share_nodes_count = count(hiera('manila_share_short_node_names', [])) + $default_storage_maps = { + 'manila-share-cfg-files' => { + 'source-dir' => '/var/lib/kolla/config_files/manila_share.json', + 'target-dir' => '/var/lib/kolla/config_files/config.json', + 'options' => 'ro', + }, + 'manila-share-cfg-data' => { + 'source-dir' => '/var/lib/config-data/puppet-generated/manila/', + 'target-dir' => '/var/lib/kolla/config_files/src', + 'options' => 'ro', + }, + 'manila-share-hosts' => { + 'source-dir' => '/etc/hosts', + 'target-dir' => '/etc/hosts', + 'options' => 'ro', + }, + 'manila-share-localtime' => { + 'source-dir' => '/etc/localtime', + 'target-dir' => '/etc/localtime', + 'options' => 'ro', + }, + 'manila-share-dev' => { + 'source-dir' => '/dev', + 'target-dir' => '/dev', + 'options' => 'rw', + }, + 'manila-share-run' => { + 'source-dir' => '/run', + 'target-dir' => '/run', + 'options' => 'rw', + }, + 'manila-share-sys' => { + 'source-dir' => '/sys', + 'target-dir' => '/sys', + 'options' => 'rw', + }, + 'manila-share-lib-modules' => { + 'source-dir' => '/lib/modules', + 'target-dir' => '/lib/modules', + 'options' => 'ro', + }, + 'manila-share-var-lib-manila' => { + 'source-dir' => '/var/lib/manila', + 'target-dir' => '/var/lib/manila', + 'options' => 'rw', + }, + 'manila-share-var-log' => { + 'source-dir' => '/var/log/containers/manila', + 'target-dir' => '/var/log/manila', + 'options' => 'rw', + }, + 'ceph-cfg-dir' => { + 'source-dir' => '/etc/ceph', + 'target-dir' => '/etc/ceph', + 'options' => 'ro', + }, + } + + # if ceph-nfs backend is used, then DBus is used for dynamic + # creation of NFS exports and DBus socket has to be mounted + # both to manila-share and ganesha containers so they can talk + # to each other + $manila_cephfs_protocol_helper_type = hiera('manila::backend::cephfs::cephfs_protocol_helper_type', '') + $nfs_ganesha = ($backend_cephfs_enabled and $manila_cephfs_protocol_helper_type == 'NFS') + if $nfs_ganesha { + $extra_storage_maps = { + 'dbus-docker' => { + 'source-dir' => '/var/run/dbus/system_bus_socket', + 'target-dir' => '/var/run/dbus/system_bus_socket', + 'options' => 'rw', + }, + 'etc-ganesha' => { + 'source-dir' => '/etc/ganesha', + 'target-dir' => '/etc/ganesha', + 'options' => 'rw', + }, + } + } else { + $extra_storage_maps = {} + } + + $storage_maps = merge($default_storage_maps, $extra_storage_maps) + pacemaker::resource::bundle { $::manila::params::share_service: image => $manila_share_docker_image, replicas => 1, @@ -78,63 +166,31 @@ class tripleo::profile::pacemaker::manila::share_bundle ( container_options => 'network=host', options => '--ipc=host --privileged=true --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS', run_command => '/bin/bash /usr/local/bin/kolla_start', - storage_maps => { - 'manila-share-cfg-files' => { - 'source-dir' => '/var/lib/kolla/config_files/manila_share.json', - 'target-dir' => '/var/lib/kolla/config_files/config.json', - 'options' => 'ro', - }, - 'manila-share-cfg-data' => { - 'source-dir' => '/var/lib/config-data/puppet-generated/manila/', - 'target-dir' => '/var/lib/kolla/config_files/src', - 'options' => 'ro', - }, - 'manila-share-hosts' => { - 'source-dir' => '/etc/hosts', - 'target-dir' => '/etc/hosts', - 'options' => 'ro', - }, - 'manila-share-localtime' => { - 'source-dir' => '/etc/localtime', - 'target-dir' => '/etc/localtime', - 'options' => 'ro', - }, - 'manila-share-dev' => { - 'source-dir' => '/dev', - 'target-dir' => '/dev', - 'options' => 'rw', - }, - 'manila-share-run' => { - 'source-dir' => '/run', - 'target-dir' => '/run', - 'options' => 'rw', - }, - 'manila-share-sys' => { - 'source-dir' => '/sys', - 'target-dir' => '/sys', - 'options' => 'rw', - }, - 'manila-share-lib-modules' => { - 'source-dir' => '/lib/modules', - 'target-dir' => '/lib/modules', - 'options' => 'ro', - }, - 'manila-share-var-lib-manila' => { - 'source-dir' => '/var/lib/manila', - 'target-dir' => '/var/lib/manila', - 'options' => 'rw', - }, - 'manila-share-var-log' => { - 'source-dir' => '/var/log/containers/manila', - 'target-dir' => '/var/log/manila', - 'options' => 'rw', - }, - 'ceph-cfg-dir' => { - 'source-dir' => '/etc/ceph', - 'target-dir' => '/etc/ceph', - 'options' => 'ro', - }, - }, + storage_maps => $storage_maps, + } + + if $nfs_ganesha { + pacemaker::constraint::order { 'ceph-nfs-then-manila-share': + first_resource => 'ceph-nfs', + second_resource => 'openstack-manila-share', + first_action => 'start', + second_action => 'start', + constraint_params => 'kind=Optional', + tries => $pcs_tries, + tag => 'pacemaker_constraint', + } + + pacemaker::constraint::colocation { 'openstack-manila-share-with-ceph-nfs': + source => 'openstack-manila-share', + target => 'ceph-nfs', + score => 'INFINITY', + tries => $pcs_tries, + tag => 'pacemaker_constraint', + } + + Pacemaker::Resource::Bundle['openstack-manila-share'] + -> Pacemaker::Constraint::Colocation['openstack-manila-share-with-ceph-nfs'] + -> Pacemaker::Constraint::Order['ceph-nfs-then-manila-share'] } } }