From a09a7e12fe685d747ed390a59cd42d0acd1399e4 Mon Sep 17 00:00:00 2001 From: Gorka Eguileor Date: Thu, 24 Jan 2019 16:15:15 +0100 Subject: [PATCH] Support NBD RBD attachments RBD connections were done using the kernel RBD module, which is always behind, feature-wise, to the Ceph clusters. Being behind means that you have to disable some features on the client by editing the ceph.conf file, which is quite inconvenient. This patch uses of the NBD driver when present in the system, and since it uses librbd we don't have those feature issues. If NBD is not present we use the kernel RBD module instead. --- Dockerfile | 2 +- Dockerfile-latest | 2 +- Dockerfile-release | 2 +- HISTORY.rst | 1 + cinderlib/nos_brick.py | 88 ++++++++++++++++++++++++++++++++++-------- 5 files changed, 76 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8aeebc6..465b26c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ LABEL maintainers="Gorka Eguileor " \ RUN yum -y install targetcli iscsi-initiator-utils device-mapper-multipath epel-release lvm2 which && \ yum -y install python2-pip python-devel gcc && \ - yum -y install python-rbd ceph-common git && \ + yum -y install python-rbd ceph-common rbd-nbd git && \ # Need new setuptools version or we'll get "SyntaxError: '<' operator not allowed in environment markers" when installing Cinder pip install 'setuptools>=38.6.0' && \ git clone 'https://github.com/openstack/cinder.git' && \ diff --git a/Dockerfile-latest b/Dockerfile-latest index 7cc2fa3..38e4293 100644 --- a/Dockerfile-latest +++ b/Dockerfile-latest @@ -9,7 +9,7 @@ LABEL maintainers="Gorka Eguileor " \ RUN yum -y install targetcli iscsi-initiator-utils device-mapper-multipath epel-release lvm2 which && \ echo yum -y install python2-pip centos-release-openstack-${RELEASE} > /root/whatever && \ yum -y install python2-pip centos-release-openstack-${RELEASE} && \ - yum -y install openstack-cinder python-rbd ceph-common && \ + yum -y install openstack-cinder python-rbd ceph-common rbd-nbd && \ pip install --no-cache-dir 'krest>=1.3.0' 'purestorage>=1.6.0' && \ yum clean all && \ rm -rf /var/cache/yum diff --git a/Dockerfile-release b/Dockerfile-release index 26ec4d0..58255eb 100644 --- a/Dockerfile-release +++ b/Dockerfile-release @@ -8,7 +8,7 @@ LABEL maintainers="Gorka Eguileor " \ RUN yum -y install targetcli iscsi-initiator-utils device-mapper-multipath epel-release lvm2 which && \ yum -y install python2-pip centos-release-openstack-$RELEASE && \ - yum -y install openstack-cinder python-rbd ceph-common && \ + yum -y install openstack-cinder python-rbd ceph-common rbd-nbd && \ yum clean all && \ rm -rf /var/cache/yum && \ pip install --no-cache-dir --process-dependency-links cinderlib 'krest>=1.3.0' 'purestorage>=1.6.0' diff --git a/HISTORY.rst b/HISTORY.rst index a14db7a..7bb149a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,6 +8,7 @@ History - Features: - List drivers available in current Cinder installation. + - Support RBD-NBD as well as RBD-KO 0.3.2 (2019-01-22) diff --git a/cinderlib/nos_brick.py b/cinderlib/nos_brick.py index 403fda8..a960ed1 100644 --- a/cinderlib/nos_brick.py +++ b/cinderlib/nos_brick.py @@ -50,13 +50,11 @@ class RBDConnector(connectors.rbd.RBDConnector): We need a third one, local attachment on non controller node. """ + rbd_nbd_installed = True + def connect_volume(self, connection_properties): # NOTE(e0ne): sanity check if ceph-common is installed. - try: - self._execute('which', 'rbd') - except putils.ProcessExecutionError: - msg = 'ceph-common package not installed' - raise exception.BrickException(msg) + self._setup_rbd_methods() # Extract connection parameters and generate config file try: @@ -73,19 +71,45 @@ class RBDConnector(connectors.rbd.RBDConnector): conf = self._create_ceph_conf(monitor_ips, monitor_ports, str(cluster_name), user, keyring) + dev_path = self._connect_volume(pool, volume, conf, + connection_properties) + return {'path': dev_path, + 'conf': conf, + 'type': 'block'} + + def _rbd_connect_volume(self, pool, volume, conf, connection_properties): # Map RBD volume if it's not already mapped - rbd_dev_path = self.get_rbd_device_name(pool, volume) - if (not os.path.islink(rbd_dev_path) or - not os.path.exists(os.path.realpath(rbd_dev_path))): + dev_path = self.get_rbd_device_name(pool, volume) + if (not os.path.islink(dev_path) or + not os.path.exists(os.path.realpath(dev_path))): cmd = ['rbd', 'map', volume, '--pool', pool, '--conf', conf] cmd += self._get_rbd_args(connection_properties) self._execute(*cmd, root_helper=self._root_helper, run_as_root=True) + return os.path.realpath(dev_path) - return {'path': os.path.realpath(rbd_dev_path), - 'conf': conf, - 'type': 'block'} + def _get_nbd_device_name(self, pool, volume, conf, connection_properties): + cmd = ('rbd-nbd', 'list-mapped', '--conf', conf) + cmd += self._get_rbd_args(connection_properties) + stdout, stderr = self._execute(*cmd, root_helper=self._root_helper, + run_as_root=True) + for line in stdout.strip().splitlines(): + pid, dev_pool, image, snap, device = line.split(None) + if dev_pool == pool and image == volume: + return device + return None + + def _nbd_connect_volume(self, pool, volume, conf, connection_properties): + dev_path = self._get_nbd_device_name(pool, volume, conf, + connection_properties) + if not dev_path: + cmd = ['rbd-nbd', 'map', volume, '--conf', conf] + cmd += self._get_rbd_args(connection_properties) + dev_path, stderr = self._execute(*cmd, + root_helper=self._root_helper, + run_as_root=True) + return dev_path.strip() def check_valid_device(self, path, run_as_root=True): """Verify an existing RBD handle is connected and valid.""" @@ -102,13 +126,45 @@ class RBDConnector(connectors.rbd.RBDConnector): pool, volume = connection_properties['name'].split('/') conf_file = device_info['conf'] - dev_name = self.get_rbd_device_name(pool, volume) - cmd = ['rbd', 'unmap', dev_name, '--conf', conf_file] - cmd += self._get_rbd_args(connection_properties) - self._execute(*cmd, root_helper=self._root_helper, - run_as_root=True) + if self.rbd_nbd_installed: + dev_path = self._get_nbd_device_name(pool, volume, conf_file, + connection_properties) + executable = 'rbd-nbd' + else: + dev_path = self.get_rbd_device_name(pool, volume) + executable = 'rbd' + + real_dev_path = os.path.realpath(dev_path) + if os.path.exists(real_dev_path): + cmd = [executable, 'unmap', dev_path, '--conf', conf_file] + cmd += self._get_rbd_args(connection_properties) + self._execute(*cmd, root_helper=self._root_helper, + run_as_root=True) fileutils.delete_if_exists(conf_file) + def _check_installed(self): + try: + self._execute('which', 'rbd') + except putils.ProcessExecutionError: + msg = 'ceph-common package not installed' + raise exception.BrickException(msg) + + try: + self._execute('which', 'rbd-nbd') + RBDConnector._connect_volume = RBDConnector._nbd_connect_volume + RBDConnector._get_rbd_args = RBDConnector._get_nbd_args + except putils.ProcessExecutionError: + RBDConnector.rbd_nbd_installed = False + + # Don't check again to speed things on following connections + RBDConnector.setup_rbd_methods = lambda *args: None + + def _get_nbd_args(self, connection_properties): + return ('--id', connection_properties['auth_username']) + + _setup_rbd_methods = _check_installed + _connect_volume = _rbd_connect_volume + ROOT_HELPER = 'sudo'