Support NBD RBD attachments

RBD connections were done using the kernel RBD module, which is always
behind, feature-wise, to the Ceph clusters.

Being behind means that you have to disable some features on the client
by editing the ceph.conf file, which is quite inconvenient.

This patch uses of the NBD driver when present in the system, and since
it uses librbd we don't have those feature issues.

If NBD is not present we use the kernel RBD module instead.

Dockerfile

@@ -8,7 +8,7 @@ LABEL maintainers="Gorka Eguileor <geguileo@redhat.com>" \
 
 RUN yum -y install targetcli iscsi-initiator-utils device-mapper-multipath epel-release lvm2 which && \
     yum -y install python2-pip python-devel gcc && \
-    yum -y install python-rbd ceph-common git && \
+    yum -y install python-rbd ceph-common rbd-nbd git && \
     # Need new setuptools version or we'll get "SyntaxError: '<' operator not allowed in environment markers" when installing Cinder
     pip install 'setuptools>=38.6.0' && \
     git clone 'https://github.com/openstack/cinder.git' && \

Dockerfile-latest

@@ -9,7 +9,7 @@ LABEL maintainers="Gorka Eguileor <geguileo@redhat.com>" \
 RUN yum -y install targetcli iscsi-initiator-utils device-mapper-multipath epel-release lvm2 which && \
     echo yum -y install python2-pip centos-release-openstack-${RELEASE} > /root/whatever && \
     yum -y install python2-pip centos-release-openstack-${RELEASE} && \
-    yum -y install openstack-cinder python-rbd ceph-common && \
+    yum -y install openstack-cinder python-rbd ceph-common rbd-nbd && \
     pip install --no-cache-dir 'krest>=1.3.0' 'purestorage>=1.6.0' && \
     yum clean all && \
     rm -rf /var/cache/yum

Dockerfile-release

@@ -8,7 +8,7 @@ LABEL maintainers="Gorka Eguileor <geguileo@redhat.com>" \
 
 RUN yum -y install targetcli iscsi-initiator-utils device-mapper-multipath epel-release lvm2 which && \
     yum -y install python2-pip centos-release-openstack-$RELEASE && \
-    yum -y install openstack-cinder python-rbd ceph-common && \
+    yum -y install openstack-cinder python-rbd ceph-common rbd-nbd && \
     yum clean all && \
     rm -rf /var/cache/yum && \
     pip install --no-cache-dir --process-dependency-links cinderlib 'krest>=1.3.0' 'purestorage>=1.6.0'

HISTORY.rst

@@ -8,6 +8,7 @@ History
 - Features:
 
   - List drivers available in current Cinder installation.
+  - Support RBD-NBD as well as RBD-KO
 
 
 0.3.2 (2019-01-22)

cinderlib/nos_brick.py

@@ -50,13 +50,11 @@ class RBDConnector(connectors.rbd.RBDConnector):
 
     We need a third one, local attachment on non controller node.
     """
+    rbd_nbd_installed = True
+
     def connect_volume(self, connection_properties):
         # NOTE(e0ne): sanity check if ceph-common is installed.
-        try:
-            self._execute('which', 'rbd')
-        except putils.ProcessExecutionError:
-            msg = 'ceph-common package not installed'
-            raise exception.BrickException(msg)
+        self._setup_rbd_methods()
 
         # Extract connection parameters and generate config file
         try:
@@ -73,19 +71,45 @@ class RBDConnector(connectors.rbd.RBDConnector):
         conf = self._create_ceph_conf(monitor_ips, monitor_ports,
                                       str(cluster_name), user,
                                       keyring)
+        dev_path = self._connect_volume(pool, volume, conf,
+                                        connection_properties)
 
+        return {'path': dev_path,
+                'conf': conf,
+                'type': 'block'}
+
+    def _rbd_connect_volume(self, pool, volume, conf, connection_properties):
         # Map RBD volume if it's not already mapped
-        rbd_dev_path = self.get_rbd_device_name(pool, volume)
-        if (not os.path.islink(rbd_dev_path) or
-                not os.path.exists(os.path.realpath(rbd_dev_path))):
+        dev_path = self.get_rbd_device_name(pool, volume)
+        if (not os.path.islink(dev_path) or
+                not os.path.exists(os.path.realpath(dev_path))):
             cmd = ['rbd', 'map', volume, '--pool', pool, '--conf', conf]
             cmd += self._get_rbd_args(connection_properties)
             self._execute(*cmd, root_helper=self._root_helper,
                           run_as_root=True)
+        return os.path.realpath(dev_path)
 
-        return {'path': os.path.realpath(rbd_dev_path),
-                'conf': conf,
-                'type': 'block'}
+    def _get_nbd_device_name(self, pool, volume, conf, connection_properties):
+        cmd = ('rbd-nbd', 'list-mapped', '--conf', conf)
+        cmd += self._get_rbd_args(connection_properties)
+        stdout, stderr = self._execute(*cmd, root_helper=self._root_helper,
+                                       run_as_root=True)
+        for line in stdout.strip().splitlines():
+            pid, dev_pool, image, snap, device = line.split(None)
+            if dev_pool == pool and image == volume:
+                return device
+        return None
+
+    def _nbd_connect_volume(self, pool, volume, conf, connection_properties):
+        dev_path = self._get_nbd_device_name(pool, volume, conf,
+                                             connection_properties)
+        if not dev_path:
+            cmd = ['rbd-nbd', 'map', volume, '--conf', conf]
+            cmd += self._get_rbd_args(connection_properties)
+            dev_path, stderr = self._execute(*cmd,
+                                             root_helper=self._root_helper,
+                                             run_as_root=True)
+        return dev_path.strip()
 
     def check_valid_device(self, path, run_as_root=True):
         """Verify an existing RBD handle is connected and valid."""
@@ -102,13 +126,45 @@ class RBDConnector(connectors.rbd.RBDConnector):
 
         pool, volume = connection_properties['name'].split('/')
         conf_file = device_info['conf']
-        dev_name = self.get_rbd_device_name(pool, volume)
-        cmd = ['rbd', 'unmap', dev_name, '--conf', conf_file]
-        cmd += self._get_rbd_args(connection_properties)
-        self._execute(*cmd, root_helper=self._root_helper,
-                      run_as_root=True)
+        if self.rbd_nbd_installed:
+            dev_path = self._get_nbd_device_name(pool, volume, conf_file,
+                                                 connection_properties)
+            executable = 'rbd-nbd'
+        else:
+            dev_path = self.get_rbd_device_name(pool, volume)
+            executable = 'rbd'
+
+        real_dev_path = os.path.realpath(dev_path)
+        if os.path.exists(real_dev_path):
+            cmd = [executable, 'unmap', dev_path, '--conf', conf_file]
+            cmd += self._get_rbd_args(connection_properties)
+            self._execute(*cmd, root_helper=self._root_helper,
+                          run_as_root=True)
         fileutils.delete_if_exists(conf_file)
 
+    def _check_installed(self):
+        try:
+            self._execute('which', 'rbd')
+        except putils.ProcessExecutionError:
+            msg = 'ceph-common package not installed'
+            raise exception.BrickException(msg)
+
+        try:
+            self._execute('which', 'rbd-nbd')
+            RBDConnector._connect_volume = RBDConnector._nbd_connect_volume
+            RBDConnector._get_rbd_args = RBDConnector._get_nbd_args
+        except putils.ProcessExecutionError:
+            RBDConnector.rbd_nbd_installed = False
+
+        # Don't check again to speed things on following connections
+        RBDConnector.setup_rbd_methods = lambda *args: None
+
+    def _get_nbd_args(self, connection_properties):
+        return ('--id', connection_properties['auth_username'])
+
+    _setup_rbd_methods = _check_installed
+    _connect_volume = _rbd_connect_volume
+
 
 ROOT_HELPER = 'sudo'