From ef2672ba1290564eea7ae347e3f9c202434b58af Mon Sep 17 00:00:00 2001 From: Gorka Eguileor Date: Sat, 26 Jan 2019 16:07:36 +0100 Subject: [PATCH] Remove rbd-nbd and improve rbd-ko We prematurely added the rbd-nbd feature, so we are removing it for the time being. We also improve rbd-ko to be able to run with the ceph-common package only installed on the container, so we no longer need to have it installed on the host. --- Dockerfile | 2 +- Dockerfile-latest | 2 +- Dockerfile-release | 2 +- HISTORY.rst | 11 ++++ cinderlib/nos_brick.py | 130 +++++++++++++++++++++-------------------- 5 files changed, 81 insertions(+), 66 deletions(-) diff --git a/Dockerfile b/Dockerfile index 465b26c..8aeebc6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ LABEL maintainers="Gorka Eguileor " \ RUN yum -y install targetcli iscsi-initiator-utils device-mapper-multipath epel-release lvm2 which && \ yum -y install python2-pip python-devel gcc && \ - yum -y install python-rbd ceph-common rbd-nbd git && \ + yum -y install python-rbd ceph-common git && \ # Need new setuptools version or we'll get "SyntaxError: '<' operator not allowed in environment markers" when installing Cinder pip install 'setuptools>=38.6.0' && \ git clone 'https://github.com/openstack/cinder.git' && \ diff --git a/Dockerfile-latest b/Dockerfile-latest index 38e4293..7cc2fa3 100644 --- a/Dockerfile-latest +++ b/Dockerfile-latest @@ -9,7 +9,7 @@ LABEL maintainers="Gorka Eguileor " \ RUN yum -y install targetcli iscsi-initiator-utils device-mapper-multipath epel-release lvm2 which && \ echo yum -y install python2-pip centos-release-openstack-${RELEASE} > /root/whatever && \ yum -y install python2-pip centos-release-openstack-${RELEASE} && \ - yum -y install openstack-cinder python-rbd ceph-common rbd-nbd && \ + yum -y install openstack-cinder python-rbd ceph-common && \ pip install --no-cache-dir 'krest>=1.3.0' 'purestorage>=1.6.0' && \ yum clean all && \ rm -rf /var/cache/yum diff --git a/Dockerfile-release b/Dockerfile-release index 58255eb..26ec4d0 100644 --- a/Dockerfile-release +++ b/Dockerfile-release @@ -8,7 +8,7 @@ LABEL maintainers="Gorka Eguileor " \ RUN yum -y install targetcli iscsi-initiator-utils device-mapper-multipath epel-release lvm2 which && \ yum -y install python2-pip centos-release-openstack-$RELEASE && \ - yum -y install openstack-cinder python-rbd ceph-common rbd-nbd && \ + yum -y install openstack-cinder python-rbd ceph-common && \ yum clean all && \ rm -rf /var/cache/yum && \ pip install --no-cache-dir --process-dependency-links cinderlib 'krest>=1.3.0' 'purestorage>=1.6.0' diff --git a/HISTORY.rst b/HISTORY.rst index 2f680fe..401fd2a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,17 @@ History ======= +0.3.4 (2019-01-26) +------------------ + +- Features: + + - RBD volumes in container without RBD installed on host + +- Removals: + + - RBD-NBD support was prematurely added, removed in this release + 0.3.3 (2019-01-24) ------------------ diff --git a/cinderlib/nos_brick.py b/cinderlib/nos_brick.py index a960ed1..99683ff 100644 --- a/cinderlib/nos_brick.py +++ b/cinderlib/nos_brick.py @@ -50,11 +50,9 @@ class RBDConnector(connectors.rbd.RBDConnector): We need a third one, local attachment on non controller node. """ - rbd_nbd_installed = True - def connect_volume(self, connection_properties): # NOTE(e0ne): sanity check if ceph-common is installed. - self._setup_rbd_methods() + self._setup_rbd_class() # Extract connection parameters and generate config file try: @@ -71,48 +69,51 @@ class RBDConnector(connectors.rbd.RBDConnector): conf = self._create_ceph_conf(monitor_ips, monitor_ports, str(cluster_name), user, keyring) - dev_path = self._connect_volume(pool, volume, conf, - connection_properties) - return {'path': dev_path, + link_name = self.get_rbd_device_name(pool, volume) + real_path = os.path.realpath(link_name) + + try: + # Map RBD volume if it's not already mapped + if not os.path.islink(link_name) or not os.path.exists(real_path): + cmd = ['rbd', 'map', volume, '--pool', pool, '--conf', conf] + cmd += self._get_rbd_args(connection_properties) + stdout, stderr = self._execute(*cmd, + root_helper=self._root_helper, + run_as_root=True) + real_path = stdout.strip() + # The host may not have RBD installed, and therefore won't + # create the symlinks, ensure they exist + if self.containerized: + self._ensure_link(real_path, link_name) + except Exception: + fileutils.delete_if_exists(conf) + + return {'path': real_path, 'conf': conf, 'type': 'block'} - def _rbd_connect_volume(self, pool, volume, conf, connection_properties): - # Map RBD volume if it's not already mapped - dev_path = self.get_rbd_device_name(pool, volume) - if (not os.path.islink(dev_path) or - not os.path.exists(os.path.realpath(dev_path))): - cmd = ['rbd', 'map', volume, '--pool', pool, '--conf', conf] - cmd += self._get_rbd_args(connection_properties) - self._execute(*cmd, root_helper=self._root_helper, + def _ensure_link(self, source, link_name): + self._ensure_dir(os.path.dirname(link_name)) + if self.im_root: + try: + os.symlink(source, link_name) + except Exception: + pass + else: + self._execute('ln', '-s', '-f', source, link_name, run_as_root=True) - return os.path.realpath(dev_path) - - def _get_nbd_device_name(self, pool, volume, conf, connection_properties): - cmd = ('rbd-nbd', 'list-mapped', '--conf', conf) - cmd += self._get_rbd_args(connection_properties) - stdout, stderr = self._execute(*cmd, root_helper=self._root_helper, - run_as_root=True) - for line in stdout.strip().splitlines(): - pid, dev_pool, image, snap, device = line.split(None) - if dev_pool == pool and image == volume: - return device - return None - - def _nbd_connect_volume(self, pool, volume, conf, connection_properties): - dev_path = self._get_nbd_device_name(pool, volume, conf, - connection_properties) - if not dev_path: - cmd = ['rbd-nbd', 'map', volume, '--conf', conf] - cmd += self._get_rbd_args(connection_properties) - dev_path, stderr = self._execute(*cmd, - root_helper=self._root_helper, - run_as_root=True) - return dev_path.strip() def check_valid_device(self, path, run_as_root=True): """Verify an existing RBD handle is connected and valid.""" + if self.im_root: + try: + with open(path, 'r') as f: + f.read(4096) + except Exception: + return False + return True + try: self._execute('dd', 'if=' + path, 'of=/dev/null', 'bs=4096', 'count=1', root_helper=self._root_helper, @@ -123,47 +124,43 @@ class RBDConnector(connectors.rbd.RBDConnector): def disconnect_volume(self, connection_properties, device_info, force=False, ignore_errors=False): - + self._setup_rbd_class() pool, volume = connection_properties['name'].split('/') conf_file = device_info['conf'] - if self.rbd_nbd_installed: - dev_path = self._get_nbd_device_name(pool, volume, conf_file, - connection_properties) - executable = 'rbd-nbd' - else: - dev_path = self.get_rbd_device_name(pool, volume) - executable = 'rbd' + link_name = self.get_rbd_device_name(pool, volume) + real_dev_path = os.path.realpath(link_name) - real_dev_path = os.path.realpath(dev_path) if os.path.exists(real_dev_path): - cmd = [executable, 'unmap', dev_path, '--conf', conf_file] + cmd = ['rbd', 'unmap', real_dev_path, '--conf', conf_file] cmd += self._get_rbd_args(connection_properties) self._execute(*cmd, root_helper=self._root_helper, run_as_root=True) + + if self.containerized: + unlink_root(link_name) fileutils.delete_if_exists(conf_file) - def _check_installed(self): + def _ensure_dir(self, path): + if self.im_root: + os.makedirs(path) + else: + self._execute('mkdir', '-p', path, run_as_root=True) + + def _setup_class(self): try: self._execute('which', 'rbd') except putils.ProcessExecutionError: msg = 'ceph-common package not installed' raise exception.BrickException(msg) - try: - self._execute('which', 'rbd-nbd') - RBDConnector._connect_volume = RBDConnector._nbd_connect_volume - RBDConnector._get_rbd_args = RBDConnector._get_nbd_args - except putils.ProcessExecutionError: - RBDConnector.rbd_nbd_installed = False + RBDConnector.im_root = os.getuid() == 0 + # Check if we are running containerized + RBDConnector.containerized = os.stat('/proc').st_dev > 4 # Don't check again to speed things on following connections - RBDConnector.setup_rbd_methods = lambda *args: None + RBDConnector._setup_rbd_class = lambda *args: None - def _get_nbd_args(self, connection_properties): - return ('--id', connection_properties['auth_username']) - - _setup_rbd_methods = _check_installed - _connect_volume = _rbd_connect_volume + _setup_rbd_class = _setup_class ROOT_HELPER = 'sudo' @@ -174,10 +171,17 @@ def unlink_root(*links, **kwargs): raise_at_end = kwargs.get('raise_at_end', False) exc = exception.ExceptionChainer() catch_exception = no_errors or raise_at_end - for link in links: - with exc.context(catch_exception, 'Unlink failed for %s', link): - putils.execute('unlink', link, run_as_root=True, + + error_msg = 'Some unlinks failed for %s' + if os.getuid() == 0: + for link in links: + with exc.context(catch_exception, error_msg, links): + os.unlink(link) + else: + with exc.context(catch_exception, error_msg, links): + putils.execute('rm', *links, run_as_root=True, root_helper=ROOT_HELPER) + if not no_errors and raise_at_end and exc: raise exc