Sync charmhelpers and charms.ceph

Change-Id: I8bf71d70d33e23d1b67be0af2b97d766b417a624

hooks/charmhelpers/contrib/openstack/context.py

@@ -545,7 +545,7 @@ class IdentityServiceContext(OSContextGenerator):
                         'internal_auth_url': internal_auth_url,
                     })
 
-                # we keep all veriables in ctxt for compatibility and
+                # we keep all variables in ctxt for compatibility and
                 # add nested dictionary for keystone_authtoken generic
                 # templating
                 if keystonemiddleware_os_release:
@@ -557,6 +557,7 @@ class IdentityServiceContext(OSContextGenerator):
                     # NOTE(jamespage) this is required for >= icehouse
                     # so a missing value just indicates keystone needs
                     # upgrading
+                    ctxt['admin_user_id'] = _resolve('service_user_id')
                     ctxt['admin_tenant_id'] = _resolve('service_tenant_id')
                     ctxt['admin_domain_id'] = _resolve('service_domain_id')
                     return ctxt

hooks/charmhelpers/contrib/openstack/utils.py

@@ -161,6 +161,7 @@ OPENSTACK_CODENAMES = OrderedDict([
     ('2022.2', 'zed'),
     ('2023.1', 'antelope'),
     ('2023.2', 'bobcat'),
+    ('2024.1', 'caracal'),
 ])
 
 # The ugly duckling - must list releases oldest to newest

hooks/charmhelpers/contrib/storage/linux/lvm.py

@@ -17,8 +17,6 @@ from subprocess import (
     CalledProcessError,
     check_call,
     check_output,
-    Popen,
-    PIPE,
 )
 
 
@@ -58,9 +56,7 @@ def remove_lvm_physical_volume(block_device):
 
     :param block_device: str: Full path of block device to scrub.
     '''
-    p = Popen(['pvremove', '-ff', block_device],
+    check_call(['pvremove', '-ff', '--yes', block_device])
-              stdin=PIPE)
-    p.communicate(input='y\n')
 
 
 def list_lvm_volume_group(block_device):

hooks/charmhelpers/core/host.py

@@ -256,8 +256,11 @@ def service_resume(service_name, init_dir="/etc/init",
     upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
     sysv_file = os.path.join(initd_dir, service_name)
     if init_is_systemd(service_name=service_name):
-        service('unmask', service_name)
+        if service('is-enabled', service_name):
-        service('enable', service_name)
+            log('service {} already enabled'.format(service_name), level=DEBUG)
+        else:
+            service('unmask', service_name)
+            service('enable', service_name)
     elif os.path.exists(upstart_file):
         override_path = os.path.join(
             init_dir, '{}.override'.format(service_name))

hooks/charmhelpers/fetch/ubuntu.py

@@ -246,6 +246,14 @@ CLOUD_ARCHIVE_POCKETS = {
     'bobcat/proposed': 'jammy-proposed/bobcat',
     'jammy-bobcat/proposed': 'jammy-proposed/bobcat',
     'jammy-proposed/bobcat': 'jammy-proposed/bobcat',
+    # caracal
+    'caracal': 'jammy-updates/caracal',
+    'jammy-caracal': 'jammy-updates/caracal',
+    'jammy-caracal/updates': 'jammy-updates/caracal',
+    'jammy-updates/caracal': 'jammy-updates/caracal',
+    'caracal/proposed': 'jammy-proposed/caracal',
+    'jammy-caracal/proposed': 'jammy-proposed/caracal',
+    'jammy-proposed/caracal': 'jammy-proposed/caracal',
 
     # OVN
     'focal-ovn-22.03': 'focal-updates/ovn-22.03',
@@ -279,6 +287,7 @@ OPENSTACK_RELEASES = (
     'zed',
     'antelope',
     'bobcat',
+    'caracal',
 )
 
 
@@ -308,6 +317,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
     ('kinetic', 'zed'),
     ('lunar', 'antelope'),
     ('mantic', 'bobcat'),
+    ('noble', 'caracal'),
 ])
 
 

hooks/charmhelpers/osplatform.py

@@ -9,19 +9,13 @@ def get_platform():
     will be returned (which is the name of the module).
     This string is used to decide which platform module should be imported.
     """
-    # linux_distribution is deprecated and will be removed in Python 3.7
+    current_platform = _get_current_platform()
-    # Warnings *not* disabled, as we certainly need to fix this.
-    if hasattr(platform, 'linux_distribution'):
-        tuple_platform = platform.linux_distribution()
-        current_platform = tuple_platform[0]
-    else:
-        current_platform = _get_platform_from_fs()
 
     if "Ubuntu" in current_platform:
         return "ubuntu"
     elif "CentOS" in current_platform:
         return "centos"
-    elif "debian" in current_platform:
+    elif "debian" in current_platform or "Debian" in current_platform:
         # Stock Python does not detect Ubuntu and instead returns debian.
         # Or at least it does in some build environments like Travis CI
         return "ubuntu"
@@ -36,6 +30,24 @@ def get_platform():
                            .format(current_platform))
 
 
+def _get_current_platform():
+    """Return the current platform information for the OS.
+
+    Attempts to lookup linux distribution information from the platform
+    module for releases of python < 3.7. For newer versions of python,
+    the platform is determined from the /etc/os-release file.
+    """
+    # linux_distribution is deprecated and will be removed in Python 3.7
+    # Warnings *not* disabled, as we certainly need to fix this.
+    if hasattr(platform, 'linux_distribution'):
+        tuple_platform = platform.linux_distribution()
+        current_platform = tuple_platform[0]
+    else:
+        current_platform = _get_platform_from_fs()
+
+    return current_platform
+
+
 def _get_platform_from_fs():
     """Get Platform from /etc/os-release."""
     with open(os.path.join(os.sep, 'etc', 'os-release')) as fin:

lib/charms_ceph/broker.py

@@ -106,6 +106,8 @@ def decode_req_encode_rsp(f):
     """Decorator to decode incoming requests and encode responses."""
 
     def decode_inner(req):
+        if isinstance(req, bytes):
+            req = req.decode('utf-8')
         return json.dumps(f(json.loads(req)))
 
     return decode_inner

lib/charms_ceph/utils.py

@@ -1324,6 +1324,16 @@ def systemd():
     return CompareHostReleases(lsb_release()['DISTRIB_CODENAME']) >= 'vivid'
 
 
+def use_bluestore():
+    """Determine whether bluestore should be used for OSD's
+
+    :returns: whether bluestore disk format should be used
+    :rtype: bool"""
+    if cmp_pkgrevno('ceph', '12.2.0') < 0:
+        return False
+    return config('bluestore')
+
+
 def bootstrap_monitor_cluster(secret):
     """Bootstrap local Ceph mon into the Ceph cluster
 
@@ -1541,21 +1551,21 @@ def get_devices(name):
 
 
 def osdize(dev, osd_format, osd_journal, ignore_errors=False, encrypt=False,
-           key_manager=CEPH_KEY_MANAGER, osd_id=None):
+           bluestore=False, key_manager=CEPH_KEY_MANAGER, osd_id=None):
     if dev.startswith('/dev'):
         osdize_dev(dev, osd_format, osd_journal,
                    ignore_errors, encrypt,
-                   key_manager, osd_id)
+                   bluestore, key_manager, osd_id)
     else:
         if cmp_pkgrevno('ceph', '14.0.0') >= 0:
             log("Directory backed OSDs can not be created on Nautilus",
                 level=WARNING)
             return
-        osdize_dir(dev, encrypt)
+        osdize_dir(dev, encrypt, bluestore)
 
 
 def osdize_dev(dev, osd_format, osd_journal, ignore_errors=False,
-               encrypt=False, key_manager=CEPH_KEY_MANAGER,
+               encrypt=False, bluestore=False, key_manager=CEPH_KEY_MANAGER,
                osd_id=None):
     """
     Prepare a block device for use as a Ceph OSD
@@ -1569,6 +1579,7 @@ def osdize_dev(dev, osd_format, osd_journal, ignore_errors=False,
     :param: ignore_errors: Don't fail in the event of any errors during
                            processing
     :param: encrypt: Encrypt block devices using 'key_manager'
+    :param: bluestore: Use bluestore native Ceph block device format
     :param: key_manager: Key management approach for encryption keys
     :raises subprocess.CalledProcessError: in the event that any supporting
                                            subprocess operation failed
@@ -1619,13 +1630,15 @@ def osdize_dev(dev, osd_format, osd_journal, ignore_errors=False,
             cmd = _ceph_volume(dev,
                                osd_journal,
                                encrypt,
+                               bluestore,
                                key_manager,
                                osd_id)
         else:
             cmd = _ceph_disk(dev,
                              osd_format,
                              osd_journal,
-                             encrypt)
+                             encrypt,
+                             bluestore)
 
         try:
             status_set('maintenance', 'Initializing device {}'.format(dev))
@@ -1656,7 +1669,7 @@ def osdize_dev(dev, osd_format, osd_journal, ignore_errors=False,
         db.flush()
 
 
-def _ceph_disk(dev, osd_format, osd_journal, encrypt=False):
+def _ceph_disk(dev, osd_format, osd_journal, encrypt=False, bluestore=False):
     """
     Prepare a device for usage as a Ceph OSD using ceph-disk
 
@@ -1664,6 +1677,7 @@ def _ceph_disk(dev, osd_format, osd_journal, encrypt=False):
                  The function looks up realpath of the device
     :param: osd_journal: List of block devices to use for OSD journals
     :param: encrypt: Use block device encryption (unsupported)
+    :param: bluestore: Use bluestore storage for OSD
     :returns: list. 'ceph-disk' command and required parameters for
                     execution by check_call
     """
@@ -1672,17 +1686,25 @@ def _ceph_disk(dev, osd_format, osd_journal, encrypt=False):
     if encrypt:
         cmd.append('--dmcrypt')
 
-    cmd.append('--bluestore')
+    if osd_format and not bluestore:
-    wal = get_devices('bluestore-wal')
+        cmd.append('--fs-type')
-    if wal:
+        cmd.append(osd_format)
-        cmd.append('--block.wal')
+
-        least_used_wal = find_least_used_utility_device(wal)
+    # NOTE(jamespage): enable experimental bluestore support
-        cmd.append(least_used_wal)
+    if use_bluestore():
-    db = get_devices('bluestore-db')
+        cmd.append('--bluestore')
-    if db:
+        wal = get_devices('bluestore-wal')
-        cmd.append('--block.db')
+        if wal:
-        least_used_db = find_least_used_utility_device(db)
+            cmd.append('--block.wal')
-        cmd.append(least_used_db)
+            least_used_wal = find_least_used_utility_device(wal)
+            cmd.append(least_used_wal)
+        db = get_devices('bluestore-db')
+        if db:
+            cmd.append('--block.db')
+            least_used_db = find_least_used_utility_device(db)
+            cmd.append(least_used_db)
+    elif cmp_pkgrevno('ceph', '12.1.0') >= 0 and not bluestore:
+        cmd.append('--filestore')
 
     cmd.append(os.path.realpath(dev))
 
@@ -1693,8 +1715,8 @@ def _ceph_disk(dev, osd_format, osd_journal, encrypt=False):
     return cmd
 
 
-def _ceph_volume(dev, osd_journal, encrypt=False, key_manager=CEPH_KEY_MANAGER,
+def _ceph_volume(dev, osd_journal, encrypt=False, bluestore=False,
-                 osd_id=None):
+                 key_manager=CEPH_KEY_MANAGER, osd_id=None):
     """
     Prepare and activate a device for usage as a Ceph OSD using ceph-volume.
 
@@ -1704,6 +1726,7 @@ def _ceph_volume(dev, osd_journal, encrypt=False, key_manager=CEPH_KEY_MANAGER,
     :param: dev: Full path to use for OSD block device setup
     :param: osd_journal: List of block devices to use for OSD journals
     :param: encrypt: Use block device encryption
+    :param: bluestore: Use bluestore storage for OSD
     :param: key_manager: dm-crypt Key Manager to use
     :param: osd_id: The OSD-id to recycle, or None to create a new one
     :raises subprocess.CalledProcessError: in the event that any supporting
@@ -1716,8 +1739,13 @@ def _ceph_volume(dev, osd_journal, encrypt=False, key_manager=CEPH_KEY_MANAGER,
     osd_fsid = str(uuid.uuid4())
     cmd.append('--osd-fsid')
     cmd.append(osd_fsid)
-    cmd.append('--bluestore')
+
-    main_device_type = 'block'
+    if bluestore:
+        cmd.append('--bluestore')
+        main_device_type = 'block'
+    else:
+        cmd.append('--filestore')
+        main_device_type = 'data'
 
     if encrypt and key_manager == CEPH_KEY_MANAGER:
         cmd.append('--dmcrypt')
@@ -1725,6 +1753,19 @@ def _ceph_volume(dev, osd_journal, encrypt=False, key_manager=CEPH_KEY_MANAGER,
     if osd_id is not None:
         cmd.extend(['--osd-id', str(osd_id)])
 
+    # On-disk journal volume creation
+    if not osd_journal and not bluestore:
+        journal_lv_type = 'journal'
+        cmd.append('--journal')
+        cmd.append(_allocate_logical_volume(
+            dev=dev,
+            lv_type=journal_lv_type,
+            osd_fsid=osd_fsid,
+            size='{}M'.format(calculate_volume_size('journal')),
+            encrypt=encrypt,
+            key_manager=key_manager)
+        )
+
     cmd.append('--data')
     cmd.append(_allocate_logical_volume(dev=dev,
                                         lv_type=main_device_type,
@@ -1732,21 +1773,36 @@ def _ceph_volume(dev, osd_journal, encrypt=False, key_manager=CEPH_KEY_MANAGER,
                                         encrypt=encrypt,
                                         key_manager=key_manager))
 
-    for extra_volume in ('wal', 'db'):
+    if bluestore:
-        devices = get_devices('bluestore-{}'.format(extra_volume))
+        for extra_volume in ('wal', 'db'):
-        if devices:
+            devices = get_devices('bluestore-{}'.format(extra_volume))
-            cmd.append('--block.{}'.format(extra_volume))
+            if devices:
-            least_used = find_least_used_utility_device(devices,
+                cmd.append('--block.{}'.format(extra_volume))
-                                                        lvs=True)
+                least_used = find_least_used_utility_device(devices,
-            cmd.append(_allocate_logical_volume(
+                                                            lvs=True)
-                dev=least_used,
+                cmd.append(_allocate_logical_volume(
-                lv_type=extra_volume,
+                    dev=least_used,
-                osd_fsid=osd_fsid,
+                    lv_type=extra_volume,
-                size='{}M'.format(calculate_volume_size(extra_volume)),
+                    osd_fsid=osd_fsid,
-                shared=True,
+                    size='{}M'.format(calculate_volume_size(extra_volume)),
-                encrypt=encrypt,
+                    shared=True,
-                key_manager=key_manager)
+                    encrypt=encrypt,
-            )
+                    key_manager=key_manager)
+                )
+
+    elif osd_journal:
+        cmd.append('--journal')
+        least_used = find_least_used_utility_device(osd_journal,
+                                                    lvs=True)
+        cmd.append(_allocate_logical_volume(
+            dev=least_used,
+            lv_type='journal',
+            osd_fsid=osd_fsid,
+            size='{}M'.format(calculate_volume_size('journal')),
+            shared=True,
+            encrypt=encrypt,
+            key_manager=key_manager)
+        )
 
     return cmd
 
@@ -1984,7 +2040,7 @@ def _allocate_logical_volume(dev, lv_type, osd_fsid,
     return "{}/{}".format(vg_name, lv_name)
 
 
-def osdize_dir(path, encrypt=False):
+def osdize_dir(path, encrypt=False, bluestore=False):
     """Ask ceph-disk to prepare a directory to become an OSD.
 
     :param path: str. The directory to osdize
@@ -2021,8 +2077,12 @@ def osdize_dir(path, encrypt=False):
     if cmp_pkgrevno('ceph', '0.60') >= 0:
         if encrypt:
             cmd.append('--dmcrypt')
-    cmd.append('--bluestore')
 
+    # NOTE(icey): enable experimental bluestore support
+    if cmp_pkgrevno('ceph', '10.2.0') >= 0 and bluestore:
+        cmd.append('--bluestore')
+    elif cmp_pkgrevno('ceph', '12.1.0') >= 0 and not bluestore:
+        cmd.append('--filestore')
     log("osdize dir cmd: {}".format(cmd))
     subprocess.check_call(cmd)