diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 42f15032..1e667fb0 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -545,7 +545,7 @@ class IdentityServiceContext(OSContextGenerator): 'internal_auth_url': internal_auth_url, }) - # we keep all veriables in ctxt for compatibility and + # we keep all variables in ctxt for compatibility and # add nested dictionary for keystone_authtoken generic # templating if keystonemiddleware_os_release: @@ -557,6 +557,7 @@ class IdentityServiceContext(OSContextGenerator): # NOTE(jamespage) this is required for >= icehouse # so a missing value just indicates keystone needs # upgrading + ctxt['admin_user_id'] = _resolve('service_user_id') ctxt['admin_tenant_id'] = _resolve('service_tenant_id') ctxt['admin_domain_id'] = _resolve('service_domain_id') return ctxt diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 429b09e5..da711c65 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -161,6 +161,7 @@ OPENSTACK_CODENAMES = OrderedDict([ ('2022.2', 'zed'), ('2023.1', 'antelope'), ('2023.2', 'bobcat'), + ('2024.1', 'caracal'), ]) # The ugly duckling - must list releases oldest to newest diff --git a/hooks/charmhelpers/contrib/storage/linux/lvm.py b/hooks/charmhelpers/contrib/storage/linux/lvm.py index d0a57211..0d294c79 100644 --- a/hooks/charmhelpers/contrib/storage/linux/lvm.py +++ b/hooks/charmhelpers/contrib/storage/linux/lvm.py @@ -17,8 +17,6 @@ from subprocess import ( CalledProcessError, check_call, check_output, - Popen, - PIPE, ) @@ -58,9 +56,7 @@ def remove_lvm_physical_volume(block_device): :param block_device: str: Full path of block device to scrub. ''' - p = Popen(['pvremove', '-ff', block_device], - stdin=PIPE) - p.communicate(input='y\n') + check_call(['pvremove', '-ff', '--yes', block_device]) def list_lvm_volume_group(block_device): diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 70dde6a5..def403c5 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -256,8 +256,11 @@ def service_resume(service_name, init_dir="/etc/init", upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) sysv_file = os.path.join(initd_dir, service_name) if init_is_systemd(service_name=service_name): - service('unmask', service_name) - service('enable', service_name) + if service('is-enabled', service_name): + log('service {} already enabled'.format(service_name), level=DEBUG) + else: + service('unmask', service_name) + service('enable', service_name) elif os.path.exists(upstart_file): override_path = os.path.join( init_dir, '{}.override'.format(service_name)) diff --git a/hooks/charmhelpers/fetch/ubuntu.py b/hooks/charmhelpers/fetch/ubuntu.py index 1be992c4..d0089eb7 100644 --- a/hooks/charmhelpers/fetch/ubuntu.py +++ b/hooks/charmhelpers/fetch/ubuntu.py @@ -246,6 +246,14 @@ CLOUD_ARCHIVE_POCKETS = { 'bobcat/proposed': 'jammy-proposed/bobcat', 'jammy-bobcat/proposed': 'jammy-proposed/bobcat', 'jammy-proposed/bobcat': 'jammy-proposed/bobcat', + # caracal + 'caracal': 'jammy-updates/caracal', + 'jammy-caracal': 'jammy-updates/caracal', + 'jammy-caracal/updates': 'jammy-updates/caracal', + 'jammy-updates/caracal': 'jammy-updates/caracal', + 'caracal/proposed': 'jammy-proposed/caracal', + 'jammy-caracal/proposed': 'jammy-proposed/caracal', + 'jammy-proposed/caracal': 'jammy-proposed/caracal', # OVN 'focal-ovn-22.03': 'focal-updates/ovn-22.03', @@ -279,6 +287,7 @@ OPENSTACK_RELEASES = ( 'zed', 'antelope', 'bobcat', + 'caracal', ) @@ -308,6 +317,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([ ('kinetic', 'zed'), ('lunar', 'antelope'), ('mantic', 'bobcat'), + ('noble', 'caracal'), ]) diff --git a/hooks/charmhelpers/osplatform.py b/hooks/charmhelpers/osplatform.py index 1ace468f..5d121866 100644 --- a/hooks/charmhelpers/osplatform.py +++ b/hooks/charmhelpers/osplatform.py @@ -9,19 +9,13 @@ def get_platform(): will be returned (which is the name of the module). This string is used to decide which platform module should be imported. """ - # linux_distribution is deprecated and will be removed in Python 3.7 - # Warnings *not* disabled, as we certainly need to fix this. - if hasattr(platform, 'linux_distribution'): - tuple_platform = platform.linux_distribution() - current_platform = tuple_platform[0] - else: - current_platform = _get_platform_from_fs() + current_platform = _get_current_platform() if "Ubuntu" in current_platform: return "ubuntu" elif "CentOS" in current_platform: return "centos" - elif "debian" in current_platform: + elif "debian" in current_platform or "Debian" in current_platform: # Stock Python does not detect Ubuntu and instead returns debian. # Or at least it does in some build environments like Travis CI return "ubuntu" @@ -36,6 +30,24 @@ def get_platform(): .format(current_platform)) +def _get_current_platform(): + """Return the current platform information for the OS. + + Attempts to lookup linux distribution information from the platform + module for releases of python < 3.7. For newer versions of python, + the platform is determined from the /etc/os-release file. + """ + # linux_distribution is deprecated and will be removed in Python 3.7 + # Warnings *not* disabled, as we certainly need to fix this. + if hasattr(platform, 'linux_distribution'): + tuple_platform = platform.linux_distribution() + current_platform = tuple_platform[0] + else: + current_platform = _get_platform_from_fs() + + return current_platform + + def _get_platform_from_fs(): """Get Platform from /etc/os-release.""" with open(os.path.join(os.sep, 'etc', 'os-release')) as fin: diff --git a/lib/charms_ceph/broker.py b/lib/charms_ceph/broker.py index 71f85f45..5df73c51 100644 --- a/lib/charms_ceph/broker.py +++ b/lib/charms_ceph/broker.py @@ -106,6 +106,8 @@ def decode_req_encode_rsp(f): """Decorator to decode incoming requests and encode responses.""" def decode_inner(req): + if isinstance(req, bytes): + req = req.decode('utf-8') return json.dumps(f(json.loads(req))) return decode_inner diff --git a/lib/charms_ceph/utils.py b/lib/charms_ceph/utils.py index 756dd9f1..94bfb9e4 100644 --- a/lib/charms_ceph/utils.py +++ b/lib/charms_ceph/utils.py @@ -1324,6 +1324,16 @@ def systemd(): return CompareHostReleases(lsb_release()['DISTRIB_CODENAME']) >= 'vivid' +def use_bluestore(): + """Determine whether bluestore should be used for OSD's + + :returns: whether bluestore disk format should be used + :rtype: bool""" + if cmp_pkgrevno('ceph', '12.2.0') < 0: + return False + return config('bluestore') + + def bootstrap_monitor_cluster(secret): """Bootstrap local Ceph mon into the Ceph cluster @@ -1541,21 +1551,21 @@ def get_devices(name): def osdize(dev, osd_format, osd_journal, ignore_errors=False, encrypt=False, - key_manager=CEPH_KEY_MANAGER, osd_id=None): + bluestore=False, key_manager=CEPH_KEY_MANAGER, osd_id=None): if dev.startswith('/dev'): osdize_dev(dev, osd_format, osd_journal, ignore_errors, encrypt, - key_manager, osd_id) + bluestore, key_manager, osd_id) else: if cmp_pkgrevno('ceph', '14.0.0') >= 0: log("Directory backed OSDs can not be created on Nautilus", level=WARNING) return - osdize_dir(dev, encrypt) + osdize_dir(dev, encrypt, bluestore) def osdize_dev(dev, osd_format, osd_journal, ignore_errors=False, - encrypt=False, key_manager=CEPH_KEY_MANAGER, + encrypt=False, bluestore=False, key_manager=CEPH_KEY_MANAGER, osd_id=None): """ Prepare a block device for use as a Ceph OSD @@ -1569,6 +1579,7 @@ def osdize_dev(dev, osd_format, osd_journal, ignore_errors=False, :param: ignore_errors: Don't fail in the event of any errors during processing :param: encrypt: Encrypt block devices using 'key_manager' + :param: bluestore: Use bluestore native Ceph block device format :param: key_manager: Key management approach for encryption keys :raises subprocess.CalledProcessError: in the event that any supporting subprocess operation failed @@ -1619,13 +1630,15 @@ def osdize_dev(dev, osd_format, osd_journal, ignore_errors=False, cmd = _ceph_volume(dev, osd_journal, encrypt, + bluestore, key_manager, osd_id) else: cmd = _ceph_disk(dev, osd_format, osd_journal, - encrypt) + encrypt, + bluestore) try: status_set('maintenance', 'Initializing device {}'.format(dev)) @@ -1656,7 +1669,7 @@ def osdize_dev(dev, osd_format, osd_journal, ignore_errors=False, db.flush() -def _ceph_disk(dev, osd_format, osd_journal, encrypt=False): +def _ceph_disk(dev, osd_format, osd_journal, encrypt=False, bluestore=False): """ Prepare a device for usage as a Ceph OSD using ceph-disk @@ -1664,6 +1677,7 @@ def _ceph_disk(dev, osd_format, osd_journal, encrypt=False): The function looks up realpath of the device :param: osd_journal: List of block devices to use for OSD journals :param: encrypt: Use block device encryption (unsupported) + :param: bluestore: Use bluestore storage for OSD :returns: list. 'ceph-disk' command and required parameters for execution by check_call """ @@ -1672,17 +1686,25 @@ def _ceph_disk(dev, osd_format, osd_journal, encrypt=False): if encrypt: cmd.append('--dmcrypt') - cmd.append('--bluestore') - wal = get_devices('bluestore-wal') - if wal: - cmd.append('--block.wal') - least_used_wal = find_least_used_utility_device(wal) - cmd.append(least_used_wal) - db = get_devices('bluestore-db') - if db: - cmd.append('--block.db') - least_used_db = find_least_used_utility_device(db) - cmd.append(least_used_db) + if osd_format and not bluestore: + cmd.append('--fs-type') + cmd.append(osd_format) + + # NOTE(jamespage): enable experimental bluestore support + if use_bluestore(): + cmd.append('--bluestore') + wal = get_devices('bluestore-wal') + if wal: + cmd.append('--block.wal') + least_used_wal = find_least_used_utility_device(wal) + cmd.append(least_used_wal) + db = get_devices('bluestore-db') + if db: + cmd.append('--block.db') + least_used_db = find_least_used_utility_device(db) + cmd.append(least_used_db) + elif cmp_pkgrevno('ceph', '12.1.0') >= 0 and not bluestore: + cmd.append('--filestore') cmd.append(os.path.realpath(dev)) @@ -1693,8 +1715,8 @@ def _ceph_disk(dev, osd_format, osd_journal, encrypt=False): return cmd -def _ceph_volume(dev, osd_journal, encrypt=False, key_manager=CEPH_KEY_MANAGER, - osd_id=None): +def _ceph_volume(dev, osd_journal, encrypt=False, bluestore=False, + key_manager=CEPH_KEY_MANAGER, osd_id=None): """ Prepare and activate a device for usage as a Ceph OSD using ceph-volume. @@ -1704,6 +1726,7 @@ def _ceph_volume(dev, osd_journal, encrypt=False, key_manager=CEPH_KEY_MANAGER, :param: dev: Full path to use for OSD block device setup :param: osd_journal: List of block devices to use for OSD journals :param: encrypt: Use block device encryption + :param: bluestore: Use bluestore storage for OSD :param: key_manager: dm-crypt Key Manager to use :param: osd_id: The OSD-id to recycle, or None to create a new one :raises subprocess.CalledProcessError: in the event that any supporting @@ -1716,8 +1739,13 @@ def _ceph_volume(dev, osd_journal, encrypt=False, key_manager=CEPH_KEY_MANAGER, osd_fsid = str(uuid.uuid4()) cmd.append('--osd-fsid') cmd.append(osd_fsid) - cmd.append('--bluestore') - main_device_type = 'block' + + if bluestore: + cmd.append('--bluestore') + main_device_type = 'block' + else: + cmd.append('--filestore') + main_device_type = 'data' if encrypt and key_manager == CEPH_KEY_MANAGER: cmd.append('--dmcrypt') @@ -1725,6 +1753,19 @@ def _ceph_volume(dev, osd_journal, encrypt=False, key_manager=CEPH_KEY_MANAGER, if osd_id is not None: cmd.extend(['--osd-id', str(osd_id)]) + # On-disk journal volume creation + if not osd_journal and not bluestore: + journal_lv_type = 'journal' + cmd.append('--journal') + cmd.append(_allocate_logical_volume( + dev=dev, + lv_type=journal_lv_type, + osd_fsid=osd_fsid, + size='{}M'.format(calculate_volume_size('journal')), + encrypt=encrypt, + key_manager=key_manager) + ) + cmd.append('--data') cmd.append(_allocate_logical_volume(dev=dev, lv_type=main_device_type, @@ -1732,21 +1773,36 @@ def _ceph_volume(dev, osd_journal, encrypt=False, key_manager=CEPH_KEY_MANAGER, encrypt=encrypt, key_manager=key_manager)) - for extra_volume in ('wal', 'db'): - devices = get_devices('bluestore-{}'.format(extra_volume)) - if devices: - cmd.append('--block.{}'.format(extra_volume)) - least_used = find_least_used_utility_device(devices, - lvs=True) - cmd.append(_allocate_logical_volume( - dev=least_used, - lv_type=extra_volume, - osd_fsid=osd_fsid, - size='{}M'.format(calculate_volume_size(extra_volume)), - shared=True, - encrypt=encrypt, - key_manager=key_manager) - ) + if bluestore: + for extra_volume in ('wal', 'db'): + devices = get_devices('bluestore-{}'.format(extra_volume)) + if devices: + cmd.append('--block.{}'.format(extra_volume)) + least_used = find_least_used_utility_device(devices, + lvs=True) + cmd.append(_allocate_logical_volume( + dev=least_used, + lv_type=extra_volume, + osd_fsid=osd_fsid, + size='{}M'.format(calculate_volume_size(extra_volume)), + shared=True, + encrypt=encrypt, + key_manager=key_manager) + ) + + elif osd_journal: + cmd.append('--journal') + least_used = find_least_used_utility_device(osd_journal, + lvs=True) + cmd.append(_allocate_logical_volume( + dev=least_used, + lv_type='journal', + osd_fsid=osd_fsid, + size='{}M'.format(calculate_volume_size('journal')), + shared=True, + encrypt=encrypt, + key_manager=key_manager) + ) return cmd @@ -1984,7 +2040,7 @@ def _allocate_logical_volume(dev, lv_type, osd_fsid, return "{}/{}".format(vg_name, lv_name) -def osdize_dir(path, encrypt=False): +def osdize_dir(path, encrypt=False, bluestore=False): """Ask ceph-disk to prepare a directory to become an OSD. :param path: str. The directory to osdize @@ -2021,8 +2077,12 @@ def osdize_dir(path, encrypt=False): if cmp_pkgrevno('ceph', '0.60') >= 0: if encrypt: cmd.append('--dmcrypt') - cmd.append('--bluestore') + # NOTE(icey): enable experimental bluestore support + if cmp_pkgrevno('ceph', '10.2.0') >= 0 and bluestore: + cmd.append('--bluestore') + elif cmp_pkgrevno('ceph', '12.1.0') >= 0 and not bluestore: + cmd.append('--filestore') log("osdize dir cmd: {}".format(cmd)) subprocess.check_call(cmd)