openstack
/
charm-nova-compute
Code Issues Proposed changes

Add support for Erasure Coded pools 

Enable support for use of Erasure Coded (EC) pools for
nova disks when RBD is used to back ephemeral storage volumes.

Add the standard set of EC based configuration options to the
charm.

Update Ceph broker request to create a replicated pool, an erasure
coding profile and an erasure coded pool (using the profile) when
pool-type == erasure-coded is specified.

Resync charm-helpers to pick changes to the standard ceph.conf
template and associated contexts for rbd default data pool mangle
due to lack for explicit support in OpenStack Services.

Update context to use metadata pool name in nova configuration
when erasure-coding is enabled.

Change-Id: Ida0b9c889ddf9fcc0847a9cee01b3206239d9318
Depends-On: Iec4de19f7b39f0b08158d96c5cc1561b40aefa10
This commit is contained in:
James Page 2020-08-04 14:04:04 +01:00
parent 94862373d4
commit 5f4f95ef13
10 changed files with 1339 additions and 429 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
config.yaml
View File

@ -415,6 +415,104 @@ options:
type: boolean
description: |
Optionally restrict Ceph key permissions to access pools as required.
pool-type:
type: string
default: replicated
description: |
Ceph pool type to use for storage - valid values include ‘replicated’
and ‘erasure-coded’.
ec-profile-name:
type: string
default:
description: |
Name for the EC profile to be created for the EC pools. If not defined
a profile name will be generated based on the name of the pool used by
the application.
ec-rbd-metadata-pool:
type: string
default:
description: |
Name of the metadata pool to be created (for RBD use-cases). If not
defined a metadata pool name will be generated based on the name of
the data pool used by the application. The metadata pool is always
replicated, not erasure coded.
ec-profile-k:
type: int
default: 1
description: |
Number of data chunks that will be used for EC data pool. K+M factors
should never be greater than the number of available zones (or hosts)
for balancing.
ec-profile-m:
type: int
default: 2
description: |
Number of coding chunks that will be used for EC data pool. K+M factors
should never be greater than the number of available zones (or hosts)
for balancing.
ec-profile-locality:
type: int
default:
description: |
(lrc plugin - l) Group the coding and data chunks into sets of size l.
For instance, for k=4 and m=2, when l=3 two groups of three are created.
Each set can be recovered without reading chunks from another set. Note
that using the lrc plugin does incur more raw storage usage than isa or
jerasure in order to reduce the cost of recovery operations.
ec-profile-crush-locality:
type: string
default:
description: |
(lrc plugin) The type of the crush bucket in which each set of chunks
defined by l will be stored. For instance, if it is set to rack, each
group of l chunks will be placed in a different rack. It is used to
create a CRUSH rule step such as step choose rack. If it is not set,
no such grouping is done.
ec-profile-durability-estimator:
type: int
default:
description: |
(shec plugin - c) The number of parity chunks each of which includes
each data chunk in its calculation range. The number is used as a
durability estimator. For instance, if c=2, 2 OSDs can be down
without losing data.
ec-profile-helper-chunks:
type: int
default:
description: |
(clay plugin - d) Number of OSDs requested to send data during
recovery of a single chunk. d needs to be chosen such that
k+1 <= d <= k+m-1. Larger the d, the better the savings.
ec-profile-scalar-mds:
type: string
default:
description: |
(clay plugin) specifies the plugin that is used as a building
block in the layered construction. It can be one of jerasure,
isa, shec (defaults to jerasure).
ec-profile-plugin:
type: string
default: jerasure
description: |
EC plugin to use for this applications pool. The following list of
plugins acceptable - jerasure, lrc, isa, shec, clay.
ec-profile-technique:
type: string
default:
description: |
EC profile technique used for this applications pool - will be
validated based on the plugin configured via ec-profile-plugin.
Supported techniques are ‘reed_sol_van’, ‘reed_sol_r6_op’,
‘cauchy_orig’, ‘cauchy_good’, ‘liber8tion’ for jerasure,
‘reed_sol_van’, ‘cauchy’ for isa and ‘single’, ‘multiple’
for shec.
ec-profile-device-class:
type: string
default:
description: |
Device class from CRUSH map to use for placement groups for
erasure profile - valid values: ssd, hdd or nvme (or leave
unset to not use a device class).
# Other config
sysctl:
type: string

84
hooks/charmhelpers/contrib/openstack/context.py
View File

@ -29,6 +29,8 @@ from subprocess import check_call, CalledProcessError
import six
import charmhelpers.contrib.storage.linux.ceph as ch_ceph
from charmhelpers.contrib.openstack.audits.openstack_security_guide import (
_config_ini as config_ini
)
@ -56,6 +58,7 @@ from charmhelpers.core.hookenv import (
status_set,
network_get_primary_address,
WARNING,
service_name,
)
from charmhelpers.core.sysctl import create as sysctl_create
@ -808,6 +811,12 @@ class CephContext(OSContextGenerator):
ctxt['mon_hosts'] = ' '.join(sorted(mon_hosts))
if config('pool-type') and config('pool-type') == 'erasure-coded':
base_pool_name = config('rbd-pool') or config('rbd-pool-name')
if not base_pool_name:
base_pool_name = service_name()
ctxt['rbd_default_data_pool'] = base_pool_name
if not os.path.isdir('/etc/ceph'):
os.mkdir('/etc/ceph')
@ -3175,3 +3184,78 @@ class SRIOVContext(OSContextGenerator):
:rtype: Dict[str,int]
"""
return self._map
class CephBlueStoreCompressionContext(OSContextGenerator):
"""Ceph BlueStore compression options."""
# Tuple with Tuples that map configuration option name to CephBrokerRq op
# property name
options = (
('bluestore-compression-algorithm',
'compression-algorithm'),
('bluestore-compression-mode',
'compression-mode'),
('bluestore-compression-required-ratio',
'compression-required-ratio'),
('bluestore-compression-min-blob-size',
'compression-min-blob-size'),
('bluestore-compression-min-blob-size-hdd',
'compression-min-blob-size-hdd'),
('bluestore-compression-min-blob-size-ssd',
'compression-min-blob-size-ssd'),
('bluestore-compression-max-blob-size',
'compression-max-blob-size'),
('bluestore-compression-max-blob-size-hdd',
'compression-max-blob-size-hdd'),
('bluestore-compression-max-blob-size-ssd',
'compression-max-blob-size-ssd'),
)
def __init__(self):
"""Initialize context by loading values from charm config.
We keep two maps, one suitable for use with CephBrokerRq's and one
suitable for template generation.
"""
charm_config = config()
# CephBrokerRq op map
self.op = {}
# Context exposed for template generation
self.ctxt = {}
for config_key, op_key in self.options:
value = charm_config.get(config_key)
self.ctxt.update({config_key.replace('-', '_'): value})
self.op.update({op_key: value})
def __call__(self):
"""Get context.
:returns: Context
:rtype: Dict[str,any]
"""
return self.ctxt
def get_op(self):
"""Get values for use in CephBrokerRq op.
:returns: Context values with CephBrokerRq op property name as key.
:rtype: Dict[str,any]
"""
return self.op
def validate(self):
"""Validate options.
:raises: AssertionError
"""
# We slip in a dummy name on class instantiation to allow validation of
# the other options. It will not affect further use.
#
# NOTE: once we retire Python 3.5 we can fold this into a in-line
# dictionary comprehension in the call to the initializer.
dummy_op = {'name': 'dummy-name'}
dummy_op.update(self.op)
pool = ch_ceph.BasePool('dummy-service', op=dummy_op)
pool.validate()

4
hooks/charmhelpers/contrib/openstack/templates/ceph.conf
View File

@ -22,3 +22,7 @@ rbd default features = {{ rbd_features }}
{{ key }} = {{ value }}
{% endfor -%}
{%- endif %}
{% if rbd_default_data_pool -%}
rbd default data pool = {{ rbd_default_data_pool }}
{% endif %}

28
hooks/charmhelpers/contrib/openstack/templates/section-ceph-bluestore-compression Normal file
View File

@ -0,0 +1,28 @@
{# section header omitted as options can belong to multiple sections #}
{% if bluestore_compression_algorithm -%}
bluestore compression algorithm = {{ bluestore_compression_algorithm }}
{% endif -%}
{% if bluestore_compression_mode -%}
bluestore compression mode = {{ bluestore_compression_mode }}
{% endif -%}
{% if bluestore_compression_required_ratio -%}
bluestore compression required ratio = {{ bluestore_compression_required_ratio }}
{% endif -%}
{% if bluestore_compression_min_blob_size -%}
bluestore compression min blob size = {{ bluestore_compression_min_blob_size }}
{% endif -%}
{% if bluestore_compression_min_blob_size_hdd -%}
bluestore compression min blob size hdd = {{ bluestore_compression_min_blob_size_hdd }}
{% endif -%}
{% if bluestore_compression_min_blob_size_ssd -%}
bluestore compression min blob size ssd = {{ bluestore_compression_min_blob_size_ssd }}
{% endif -%}
{% if bluestore_compression_max_blob_size -%}
bluestore compression max blob size = {{ bluestore_compression_max_blob_size }}
{% endif -%}
{% if bluestore_compression_max_blob_size_hdd -%}
bluestore compression max blob size hdd = {{ bluestore_compression_max_blob_size_hdd }}
{% endif -%}
{% if bluestore_compression_max_blob_size_ssd -%}
bluestore compression max blob size ssd = {{ bluestore_compression_max_blob_size_ssd }}
{% endif -%}

0
hooks/charmhelpers/contrib/python/__init__.py Normal file
View File

1395
hooks/charmhelpers/contrib/storage/linux/ceph.py
View File

File diff suppressed because it is too large Load Diff

9
hooks/nova_compute_context.py
View File

@ -365,7 +365,14 @@ class NovaComputeCephContext(context.CephContext):
ctxt['service_name'] = svc
ctxt['rbd_user'] = svc
ctxt['rbd_secret_uuid'] = CEPH_SECRET_UUID
ctxt['rbd_pool'] = config('rbd-pool')
if config('pool-type') == 'erasure-coded':
ctxt['rbd_pool'] = (
config('ec-rbd-metadata-pool') or
"{}-metadata".format(config('rbd-pool'))
)
else:
ctxt['rbd_pool'] = config('rbd-pool')
if (config('libvirt-image-backend') == 'rbd' and
assert_libvirt_rbd_imagebackend_allowed()):

69
hooks/nova_compute_hooks.py
View File

@ -381,11 +381,74 @@ def get_ceph_request():
rq = CephBrokerRq()
if (config('libvirt-image-backend') == 'rbd' and
assert_libvirt_rbd_imagebackend_allowed()):
name = config('rbd-pool')
pool_name = config('rbd-pool')
replicas = config('ceph-osd-replication-count')
weight = config('ceph-pool-weight')
rq.add_op_create_pool(name=name, replica_count=replicas, weight=weight,
group='vms', app_name='rbd')
if config('pool-type') == 'erasure-coded':
# General EC plugin config
plugin = config('ec-profile-plugin')
technique = config('ec-profile-technique')
device_class = config('ec-profile-device-class')
metadata_pool_name = (
config('ec-rbd-metadata-pool') or
"{}-metadata".format(pool_name)
)
bdm_k = config('ec-profile-k')
bdm_m = config('ec-profile-m')
# LRC plugin config
bdm_l = config('ec-profile-locality')
crush_locality = config('ec-profile-crush-locality')
# SHEC plugin config
bdm_c = config('ec-profile-durability-estimator')
# CLAY plugin config
bdm_d = config('ec-profile-helper-chunks')
scalar_mds = config('ec-profile-scalar-mds')
# Profile name
profile_name = (
config('ec-profile-name') or
"{}-profile".format(pool_name)
)
# Metadata sizing is approximately 1% of overall data weight
# but is in effect driven by the number of rbd's rather than
# their size - so it can be very lightweight.
metadata_weight = weight * 0.01
# Resize data pool weight to accomodate metadata weight
weight = weight - metadata_weight
# Create metadata pool
rq.add_op_create_pool(
name=metadata_pool_name, replica_count=replicas,
weight=metadata_weight, group='vms', app_name='rbd'
)
# Create erasure profile
rq.add_op_create_erasure_profile(
name=profile_name,
k=bdm_k, m=bdm_m,
lrc_locality=bdm_l,
lrc_crush_locality=crush_locality,
shec_durability_estimator=bdm_c,
clay_helper_chunks=bdm_d,
clay_scalar_mds=scalar_mds,
device_class=device_class,
erasure_type=plugin,
erasure_technique=technique
)
# Create EC data pool
rq.add_op_create_erasure_pool(
name=pool_name,
erasure_profile=profile_name,
weight=weight,
group="vms",
app_name="rbd",
allow_ec_overwrites=True
)
else:
rq.add_op_create_pool(name=pool_name, replica_count=replicas,
weight=weight,
group='vms', app_name='rbd')
if config('restrict-ceph-pools'):
rq.add_op_request_access_to_group(
name="volumes",

25
unit_tests/test_nova_compute_contexts.py
View File

@ -932,3 +932,28 @@ class InstanceConsoleContextTest(CharmTestCase):
ctxt = context.InstanceConsoleContext()()
self.assertEqual(ctxt['spice_agent_enabled'], True, str(ctxt))
class NovaComputeCephContextTest(CharmTestCase):
def setUp(self):
super().setUp(context, TO_PATCH)
self.config.side_effect = self.test_config.get
self.os_release.return_value = 'queens'
@patch('charmhelpers.contrib.openstack.context.CephContext.__call__')
def test_rbd_replicated_pool(self, mock_call):
mock_call.return_value = {'mon_hosts': 'foo,bar'}
ctxt = context.NovaComputeCephContext()()
self.assertEqual(ctxt['rbd_pool'], 'nova')
@patch('charmhelpers.contrib.openstack.context.CephContext.__call__')
def test_rbd_ec_pool(self, mock_call):
self.test_config.set('pool-type', 'erasure-coded')
mock_call.return_value = {'mon_hosts': 'foo,bar'}
ctxt = context.NovaComputeCephContext()()
self.assertEqual(ctxt['rbd_pool'], 'nova-metadata')
self.test_config.set('ec-rbd-metadata-pool', 'nova-newmetadata')
ctxt = context.NovaComputeCephContext()()
self.assertEqual(ctxt['rbd_pool'], 'nova-newmetadata')

56
unit_tests/test_nova_compute_hooks.py
View File

@ -683,6 +683,62 @@ class NovaComputeRelationsTests(CharmTestCase):
mock_request_access.assert_not_called()
self.assertEqual(expected, result)
@patch('charmhelpers.contrib.storage.linux.ceph.CephBrokerRq'
'.add_op_create_erasure_pool')
@patch('charmhelpers.contrib.storage.linux.ceph.CephBrokerRq'
'.add_op_create_erasure_profile')
@patch('charmhelpers.contrib.storage.linux.ceph.CephBrokerRq'
'.add_op_request_access_to_group')
@patch('charmhelpers.contrib.storage.linux.ceph.CephBrokerRq'
'.add_op_create_pool')
@patch('uuid.uuid1')
def test_get_ceph_request_rbd_ec(self, uuid1, mock_create_pool,
mock_request_access,
mock_create_erasure_profile,
mock_create_erasure_pool):
self.assert_libvirt_rbd_imagebackend_allowed.return_value = True
self.test_config.set('rbd-pool', 'nova')
self.test_config.set('ceph-osd-replication-count', 3)
self.test_config.set('ceph-pool-weight', 28)
self.test_config.set('libvirt-image-backend', 'rbd')
self.test_config.set('pool-type', 'erasure-coded')
self.test_config.set('ec-profile-plugin', 'shec')
self.test_config.set('ec-profile-k', 6)
self.test_config.set('ec-profile-m', 2)
self.test_config.set('ec-profile-durability-estimator', 2)
uuid1.return_value = 'my_uuid'
expected = hooks.CephBrokerRq(request_id="my_uuid")
result = hooks.get_ceph_request()
mock_create_pool.assert_called_with(
name='nova-metadata',
replica_count=3,
weight=0.28,
group='vms',
app_name='rbd'
)
mock_create_erasure_profile.assert_called_with(
name='nova-profile',
k=6, m=2,
lrc_locality=None,
lrc_crush_locality=None,
shec_durability_estimator=2,
clay_helper_chunks=None,
clay_scalar_mds=None,
device_class=None,
erasure_type='shec',
erasure_technique=None
)
mock_create_erasure_pool.assert_called_with(
name='nova',
erasure_profile='nova-profile',
weight=27.72,
group='vms',
app_name='rbd',
allow_ec_overwrites=True
)
mock_request_access.assert_not_called()
self.assertEqual(expected, result)
@patch('charmhelpers.contrib.storage.linux.ceph.CephBrokerRq'
'.add_op_request_access_to_group')
@patch('charmhelpers.contrib.storage.linux.ceph.CephBrokerRq'