openstack
/
charm-ceph-mon
Code Issues Proposed changes

Update Amulet defs, series metadata and c-h sync 

- Sync charm helpers if applicable.

- Fix test executable hashbags for virtualenv prep.

- Add Yakkety-Newton Amulet test definitions.

- Prep Xenial-Ocata Amulet test definitions (not yet enabled).

- Prep Zesty-Ocata Amulet test definitions (not yet enabled).

- Remove Precise charm series metadata if present.

- Remove Precise Amulet test definitions if present.

Change-Id: I00a4c855d4da01b22110f06bf2820bca0bcde7b9
This commit is contained in:
Ryan Beisner 2016-11-23 15:16:18 -06:00
parent d9c538e065
commit 5d8541b573
26 changed files with 1749 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
hooks/charmhelpers/contrib/openstack/amulet/deployment.py
View File

@ -69,9 +69,9 @@ class OpenStackAmuletDeployment(AmuletDeployment):
# Charms outside the ~openstack-charmers
base_charms = {
'mysql': ['precise', 'trusty'],
'mongodb': ['precise', 'trusty'],
'nrpe': ['precise', 'trusty', 'wily', 'xenial'],
'mysql': ['trusty'],
'mongodb': ['trusty'],
'nrpe': ['trusty', 'xenial'],
}
for svc in other_services:
@ -260,31 +260,20 @@ class OpenStackAmuletDeployment(AmuletDeployment):
release.
"""
# Must be ordered by OpenStack release (not by Ubuntu release):
(self.precise_essex, self.precise_folsom, self.precise_grizzly,
self.precise_havana, self.precise_icehouse,
self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
self.trusty_kilo, self.vivid_kilo, self.trusty_liberty,
self.wily_liberty, self.trusty_mitaka,
self.xenial_mitaka, self.xenial_newton,
self.yakkety_newton) = range(16)
(self.trusty_icehouse, self.trusty_kilo, self.trusty_liberty,
self.trusty_mitaka, self.xenial_mitaka, self.xenial_newton,
self.yakkety_newton, self.xenial_ocata, self.zesty_ocata) = range(9)
releases = {
('precise', None): self.precise_essex,
('precise', 'cloud:precise-folsom'): self.precise_folsom,
('precise', 'cloud:precise-grizzly'): self.precise_grizzly,
('precise', 'cloud:precise-havana'): self.precise_havana,
('precise', 'cloud:precise-icehouse'): self.precise_icehouse,
('trusty', None): self.trusty_icehouse,
('trusty', 'cloud:trusty-juno'): self.trusty_juno,
('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
('trusty', 'cloud:trusty-mitaka'): self.trusty_mitaka,
('utopic', None): self.utopic_juno,
('vivid', None): self.vivid_kilo,
('wily', None): self.wily_liberty,
('xenial', None): self.xenial_mitaka,
('xenial', 'cloud:xenial-newton'): self.xenial_newton,
('xenial', 'cloud:xenial-ocata'): self.xenial_ocata,
('yakkety', None): self.yakkety_newton,
('zesty', None): self.zesty_ocata,
}
return releases[(self.series, self.openstack)]
@ -294,16 +283,10 @@ class OpenStackAmuletDeployment(AmuletDeployment):
Return a string representing the openstack release.
"""
releases = OrderedDict([
('precise', 'essex'),
('quantal', 'folsom'),
('raring', 'grizzly'),
('saucy', 'havana'),
('trusty', 'icehouse'),
('utopic', 'juno'),
('vivid', 'kilo'),
('wily', 'liberty'),
('xenial', 'mitaka'),
('yakkety', 'newton'),
('zesty', 'ocata'),
])
if self.openstack:
os_origin = self.openstack.split(':')[1]

11
hooks/charmhelpers/contrib/openstack/amulet/utils.py
View File

@ -28,6 +28,7 @@ import keystoneclient.v2_0 as keystone_client
from keystoneclient.auth.identity import v3 as keystone_id_v3
from keystoneclient import session as keystone_session
from keystoneclient.v3 import client as keystone_client_v3
from novaclient import exceptions
import novaclient.client as nova_client
import pika
@ -377,6 +378,16 @@ class OpenStackAmuletUtils(AmuletUtils):
tenant_name=tenant,
auth_version='2.0')
def create_flavor(self, nova, name, ram, vcpus, disk, flavorid="auto",
ephemeral=0, swap=0, rxtx_factor=1.0, is_public=True):
"""Create the specified flavor."""
try:
nova.flavors.find(name=name)
except (exceptions.NotFound, exceptions.NoUniqueMatch):
self.log.debug('Creating flavor ({})'.format(name))
nova.flavors.create(name, ram, vcpus, disk, flavorid,
ephemeral, swap, rxtx_factor, is_public)
def create_cirros_image(self, glance, image_name):
"""Download the latest cirros image and upload it to glance,
validate and return a resource pointer.

6
hooks/charmhelpers/contrib/openstack/context.py
View File

@ -344,6 +344,10 @@ class IdentityServiceContext(OSContextGenerator):
'auth_protocol': auth_protocol,
'api_version': api_version})
if float(api_version) > 2:
ctxt.update({'admin_domain_name':
rdata.get('service_domain')})
if self.context_complete(ctxt):
# NOTE(jamespage) this is required for >= icehouse
# so a missing value just indicates keystone needs
@ -644,7 +648,7 @@ class ApacheSSLContext(OSContextGenerator):
service_namespace = None
def enable_modules(self):
cmd = ['a2enmod', 'ssl', 'proxy', 'proxy_http']
cmd = ['a2enmod', 'ssl', 'proxy', 'proxy_http', 'headers']
check_call(cmd)
def configure_cert(self, cn=None):

2
hooks/charmhelpers/contrib/openstack/neutron.py
View File

@ -32,6 +32,7 @@ def headers_package():
kver = check_output(['uname', '-r']).decode('UTF-8').strip()
return 'linux-headers-%s' % kver
QUANTUM_CONF_DIR = '/etc/quantum'
@ -91,6 +92,7 @@ def quantum_plugins():
}
}
NEUTRON_CONF_DIR = '/etc/neutron'

22
hooks/charmhelpers/contrib/openstack/utils.py
View File

@ -109,7 +109,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
('wily', 'liberty'),
('xenial', 'mitaka'),
('yakkety', 'newton'),
('zebra', 'ocata'), # TODO: upload with real Z name
('zesty', 'ocata'),
])
@ -152,6 +152,8 @@ SWIFT_CODENAMES = OrderedDict([
['2.5.0', '2.6.0', '2.7.0']),
('newton',
['2.8.0', '2.9.0', '2.10.0']),
('ocata',
['2.11.0']),
])
# >= Liberty version->codename mapping
@ -410,14 +412,26 @@ def get_os_version_package(pkg, fatal=True):
os_rel = None
def os_release(package, base='essex'):
def reset_os_release():
'''Unset the cached os_release version'''
global os_rel
os_rel = None
def os_release(package, base='essex', reset_cache=False):
'''
Returns OpenStack release codename from a cached global.
If reset_cache then unset the cached os_release version and return the
freshly determined version.
If the codename can not be determined from either an installed package or
the installation source, the earliest release supported by the charm should
be returned.
'''
global os_rel
if reset_cache:
reset_os_release()
if os_rel:
return os_rel
os_rel = (git_os_codename_install_source(config('openstack-origin-git')) or
@ -535,6 +549,9 @@ def configure_installation_source(rel):
'newton': 'xenial-updates/newton',
'newton/updates': 'xenial-updates/newton',
'newton/proposed': 'xenial-proposed/newton',
'zesty': 'zesty-updates/ocata',
'zesty/updates': 'xenial-updates/ocata',
'zesty/proposed': 'xenial-proposed/ocata',
}
try:
@ -668,6 +685,7 @@ def clean_storage(block_device):
else:
zap_disk(block_device)
is_ip = ip.is_ip
ns_query = ip.ns_query
get_host_ip = ip.get_host_ip

2
hooks/charmhelpers/core/hookenv.py
View File

@ -332,6 +332,8 @@ def config(scope=None):
config_cmd_line = ['config-get']
if scope is not None:
config_cmd_line.append(scope)
else:
config_cmd_line.append('--all')
config_cmd_line.append('--format=json')
try:
config_data = json.loads(

17
hooks/charmhelpers/core/host.py
View File

@ -732,3 +732,20 @@ def get_total_ram():
assert unit == 'kB', 'Unknown unit'
return int(value) * 1024 # Classic, not KiB.
raise NotImplementedError()
UPSTART_CONTAINER_TYPE = '/run/container_type'
def is_container():
"""Determine whether unit is running in a container
@return: boolean indicating if unit is in a container
"""
if init_is_systemd():
# Detect using systemd-detect-virt
return subprocess.call(['systemd-detect-virt',
'--container']) == 0
else:
# Detect using upstart container file marker
return os.path.exists(UPSTART_CONTAINER_TYPE)

2
hooks/charmhelpers/core/kernel_factory/ubuntu.py
View File

@ -5,7 +5,7 @@ def persistent_modprobe(module):
"""Load a kernel module and configure for auto-load on reboot."""
with open('/etc/modules', 'r+') as modules:
if module not in modules.read():
modules.write(module)
modules.write(module + "\n")
def update_initramfs(version='all'):

8
hooks/charmhelpers/fetch/ubuntu.py
View File

@ -105,6 +105,14 @@ CLOUD_ARCHIVE_POCKETS = {
'newton/proposed': 'xenial-proposed/newton',
'xenial-newton/proposed': 'xenial-proposed/newton',
'xenial-proposed/newton': 'xenial-proposed/newton',
# Ocata
'ocata': 'xenial-updates/ocata',
'xenial-ocata': 'xenial-updates/ocata',
'xenial-ocata/updates': 'xenial-updates/ocata',
'xenial-updates/ocata': 'xenial-updates/ocata',
'ocata/proposed': 'xenial-proposed/ocata',
'xenial-ocata/proposed': 'xenial-proposed/ocata',
'xenial-ocata/newton': 'xenial-proposed/ocata',
}
APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT.

90
lib/ceph/__init__.py
View File

@ -23,6 +23,7 @@ import re
import sys
import errno
import shutil
import pyudev
from charmhelpers.core import hookenv
from charmhelpers.core.host import (
@ -50,13 +51,15 @@ from charmhelpers.contrib.storage.linux.utils import (
is_block_device,
zap_disk,
is_device_mounted)
from charmhelpers.contrib.openstack.utils import (
get_os_codename_install_source)
LEADER = 'leader'
PEON = 'peon'
QUORUM = [LEADER, PEON]
PACKAGES = ['ceph', 'gdisk', 'ntp', 'btrfs-tools', 'python-ceph',
'radosgw', 'xfsprogs']
'radosgw', 'xfsprogs', 'python-pyudev']
LinkSpeed = {
"BASE_10": 10,
@ -100,6 +103,23 @@ NETWORK_ADAPTER_SYSCTLS = {
}
def unmounted_disks():
"""List of unmounted block devices on the current host."""
disks = []
context = pyudev.Context()
for device in context.list_devices(DEVTYPE='disk'):
if device['SUBSYSTEM'] == 'block':
matched = False
for block_type in [u'dm', u'loop', u'ram', u'nbd']:
if block_type in device.device_node:
matched = True
if matched:
continue
disks.append(device.device_node)
log("Found disks: {}".format(disks))
return [disk for disk in disks if not is_device_mounted(disk)]
def save_sysctls(sysctl_dict, save_location):
"""
Persist the sysctls to the hard drive.
@ -1344,7 +1364,7 @@ def roll_monitor_cluster(new_version, upgrade_key):
version=new_version)
else:
# Check if the previous node has finished
status_set('blocked',
status_set('waiting',
'Waiting on {} to finish upgrading'.format(
mon_sorted_list[position - 1]))
wait_on_previous_node(upgrade_key=upgrade_key,
@ -1361,11 +1381,10 @@ def roll_monitor_cluster(new_version, upgrade_key):
status_set('blocked', 'failed to upgrade monitor')
def upgrade_monitor():
def upgrade_monitor(new_version):
current_version = get_version()
status_set("maintenance", "Upgrading monitor")
log("Current ceph version is {}".format(current_version))
new_version = config('release-version')
log("Upgrading to: {}".format(new_version))
try:
@ -1393,7 +1412,6 @@ def upgrade_monitor():
service_start('ceph-mon@{}'.format(mon_id))
else:
service_start('ceph-mon-all')
status_set("active", "")
except subprocess.CalledProcessError as err:
log("Stopping ceph and upgrading packages failed "
"with message: {}".format(err.message))
@ -1415,9 +1433,9 @@ def lock_and_roll(upgrade_key, service, my_name, version):
# This should be quick
if service == 'osd':
upgrade_osd()
upgrade_osd(version)
elif service == 'mon':
upgrade_monitor()
upgrade_monitor(version)
else:
log("Unknown service {}. Unable to upgrade".format(service),
level=ERROR)
@ -1541,11 +1559,10 @@ def roll_osd_cluster(new_version, upgrade_key):
status_set('blocked', 'failed to upgrade osd')
def upgrade_osd():
def upgrade_osd(new_version):
current_version = get_version()
status_set("maintenance", "Upgrading osd")
log("Current ceph version is {}".format(current_version))
new_version = config('release-version')
log("Upgrading to: {}".format(new_version))
try:
@ -1578,3 +1595,58 @@ def upgrade_osd():
"with message: {}".format(err.message))
status_set("blocked", "Upgrade to {} failed".format(new_version))
sys.exit(1)
def list_pools(service):
"""
This will list the current pools that Ceph has
:param service: String service id to run under
:return: list. Returns a list of the ceph pools. Raises CalledProcessError
if the subprocess fails to run.
"""
try:
pool_list = []
pools = subprocess.check_output(['rados', '--id', service, 'lspools'])
for pool in pools.splitlines():
pool_list.append(pool)
return pool_list
except subprocess.CalledProcessError as err:
log("rados lspools failed with error: {}".format(err.output))
raise
# A dict of valid ceph upgrade paths. Mapping is old -> new
UPGRADE_PATHS = {
'firefly': 'hammer',
'hammer': 'jewel',
}
# Map UCA codenames to ceph codenames
UCA_CODENAME_MAP = {
'icehouse': 'firefly',
'juno': 'firefly',
'kilo': 'hammer',
'liberty': 'hammer',
'mitaka': 'jewel',
}
def pretty_print_upgrade_paths():
'''Pretty print supported upgrade paths for ceph'''
lines = []
for key, value in UPGRADE_PATHS.iteritems():
lines.append("{} -> {}".format(key, value))
return lines
def resolve_ceph_version(source):
'''
Resolves a version of ceph based on source configuration
based on Ubuntu Cloud Archive pockets.
@param: source: source configuration option of charm
@returns: ceph release codename or None if not resolvable
'''
os_release = get_os_codename_install_source(source)
return UCA_CODENAME_MAP.get(os_release)

2
lib/ceph/ceph_broker.py
View File

@ -424,7 +424,7 @@ def handle_create_cephfs(request, service):
log("CephFS already created")
return
# Finally create CephFS
# Finally create CephFS
try:
check_output(["ceph",
'--id', service,

1514
lib/ceph/ceph_helpers.py Normal file
View File

File diff suppressed because it is too large Load Diff

1
metadata.yaml
View File

@ -12,7 +12,6 @@ tags:
series:
- xenial
- trusty
- precise
- yakkety
peers:
mon:

4
tests/basic_deployment.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/env python
#
# Copyright 2016 Canonical Ltd
#
@ -228,7 +228,7 @@ class CephBasicDeployment(OpenStackAmuletDeployment):
'cinder-volume'],
}
if self._get_openstack_release() < self.vivid_kilo:
if self._get_openstack_release() < self.xenial_mitaka:
# For upstart systems only. Ceph services under systemd
# are checked by process name instead.
ceph_services = [

2
tests/charmhelpers/contrib/amulet/utils.py
View File

@ -546,7 +546,7 @@ class AmuletUtils(object):
raise if it is present.
:returns: List of process IDs
"""
cmd = 'pidof -x {}'.format(process_name)
cmd = 'pidof -x "{}"'.format(process_name)
if not expect_success:
cmd += " || exit 0 && exit 1"
output, code = sentry_unit.run(cmd)

35
tests/charmhelpers/contrib/openstack/amulet/deployment.py
View File

@ -69,9 +69,9 @@ class OpenStackAmuletDeployment(AmuletDeployment):
# Charms outside the ~openstack-charmers
base_charms = {
'mysql': ['precise', 'trusty'],
'mongodb': ['precise', 'trusty'],
'nrpe': ['precise', 'trusty', 'wily', 'xenial'],
'mysql': ['trusty'],
'mongodb': ['trusty'],
'nrpe': ['trusty', 'xenial'],
}
for svc in other_services:
@ -260,31 +260,20 @@ class OpenStackAmuletDeployment(AmuletDeployment):
release.
"""
# Must be ordered by OpenStack release (not by Ubuntu release):
(self.precise_essex, self.precise_folsom, self.precise_grizzly,
self.precise_havana, self.precise_icehouse,
self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
self.trusty_kilo, self.vivid_kilo, self.trusty_liberty,
self.wily_liberty, self.trusty_mitaka,
self.xenial_mitaka, self.xenial_newton,
self.yakkety_newton) = range(16)
(self.trusty_icehouse, self.trusty_kilo, self.trusty_liberty,
self.trusty_mitaka, self.xenial_mitaka, self.xenial_newton,
self.yakkety_newton, self.xenial_ocata, self.zesty_ocata) = range(9)
releases = {
('precise', None): self.precise_essex,
('precise', 'cloud:precise-folsom'): self.precise_folsom,
('precise', 'cloud:precise-grizzly'): self.precise_grizzly,
('precise', 'cloud:precise-havana'): self.precise_havana,
('precise', 'cloud:precise-icehouse'): self.precise_icehouse,
('trusty', None): self.trusty_icehouse,
('trusty', 'cloud:trusty-juno'): self.trusty_juno,
('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
('trusty', 'cloud:trusty-mitaka'): self.trusty_mitaka,
('utopic', None): self.utopic_juno,
('vivid', None): self.vivid_kilo,
('wily', None): self.wily_liberty,
('xenial', None): self.xenial_mitaka,
('xenial', 'cloud:xenial-newton'): self.xenial_newton,
('xenial', 'cloud:xenial-ocata'): self.xenial_ocata,
('yakkety', None): self.yakkety_newton,
('zesty', None): self.zesty_ocata,
}
return releases[(self.series, self.openstack)]
@ -294,16 +283,10 @@ class OpenStackAmuletDeployment(AmuletDeployment):
Return a string representing the openstack release.
"""
releases = OrderedDict([
('precise', 'essex'),
('quantal', 'folsom'),
('raring', 'grizzly'),
('saucy', 'havana'),
('trusty', 'icehouse'),
('utopic', 'juno'),
('vivid', 'kilo'),
('wily', 'liberty'),
('xenial', 'mitaka'),
('yakkety', 'newton'),
('zesty', 'ocata'),
])
if self.openstack:
os_origin = self.openstack.split(':')[1]

11
tests/charmhelpers/contrib/openstack/amulet/utils.py
View File

@ -28,6 +28,7 @@ import keystoneclient.v2_0 as keystone_client
from keystoneclient.auth.identity import v3 as keystone_id_v3
from keystoneclient import session as keystone_session
from keystoneclient.v3 import client as keystone_client_v3
from novaclient import exceptions
import novaclient.client as nova_client
import pika
@ -377,6 +378,16 @@ class OpenStackAmuletUtils(AmuletUtils):
tenant_name=tenant,
auth_version='2.0')
def create_flavor(self, nova, name, ram, vcpus, disk, flavorid="auto",
ephemeral=0, swap=0, rxtx_factor=1.0, is_public=True):
"""Create the specified flavor."""
try:
nova.flavors.find(name=name)
except (exceptions.NotFound, exceptions.NoUniqueMatch):
self.log.debug('Creating flavor ({})'.format(name))
nova.flavors.create(name, ram, vcpus, disk, flavorid,
ephemeral, swap, rxtx_factor, is_public)
def create_cirros_image(self, glance, image_name):
"""Download the latest cirros image and upload it to glance,
validate and return a resource pointer.

2
tests/gate-basic-trusty-icehouse
View File

@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/env python
#
# Copyright 2016 Canonical Ltd
#

2
tests/gate-basic-trusty-kilo
View File

@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/env python
#
# Copyright 2016 Canonical Ltd
#

2
tests/gate-basic-trusty-liberty
View File

@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/env python
#
# Copyright 2016 Canonical Ltd
#

2
tests/gate-basic-trusty-mitaka
View File

@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/env python
#
# Copyright 2016 Canonical Ltd
#

2
tests/gate-basic-xenial-mitaka
View File

@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/env python
#
# Copyright 2016 Canonical Ltd
#

2
tests/gate-basic-xenial-newton
View File

@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/env python
#
# Copyright 2016 Canonical Ltd
#

25
tests/gate-basic-xenial-ocata Normal file
View File

@ -0,0 +1,25 @@
#!/usr/bin/env python
#
# Copyright 2016 Canonical Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Amulet tests on a basic ceph deployment on xenial-ocata."""
from basic_deployment import CephBasicDeployment
if __name__ == '__main__':
deployment = CephBasicDeployment(series='xenial',
openstack='cloud:xenial-ocata',
source='cloud:xenial-updates/ocata')
deployment.run_tests()

2
tests/gate-basic-yakkety-newton Normal file → Executable file
View File

@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/env python
#
# Copyright 2016 Canonical Ltd
#

23
tests/gate-basic-zesty-ocata Normal file
View File

@ -0,0 +1,23 @@
#!/usr/bin/env python
#
# Copyright 2016 Canonical Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Amulet tests on a basic ceph deployment on zesty-ocata."""
from basic_deployment import CephBasicDeployment
if __name__ == '__main__':
deployment = CephBasicDeployment(series='zesty')
deployment.run_tests()