openstack
/
charm-glance
Code Issues Proposed changes

py3: Switch to using Python 3 for rocky or later 

Switch package install to Python 3 for OpenStack Rocky or later.

When upgrading, remove any python-* packages that where explicitly
installed and then autoremove --purge any dependencies that are
no longer required.

Tidy install on ceph relation joined - python-ceph is a dependency
of ceph-common, so no need to explicitly install.

Change-Id: I7b66bb3ab9f9130c9054411819d51434480cac97
This commit is contained in:
James Page 2018-09-07 15:15:43 +01:00
parent 52c3112a13
commit 6001f58a48
15 changed files with 160 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
charmhelpers/contrib/openstack/amulet/utils.py
View File

@ -24,7 +24,8 @@ import urlparse
import cinderclient.v1.client as cinder_client
import cinderclient.v2.client as cinder_clientv2
import glanceclient.v1.client as glance_client
import glanceclient.v1 as glance_client
import glanceclient.v2 as glance_clientv2
import heatclient.v1.client as heat_client
from keystoneclient.v2_0 import client as keystone_client
from keystoneauth1.identity import (
@ -623,7 +624,7 @@ class OpenStackAmuletUtils(AmuletUtils):
ep = keystone.service_catalog.url_for(service_type='image',
interface='adminURL')
if keystone.session:
return glance_client.Client(ep, session=keystone.session)
return glance_clientv2.Client("2", session=keystone.session)
else:
return glance_client.Client(ep, token=keystone.auth_token)
@ -711,10 +712,19 @@ class OpenStackAmuletUtils(AmuletUtils):
f.close()
# Create glance image
with open(local_path) as f:
image = glance.images.create(name=image_name, is_public=True,
disk_format='qcow2',
container_format='bare', data=f)
if float(glance.version) < 2.0:
with open(local_path) as fimage:
image = glance.images.create(name=image_name, is_public=True,
disk_format='qcow2',
container_format='bare',
data=fimage)
else:
image = glance.images.create(
name=image_name,
disk_format="qcow2",
visibility="public",
container_format="bare")
glance.images.upload(image.id, open(local_path, 'rb'))
# Wait for image to reach active status
img_id = image.id
@ -729,9 +739,14 @@ class OpenStackAmuletUtils(AmuletUtils):
self.log.debug('Validating image attributes...')
val_img_name = glance.images.get(img_id).name
val_img_stat = glance.images.get(img_id).status
val_img_pub = glance.images.get(img_id).is_public
val_img_cfmt = glance.images.get(img_id).container_format
val_img_dfmt = glance.images.get(img_id).disk_format
if float(glance.version) < 2.0:
val_img_pub = glance.images.get(img_id).is_public
else:
val_img_pub = glance.images.get(img_id).visibility == "public"
msg_attr = ('Image attributes - name:{} public:{} id:{} stat:{} '
'container fmt:{} disk fmt:{}'.format(
val_img_name, val_img_pub, img_id,
@ -998,6 +1013,9 @@ class OpenStackAmuletUtils(AmuletUtils):
cmd, code, output))
amulet.raise_status(amulet.FAIL, msg=msg)
# For mimic ceph osd lspools output
output = output.replace("\n", ",")
# Example output: 0 data,1 metadata,2 rbd,3 cinder,4 glance,
for pool in str(output).split(','):
pool_id_name = pool.split(' ')

8
charmhelpers/contrib/openstack/context.py
View File

@ -1519,6 +1519,14 @@ class NeutronAPIContext(OSContextGenerator):
'rel_key': 'enable-qos',
'default': False,
},
'enable_nsg_logging': {
'rel_key': 'enable-nsg-logging',
'default': False,
},
'nsg_log_output_base': {
'rel_key': 'nsg-log-output-base',
'default': None,
},
}
ctxt = self.get_neutron_options({})
for rid in relation_ids('neutron-plugin-api'):

14
charmhelpers/contrib/openstack/utils.py
View File

@ -1736,7 +1736,12 @@ def is_unit_upgrading_set():
def series_upgrade_prepare(pause_unit_helper=None, configs=None):
""" Run common series upgrade prepare tasks."""
""" Run common series upgrade prepare tasks.
:param pause_unit_helper: function: Function to pause unit
:param configs: OSConfigRenderer object: Configurations
:returns None:
"""
set_unit_upgrading()
if pause_unit_helper and configs:
if not is_unit_paused_set():
@ -1744,7 +1749,12 @@ def series_upgrade_prepare(pause_unit_helper=None, configs=None):
def series_upgrade_complete(resume_unit_helper=None, configs=None):
""" Run common series upgrade complete tasks."""
""" Run common series upgrade complete tasks.
:param resume_unit_helper: function: Function to resume unit
:param configs: OSConfigRenderer object: Configurations
:returns None:
"""
clear_unit_paused()
clear_unit_upgrading()
if configs:

3
charmhelpers/core/hookenv.py
View File

@ -48,6 +48,7 @@ INFO = "INFO"
DEBUG = "DEBUG"
TRACE = "TRACE"
MARKER = object()
SH_MAX_ARG = 131071
cache = {}
@ -98,7 +99,7 @@ def log(message, level=None):
command += ['-l', level]
if not isinstance(message, six.string_types):
message = repr(message)
command += [message]
command += [message[:SH_MAX_ARG]]
# Missing juju-log should not cause failures in unit tests
# Send log output to stderr
try:

2
charmhelpers/fetch/__init__.py
View File

@ -84,6 +84,7 @@ module = "charmhelpers.fetch.%s" % __platform__
fetch = importlib.import_module(module)
filter_installed_packages = fetch.filter_installed_packages
filter_missing_packages = fetch.filter_missing_packages
install = fetch.apt_install
upgrade = fetch.apt_upgrade
update = _fetch_update = fetch.apt_update
@ -96,6 +97,7 @@ if __platform__ == "ubuntu":
apt_update = fetch.apt_update
apt_upgrade = fetch.apt_upgrade
apt_purge = fetch.apt_purge
apt_autoremove = fetch.apt_autoremove
apt_mark = fetch.apt_mark
apt_hold = fetch.apt_hold
apt_unhold = fetch.apt_unhold

4
charmhelpers/fetch/bzrurl.py
View File

@ -13,7 +13,7 @@
# limitations under the License.
import os
from subprocess import check_call
from subprocess import STDOUT, check_output
from charmhelpers.fetch import (
BaseFetchHandler,
UnhandledSource,
@ -55,7 +55,7 @@ class BzrUrlFetchHandler(BaseFetchHandler):
cmd = ['bzr', 'branch']
cmd += cmd_opts
cmd += [source, dest]
check_call(cmd)
check_output(cmd, stderr=STDOUT)
def install(self, source, dest=None, revno=None):
url_parts = self.parse_url(source)

4
charmhelpers/fetch/giturl.py
View File

@ -13,7 +13,7 @@
# limitations under the License.
import os
from subprocess import check_call, CalledProcessError
from subprocess import check_output, CalledProcessError, STDOUT
from charmhelpers.fetch import (
BaseFetchHandler,
UnhandledSource,
@ -50,7 +50,7 @@ class GitUrlFetchHandler(BaseFetchHandler):
cmd = ['git', 'clone', source, dest, '--branch', branch]
if depth:
cmd.extend(['--depth', depth])
check_call(cmd)
check_output(cmd, stderr=STDOUT)
def install(self, source, branch="master", dest=None, depth=None):
url_parts = self.parse_url(source)

20
charmhelpers/fetch/ubuntu.py
View File

@ -189,6 +189,18 @@ def filter_installed_packages(packages):
return _pkgs
def filter_missing_packages(packages):
"""Return a list of packages that are installed.
:param packages: list of packages to evaluate.
:returns list: Packages that are installed.
"""
return list(
set(packages) -
set(filter_installed_packages(packages))
)
def apt_cache(in_memory=True, progress=None):
"""Build and return an apt cache."""
from apt import apt_pkg
@ -248,6 +260,14 @@ def apt_purge(packages, fatal=False):
_run_apt_command(cmd, fatal)
def apt_autoremove(purge=True, fatal=False):
"""Purge one or more packages."""
cmd = ['apt-get', '--assume-yes', 'autoremove']
if purge:
cmd.append('--purge')
_run_apt_command(cmd, fatal)
def apt_mark(packages, mark, fatal=False):
"""Flag one or more packages using apt-mark."""
log("Marking {} as {}".format(packages, mark))

8
hooks/glance_relations.py
View File

@ -105,6 +105,7 @@ from charmhelpers.contrib.openstack.utils import (
os_requires_version,
series_upgrade_prepare,
series_upgrade_complete,
CompareOpenStackReleases,
)
from charmhelpers.contrib.storage.linux.ceph import (
send_request_if_needed,
@ -263,7 +264,7 @@ def object_store_joined():
@hooks.hook('ceph-relation-joined')
def ceph_joined():
apt_install(['ceph-common', 'python-ceph'])
apt_install(['ceph-common'])
def get_ceph_request():
@ -586,7 +587,10 @@ def install_packages_for_cinder_store():
optional_packages = ["python-cinderclient",
"python-os-brick",
"python-oslo.rootwrap"]
apt_install(filter_installed_packages(optional_packages), fatal=True)
release = os_release('glance-common')
cmp_release = CompareOpenStackReleases(release)
if cmp_release < 'rocky':
apt_install(filter_installed_packages(optional_packages), fatal=True)
@hooks.hook('cinder-volume-service-relation-joined')

40
hooks/glance_utils.py
View File

@ -27,7 +27,10 @@ from charmhelpers.fetch import (
apt_upgrade,
apt_update,
apt_install,
add_source)
add_source,
apt_autoremove,
apt_purge,
filter_missing_packages)
from charmhelpers.core.hookenv import (
config,
@ -87,6 +90,16 @@ PACKAGES = [
"apache2", "glance", "python-mysqldb", "python-swiftclient",
"python-psycopg2", "python-keystone", "python-six", "uuid", "haproxy", ]
PY3_PACKAGES = [
"python3-glance",
"python3-rados",
"python3-rbd",
"python3-swiftclient",
"python3-cinderclient",
"python3-os-brick",
"python3-oslo.rootwrap",
]
VERSION_PACKAGE = 'glance-common'
SERVICES = [
@ -247,9 +260,29 @@ def register_configs():
def determine_packages():
packages = set(PACKAGES)
packages |= set(token_cache_pkgs(source=config('openstack-origin')))
if CompareOpenStackReleases(os_release(VERSION_PACKAGE)) >= 'rocky':
packages = [p for p in packages if not p.startswith('python-')]
packages.extend(PY3_PACKAGES)
return sorted(packages)
def determine_purge_packages():
'''
Determine list of packages that where previously installed which are no
longer needed.
:returns: list of package names
'''
if CompareOpenStackReleases(os_release('cinder')) >= 'rocky':
pkgs = [p for p in PACKAGES if p.startswith('python-')]
pkgs.append('python-glance')
pkgs.append('python-memcache')
pkgs.extend(["python-cinderclient",
"python-os-brick",
"python-oslo.rootwrap"])
return []
# NOTE(jamespage): Retry deals with sync issues during one-shot HA deploys.
# mysql might be restarting or suchlike.
@retry_on_exception(5, base_delay=3, exc_type=subprocess.CalledProcessError)
@ -284,6 +317,11 @@ def do_openstack_upgrade(configs):
reset_os_release()
apt_install(determine_packages(), fatal=True)
installed_packages = filter_missing_packages(determine_purge_packages())
if installed_packages:
apt_purge(installed_packages, fatal=True)
apt_autoremove(purge=True, fatal=True)
# set CONFIGS to load templates from new release and regenerate config
configs.set_release(openstack_release=new_os_rel)
configs.write_all()

3
tests/charmhelpers/contrib/openstack/amulet/utils.py
View File

@ -1013,6 +1013,9 @@ class OpenStackAmuletUtils(AmuletUtils):
cmd, code, output))
amulet.raise_status(amulet.FAIL, msg=msg)
# For mimic ceph osd lspools output
output = output.replace("\n", ",")
# Example output: 0 data,1 metadata,2 rbd,3 cinder,4 glance,
for pool in str(output).split(','):
pool_id_name = pool.split(' ')

3
tests/charmhelpers/core/hookenv.py
View File

@ -48,6 +48,7 @@ INFO = "INFO"
DEBUG = "DEBUG"
TRACE = "TRACE"
MARKER = object()
SH_MAX_ARG = 131071
cache = {}
@ -98,7 +99,7 @@ def log(message, level=None):
command += ['-l', level]
if not isinstance(message, six.string_types):
message = repr(message)
command += [message]
command += [message[:SH_MAX_ARG]]
# Missing juju-log should not cause failures in unit tests
# Send log output to stderr
try:

2
tox.ini
View File

@ -67,7 +67,7 @@ basepython = python2.7
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
commands =
bundletester -vl DEBUG -r json -o func-results.json gate-basic-bionic-queens --no-destroy
bundletester -vl DEBUG -r json -o func-results.json gate-basic-bionic-rocky --no-destroy
[testenv:func27-dfs]
# Charm Functional Test

11
unit_tests/test_glance_relations.py
View File

@ -112,6 +112,7 @@ class GlanceRelationTests(CharmTestCase):
super(GlanceRelationTests, self).setUp(relations, TO_PATCH)
self.config.side_effect = self.test_config.get
self.restart_on_change.return_value = None
self.os_release.return_value = 'icehouse'
def test_install_hook(self):
repo = 'cloud:precise-grizzly'
@ -297,7 +298,7 @@ class GlanceRelationTests(CharmTestCase):
def test_ceph_joined(self):
relations.ceph_joined()
self.apt_install.assert_called_with(['ceph-common', 'python-ceph'])
self.apt_install.assert_called_with(['ceph-common'])
@patch.object(relations, 'CONFIGS')
def test_ceph_changed_missing_relation_data(self, configs):
@ -815,6 +816,14 @@ class GlanceRelationTests(CharmTestCase):
"python-oslo.rootwrap"], fatal=True
)
@patch.object(relations, 'CONFIGS')
def test_cinder_volume_joined_rocky(self, configs):
self.filter_installed_packages.side_effect = lambda pkgs: pkgs
self.os_release.return_value = 'rocky'
relations.cinder_volume_service_relation_joined()
self.assertTrue(configs.write_all.called)
self.apt_install.assert_not_called()
@patch.object(relations, 'CONFIGS')
def test_storage_backend_changed(self, configs):
self.filter_installed_packages.side_effect = lambda pkgs: pkgs

26
unit_tests/test_glance_utils.py
View File

@ -36,6 +36,9 @@ TO_PATCH = [
'apt_update',
'apt_upgrade',
'apt_install',
'apt_purge',
'apt_autoremove',
'filter_missing_packages',
'mkdir',
'os_release',
'service_start',
@ -161,6 +164,7 @@ class TestGlanceUtils(CharmTestCase):
@patch.object(utils, 'token_cache_pkgs')
def test_determine_packages(self, token_cache_pkgs):
self.config.side_effect = None
self.os_release.return_value = 'queens'
token_cache_pkgs.return_value = []
ex = utils.PACKAGES
self.assertEqual(set(ex), set(utils.determine_packages()))
@ -172,6 +176,7 @@ class TestGlanceUtils(CharmTestCase):
def test_openstack_upgrade_leader(self, migrate):
self.config.side_effect = None
self.config.return_value = 'cloud:precise-havana'
self.os_release.return_value = 'havana'
self.is_elected_leader.return_value = True
self.get_os_codename_install_source.return_value = 'havana'
configs = MagicMock()
@ -184,10 +189,31 @@ class TestGlanceUtils(CharmTestCase):
configs.set_release.assert_called_with(openstack_release='havana')
self.assertTrue(migrate.called)
@patch.object(utils, 'migrate_database')
def test_openstack_upgrade_rocky(self, migrate):
self.config.side_effect = None
self.config.return_value = 'cloud:bionic-rocky'
self.os_release.return_value = 'rocky'
self.is_elected_leader.return_value = True
self.get_os_codename_install_source.return_value = 'rocky'
self.filter_missing_packages.return_value = ['python-glance']
configs = MagicMock()
utils.do_openstack_upgrade(configs)
self.assertTrue(configs.write_all.called)
self.apt_install.assert_called_with(utils.determine_packages(),
fatal=True)
self.apt_upgrade.assert_called_with(options=DPKG_OPTS,
fatal=True, dist=True)
self.apt_purge.assert_called_with(['python-glance'], fatal=True)
self.apt_autoremove.assert_called_with(purge=True, fatal=True)
configs.set_release.assert_called_with(openstack_release='rocky')
self.assertTrue(migrate.called)
@patch.object(utils, 'migrate_database')
def test_openstack_upgrade_not_leader(self, migrate):
self.config.side_effect = None
self.config.return_value = 'cloud:precise-havana'
self.os_release.return_value = 'havana'
self.is_elected_leader.return_value = False
self.get_os_codename_install_source.return_value = 'havana'
configs = MagicMock()