From 05f5c6e7f0438d181446c0adab872abb751424a5 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Tue, 12 Jan 2021 15:24:12 +0000 Subject: [PATCH] Updates for testing period for 20.01 release Includes updates to charmhelpers/charms.openstack for cert_utils and unit-get for the install hook error on Juju 2.9 * charm-helpers sync for classic charms * rebuild for reactive charms * ensure tox.ini is from release-tools * ensure requirements.txt files are from release-tools * On reactive charms: - ensure master branch for charms.openstack - ensure master branch for charm-helpers Change-Id: I40621954e3108f73b79257755aa518646e52c88f --- charmhelpers/contrib/charmsupport/nrpe.py | 16 ++++++- .../contrib/hardening/audits/apache.py | 5 ++ charmhelpers/contrib/openstack/ip.py | 21 ++++++++- charmhelpers/contrib/openstack/utils.py | 30 ++++++++++++ charmhelpers/fetch/ubuntu.py | 2 +- charmhelpers/fetch/ubuntu_apt_pkg.py | 47 ++++++++++++++++++- test-requirements.txt | 2 + 7 files changed, 119 insertions(+), 4 deletions(-) diff --git a/charmhelpers/contrib/charmsupport/nrpe.py b/charmhelpers/contrib/charmsupport/nrpe.py index 14b80d9..c87cf48 100644 --- a/charmhelpers/contrib/charmsupport/nrpe.py +++ b/charmhelpers/contrib/charmsupport/nrpe.py @@ -139,10 +139,11 @@ define service {{ """{description} check_command check_nrpe!{command} servicegroups {nagios_servicegroup} +{service_config_overrides} }} """) - def __init__(self, shortname, description, check_cmd): + def __init__(self, shortname, description, check_cmd, max_check_attempts=None): super(Check, self).__init__() # XXX: could be better to calculate this from the service name if not re.match(self.shortname_re, shortname): @@ -155,6 +156,7 @@ define service {{ # The default is: illegal_object_name_chars=`~!$%^&*"|'<>?,()= self.description = description self.check_cmd = self._locate_cmd(check_cmd) + self.max_check_attempts = max_check_attempts def _get_check_filename(self): return os.path.join(NRPE.nrpe_confdir, '{}.cfg'.format(self.command)) @@ -216,12 +218,19 @@ define service {{ nagios_servicegroups): self._remove_service_files() + if self.max_check_attempts: + service_config_overrides = ' max_check_attempts {}'.format( + self.max_check_attempts + ) # Note indentation is here rather than in the template to avoid trailing spaces + else: + service_config_overrides = '' # empty string to avoid printing 'None' templ_vars = { 'nagios_hostname': hostname, 'nagios_servicegroup': nagios_servicegroups, 'description': self.description, 'shortname': self.shortname, 'command': self.command, + 'service_config_overrides': service_config_overrides, } nrpe_service_text = Check.service_template.format(**templ_vars) nrpe_service_file = self._get_service_filename(hostname) @@ -327,6 +336,11 @@ class NRPE(object): nrpe_monitors[nrpecheck.shortname] = { "command": nrpecheck.command, } + # If we were passed max_check_attempts, add that to the relation data + try: + nrpe_monitors[nrpecheck.shortname]['max_check_attempts'] = nrpecheck.max_check_attempts + except AttributeError: + pass # update-status hooks are configured to firing every 5 minutes by # default. When nagios-nrpe-server is restarted, the nagios server diff --git a/charmhelpers/contrib/hardening/audits/apache.py b/charmhelpers/contrib/hardening/audits/apache.py index 04825f5..c153762 100644 --- a/charmhelpers/contrib/hardening/audits/apache.py +++ b/charmhelpers/contrib/hardening/audits/apache.py @@ -98,3 +98,8 @@ class DisabledModuleAudit(BaseAudit): def _restart_apache(): """Restarts the apache process""" subprocess.check_output(['service', 'apache2', 'restart']) + + @staticmethod + def is_ssl_enabled(): + """Check if SSL module is enabled or not""" + return 'ssl' in DisabledModuleAudit._get_loaded_modules() diff --git a/charmhelpers/contrib/openstack/ip.py b/charmhelpers/contrib/openstack/ip.py index 89cf276..6557330 100644 --- a/charmhelpers/contrib/openstack/ip.py +++ b/charmhelpers/contrib/openstack/ip.py @@ -123,6 +123,25 @@ def _get_address_override(endpoint_type=PUBLIC): return addr_override.format(service_name=service_name()) +def local_address(unit_get_fallback='public-address'): + """Return a network address for this unit. + + Attempt to retrieve a 'default' IP address for this unit + from network-get. If this is running with an old version of Juju then + fallback to unit_get. + + :param unit_get_fallback: Either 'public-address' or 'private-address'. + Only used with old versions of Juju. + :type unit_get_fallback: str + :returns: IP Address + :rtype: str + """ + try: + return network_get_primary_address('juju-info') + except NotImplementedError: + return unit_get(unit_get_fallback) + + def resolve_address(endpoint_type=PUBLIC, override=True): """Return unit address depending on net config. @@ -176,7 +195,7 @@ def resolve_address(endpoint_type=PUBLIC, override=True): if config('prefer-ipv6'): fallback_addr = get_ipv6_addr(exc_list=vips)[0] else: - fallback_addr = unit_get(net_fallback) + fallback_addr = local_address(unit_get_fallback=net_fallback) if net_addr: resolved_address = get_address_in_network(net_addr, fallback_addr) diff --git a/charmhelpers/contrib/openstack/utils.py b/charmhelpers/contrib/openstack/utils.py index f4c7621..f27aa6c 100644 --- a/charmhelpers/contrib/openstack/utils.py +++ b/charmhelpers/contrib/openstack/utils.py @@ -90,13 +90,16 @@ from charmhelpers.core.host import ( service_start, restart_on_change_helper, ) + from charmhelpers.fetch import ( apt_cache, + apt_install, import_key as fetch_import_key, add_source as fetch_add_source, SourceConfigError, GPGKeyError, get_upstream_version, + filter_installed_packages, filter_missing_packages, ubuntu_apt_pkg as apt, ) @@ -480,9 +483,14 @@ def get_swift_codename(version): return None +@deprecate("moved to charmhelpers.contrib.openstack.utils.get_installed_os_version()", "2021-01", log=juju_log) def get_os_codename_package(package, fatal=True): '''Derive OpenStack release codename from an installed package.''' + codename = get_installed_os_version() + if codename: + return codename + if snap_install_requested(): cmd = ['snap', 'list', package] try: @@ -570,6 +578,28 @@ def get_os_version_package(pkg, fatal=True): # error_out(e) +def get_installed_os_version(): + apt_install(filter_installed_packages(['openstack-release']), fatal=False) + print("OpenStack Release: {}".format(openstack_release())) + return openstack_release().get('OPENSTACK_CODENAME') + + +@cached +def openstack_release(): + """Return /etc/os-release in a dict.""" + d = {} + try: + with open('/etc/openstack-release', 'r') as lsb: + for l in lsb: + s = l.split('=') + if len(s) != 2: + continue + d[s[0].strip()] = s[1].strip() + except FileNotFoundError: + pass + return d + + # Module local cache variable for the os_release. _os_rel = None diff --git a/charmhelpers/fetch/ubuntu.py b/charmhelpers/fetch/ubuntu.py index 3315284..b595301 100644 --- a/charmhelpers/fetch/ubuntu.py +++ b/charmhelpers/fetch/ubuntu.py @@ -646,7 +646,7 @@ def _add_apt_repository(spec): # passed as environment variables (See lp:1433761). This is not the case # LTS and non-LTS releases below bionic. _run_with_retries(['add-apt-repository', '--yes', spec], - cmd_env=env_proxy_settings(['https'])) + cmd_env=env_proxy_settings(['https', 'http'])) def _add_cloud_pocket(pocket): diff --git a/charmhelpers/fetch/ubuntu_apt_pkg.py b/charmhelpers/fetch/ubuntu_apt_pkg.py index 929a75d..a2fbe0e 100644 --- a/charmhelpers/fetch/ubuntu_apt_pkg.py +++ b/charmhelpers/fetch/ubuntu_apt_pkg.py @@ -129,7 +129,7 @@ class Cache(object): else: data = line.split(None, 4) status = data.pop(0) - if status != 'ii': + if status not in ('ii', 'hi'): continue pkg = {} pkg.update({k.lower(): v for k, v in zip(headings, data)}) @@ -265,3 +265,48 @@ def version_compare(a, b): raise RuntimeError('Unable to compare "{}" and "{}", according to ' 'our logic they are neither greater, equal nor ' 'less than each other.') + + +class PkgVersion(): + """Allow package versions to be compared. + + For example:: + + >>> import charmhelpers.fetch as fetch + >>> (fetch.apt_pkg.PkgVersion('2:20.4.0') < + ... fetch.apt_pkg.PkgVersion('2:20.5.0')) + True + >>> pkgs = [fetch.apt_pkg.PkgVersion('2:20.4.0'), + ... fetch.apt_pkg.PkgVersion('2:21.4.0'), + ... fetch.apt_pkg.PkgVersion('2:17.4.0')] + >>> pkgs.sort() + >>> pkgs + [2:17.4.0, 2:20.4.0, 2:21.4.0] + """ + + def __init__(self, version): + self.version = version + + def __lt__(self, other): + return version_compare(self.version, other.version) == -1 + + def __le__(self, other): + return self.__lt__(other) or self.__eq__(other) + + def __gt__(self, other): + return version_compare(self.version, other.version) == 1 + + def __ge__(self, other): + return self.__gt__(other) or self.__eq__(other) + + def __eq__(self, other): + return version_compare(self.version, other.version) == 0 + + def __ne__(self, other): + return not self.__eq__(other) + + def __repr__(self): + return self.version + + def __hash__(self): + return hash(repr(self)) diff --git a/test-requirements.txt b/test-requirements.txt index 1aa9635..9aea716 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -37,6 +37,8 @@ importlib-resources<3.0.0; python_version < '3.6' # dropped support for python 3.5: osprofiler<2.7.0;python_version<'3.6' stevedore<1.31.0;python_version<'3.6' +debtcollector<1.22.0;python_version<'3.6' +oslo.utils<=3.41.0;python_version<'3.6' coverage>=4.5.2 pyudev # for ceph-* charm unit tests (need to fix the ceph-* charm unit tests/mocking)