From 557b63b33e5f5e5cc7d7587a19d3efce58e96a5f Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Sat, 3 Apr 2021 20:18:03 +0100 Subject: [PATCH] 21.04 libraries freeze for charms on master branch * charm-helpers sync for classic charms * build.lock file for reactive charms * ensure tox.ini is from release-tools * ensure requirements.txt files are from release-tools * On reactive charms: - ensure stable/21.04 branch for charms.openstack - ensure stable/21.04 branch for charm-helpers Change-Id: I14762601bb124cfb03bd3f427fa4b1243ed2377b --- charm-helpers-hooks.yaml | 2 +- .../contrib/openstack/deferred_events.py | 410 ++++++++++++++++++ .../contrib/openstack/exceptions.py | 5 + .../openstack/files/policy_rc_d_script.py | 196 +++++++++ .../contrib/openstack/policy_rcd.py | 173 ++++++++ hooks/charmhelpers/contrib/openstack/utils.py | 188 +++++++- hooks/charmhelpers/core/hookenv.py | 9 + hooks/charmhelpers/core/host.py | 162 +++++-- .../charmhelpers/core/host_factory/ubuntu.py | 12 +- hooks/charmhelpers/fetch/__init__.py | 1 + hooks/charmhelpers/fetch/ubuntu.py | 37 +- test-requirements.txt | 4 +- 12 files changed, 1152 insertions(+), 47 deletions(-) create mode 100644 hooks/charmhelpers/contrib/openstack/deferred_events.py create mode 100755 hooks/charmhelpers/contrib/openstack/files/policy_rc_d_script.py create mode 100644 hooks/charmhelpers/contrib/openstack/policy_rcd.py diff --git a/charm-helpers-hooks.yaml b/charm-helpers-hooks.yaml index 30246ded..326e3058 100644 --- a/charm-helpers-hooks.yaml +++ b/charm-helpers-hooks.yaml @@ -1,4 +1,4 @@ -repo: https://github.com/juju/charm-helpers +repo: https://github.com/juju/charm-helpers@stable/21.04 destination: hooks/charmhelpers include: - core diff --git a/hooks/charmhelpers/contrib/openstack/deferred_events.py b/hooks/charmhelpers/contrib/openstack/deferred_events.py new file mode 100644 index 00000000..fd073a04 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/deferred_events.py @@ -0,0 +1,410 @@ +# Copyright 2021 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module for managing deferred service events. + +This module is used to manage deferred service events from both charm actions +and package actions. +""" + +import datetime +import glob +import yaml +import os +import time +import uuid + +import charmhelpers.contrib.openstack.policy_rcd as policy_rcd +import charmhelpers.core.hookenv as hookenv +import charmhelpers.core.host as host +import charmhelpers.core.unitdata as unitdata + +import subprocess + + +# Deferred events generated from the charm are stored along side those +# generated from packaging. +DEFERRED_EVENTS_DIR = policy_rcd.POLICY_DEFERRED_EVENTS_DIR + + +class ServiceEvent(): + + def __init__(self, timestamp, service, reason, action, + policy_requestor_name=None, policy_requestor_type=None): + self.timestamp = timestamp + self.service = service + self.reason = reason + self.action = action + if not policy_requestor_name: + self.policy_requestor_name = hookenv.service_name() + if not policy_requestor_type: + self.policy_requestor_type = 'charm' + + def __eq__(self, other): + for attr in vars(self): + if getattr(self, attr) != getattr(other, attr): + return False + return True + + def matching_request(self, other): + for attr in ['service', 'action', 'reason']: + if getattr(self, attr) != getattr(other, attr): + return False + return True + + @classmethod + def from_dict(cls, data): + return cls( + data['timestamp'], + data['service'], + data['reason'], + data['action'], + data.get('policy_requestor_name'), + data.get('policy_requestor_type')) + + +def deferred_events_files(): + """Deferred event files + + Deferred event files that were generated by service_name() policy. + + :returns: Deferred event files + :rtype: List[str] + """ + return glob.glob('{}/*.deferred'.format(DEFERRED_EVENTS_DIR)) + + +def read_event_file(file_name): + """Read a file and return the corresponding objects. + + :param file_name: Name of file to read. + :type file_name: str + :returns: ServiceEvent from file. + :rtype: ServiceEvent + """ + with open(file_name, 'r') as f: + contents = yaml.safe_load(f) + event = ServiceEvent( + contents['timestamp'], + contents['service'], + contents['reason'], + contents['action']) + return event + + +def deferred_events(): + """Get list of deferred events. + + List of deferred events. Events are represented by dicts of the form: + + { + action: restart, + policy_requestor_name: neutron-openvswitch, + policy_requestor_type: charm, + reason: 'Pkg update', + service: openvswitch-switch, + time: 1614328743} + + :returns: List of deferred events. + :rtype: List[ServiceEvent] + """ + events = [] + for defer_file in deferred_events_files(): + events.append((defer_file, read_event_file(defer_file))) + return events + + +def duplicate_event_files(event): + """Get list of event files that have equivalent deferred events. + + :param event: Event to compare + :type event: ServiceEvent + :returns: List of event files + :rtype: List[str] + """ + duplicates = [] + for event_file, existing_event in deferred_events(): + if event.matching_request(existing_event): + duplicates.append(event_file) + return duplicates + + +def get_event_record_file(policy_requestor_type, policy_requestor_name): + """Generate filename for storing a new event. + + :param policy_requestor_type: System that blocked event + :type policy_requestor_type: str + :param policy_requestor_name: Name of application that blocked event + :type policy_requestor_name: str + :returns: File name + :rtype: str + """ + file_name = '{}/{}-{}-{}.deferred'.format( + DEFERRED_EVENTS_DIR, + policy_requestor_type, + policy_requestor_name, + uuid.uuid1()) + return file_name + + +def save_event(event): + """Write deferred events to backend. + + :param event: Event to save + :type event: ServiceEvent + """ + requestor_name = hookenv.service_name() + requestor_type = 'charm' + init_policy_log_dir() + if duplicate_event_files(event): + hookenv.log( + "Not writing new event, existing event found. {} {} {}".format( + event.service, + event.action, + event.reason), + level="DEBUG") + else: + record_file = get_event_record_file( + policy_requestor_type=requestor_type, + policy_requestor_name=requestor_name) + + with open(record_file, 'w') as f: + data = { + 'timestamp': event.timestamp, + 'service': event.service, + 'action': event.action, + 'reason': event.reason, + 'policy_requestor_type': requestor_type, + 'policy_requestor_name': requestor_name} + yaml.dump(data, f) + + +def clear_deferred_events(svcs, action): + """Remove any outstanding deferred events. + + Remove a deferred event if its service is in the services list and its + action matches. + + :param svcs: List of services to remove. + :type svcs: List[str] + :param action: Action to remove + :type action: str + """ + # XXX This function is not currently processing the action. It needs to + # match the action and also take account of try-restart and the + # equivalnce of stop-start and restart. + for defer_file in deferred_events_files(): + deferred_event = read_event_file(defer_file) + if deferred_event.service in svcs: + os.remove(defer_file) + + +def init_policy_log_dir(): + """Ensure directory to store events exists.""" + if not os.path.exists(DEFERRED_EVENTS_DIR): + os.mkdir(DEFERRED_EVENTS_DIR) + + +def get_deferred_events(): + """Return a list of deferred events requested by the charm and packages. + + :returns: List of deferred events + :rtype: List[ServiceEvent] + """ + events = [] + for _, event in deferred_events(): + events.append(event) + return events + + +def get_deferred_restarts(): + """List of deferred restart events requested by the charm and packages. + + :returns: List of deferred restarts + :rtype: List[ServiceEvent] + """ + return [e for e in get_deferred_events() if e.action == 'restart'] + + +def clear_deferred_restarts(services): + """Clear deferred restart events targetted at `services`. + + :param services: Services with deferred actions to clear. + :type services: List[str] + """ + clear_deferred_events(services, 'restart') + + +def process_svc_restart(service): + """Respond to a service restart having occured. + + :param service: Services that the action was performed against. + :type service: str + """ + clear_deferred_restarts([service]) + + +def is_restart_permitted(): + """Check whether restarts are permitted. + + :returns: Whether restarts are permitted + :rtype: bool + """ + if hookenv.config('enable-auto-restarts') is None: + return True + return hookenv.config('enable-auto-restarts') + + +def check_and_record_restart_request(service, changed_files): + """Check if restarts are permitted, if they are not log the request. + + :param service: Service to be restarted + :type service: str + :param changed_files: Files that have changed to trigger restarts. + :type changed_files: List[str] + :returns: Whether restarts are permitted + :rtype: bool + """ + changed_files = sorted(list(set(changed_files))) + permitted = is_restart_permitted() + if not permitted: + save_event(ServiceEvent( + timestamp=round(time.time()), + service=service, + reason='File(s) changed: {}'.format( + ', '.join(changed_files)), + action='restart')) + return permitted + + +def deferrable_svc_restart(service, reason=None): + """Restarts service if permitted, if not defer it. + + :param service: Service to be restarted + :type service: str + :param reason: Reason for restart + :type reason: Union[str, None] + """ + if is_restart_permitted(): + host.service_restart(service) + else: + save_event(ServiceEvent( + timestamp=round(time.time()), + service=service, + reason=reason, + action='restart')) + + +def configure_deferred_restarts(services): + """Setup deferred restarts. + + :param services: Services to block restarts of. + :type services: List[str] + """ + policy_rcd.install_policy_rcd() + if is_restart_permitted(): + policy_rcd.remove_policy_file() + else: + blocked_actions = ['stop', 'restart', 'try-restart'] + for svc in services: + policy_rcd.add_policy_block(svc, blocked_actions) + + +def get_service_start_time(service): + """Find point in time when the systemd unit transitioned to active state. + + :param service: Services to check timetsamp of. + :type service: str + """ + start_time = None + out = subprocess.check_output( + [ + 'systemctl', + 'show', + service, + '--property=ActiveEnterTimestamp']) + str_time = out.decode().rstrip().replace('ActiveEnterTimestamp=', '') + if str_time: + start_time = datetime.datetime.strptime( + str_time, + '%a %Y-%m-%d %H:%M:%S %Z') + return start_time + + +def check_restart_timestamps(): + """Check deferred restarts against systemd units start time. + + Check if a service has a deferred event and clear it if it has been + subsequently restarted. + """ + for event in get_deferred_restarts(): + start_time = get_service_start_time(event.service) + deferred_restart_time = datetime.datetime.fromtimestamp( + event.timestamp) + if start_time and start_time < deferred_restart_time: + hookenv.log( + ("Restart still required, {} was started at {}, restart was " + "requested after that at {}").format( + event.service, + start_time, + deferred_restart_time), + level='DEBUG') + else: + clear_deferred_restarts([event.service]) + + +def set_deferred_hook(hookname): + """Record that a hook has been deferred. + + :param hookname: Name of hook that was deferred. + :type hookname: str + """ + with unitdata.HookData()() as t: + kv = t[0] + deferred_hooks = kv.get('deferred-hooks', []) + if hookname not in deferred_hooks: + deferred_hooks.append(hookname) + kv.set('deferred-hooks', sorted(list(set(deferred_hooks)))) + + +def get_deferred_hooks(): + """Get a list of deferred hooks. + + :returns: List of hook names. + :rtype: List[str] + """ + with unitdata.HookData()() as t: + kv = t[0] + return kv.get('deferred-hooks', []) + + +def clear_deferred_hooks(): + """Clear any deferred hooks.""" + with unitdata.HookData()() as t: + kv = t[0] + kv.set('deferred-hooks', []) + + +def clear_deferred_hook(hookname): + """Clear a specific deferred hooks. + + :param hookname: Name of hook to remove. + :type hookname: str + """ + with unitdata.HookData()() as t: + kv = t[0] + deferred_hooks = kv.get('deferred-hooks', []) + if hookname in deferred_hooks: + deferred_hooks.remove(hookname) + kv.set('deferred-hooks', deferred_hooks) diff --git a/hooks/charmhelpers/contrib/openstack/exceptions.py b/hooks/charmhelpers/contrib/openstack/exceptions.py index f85ae4f4..b2330637 100644 --- a/hooks/charmhelpers/contrib/openstack/exceptions.py +++ b/hooks/charmhelpers/contrib/openstack/exceptions.py @@ -19,3 +19,8 @@ class OSContextError(Exception): This exception is principally used in contrib.openstack.context """ pass + + +class ServiceActionError(Exception): + """Raised when a service action (stop/start/ etc) failed.""" + pass diff --git a/hooks/charmhelpers/contrib/openstack/files/policy_rc_d_script.py b/hooks/charmhelpers/contrib/openstack/files/policy_rc_d_script.py new file mode 100755 index 00000000..344a7662 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/files/policy_rc_d_script.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 + +"""This script is an implemenation of policy-rc.d + +For further information on policy-rc.d see *1 + +*1 https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt +""" +import collections +import glob +import os +import logging +import sys +import time +import uuid +import yaml + + +SystemPolicy = collections.namedtuple( + 'SystemPolicy', + [ + 'policy_requestor_name', + 'policy_requestor_type', + 'service', + 'blocked_actions']) + +DEFAULT_POLICY_CONFIG_DIR = '/etc/policy-rc.d' +DEFAULT_POLICY_LOG_DIR = '/var/lib/policy-rc.d' + + +def read_policy_file(policy_file): + """Return system policies from given file. + + :param file_name: Name of file to read. + :type file_name: str + :returns: Policy + :rtype: List[SystemPolicy] + """ + policies = [] + if os.path.exists(policy_file): + with open(policy_file, 'r') as f: + policy = yaml.safe_load(f) + for service, actions in policy['blocked_actions'].items(): + service = service.replace('.service', '') + policies.append(SystemPolicy( + policy_requestor_name=policy['policy_requestor_name'], + policy_requestor_type=policy['policy_requestor_type'], + service=service, + blocked_actions=actions)) + return policies + + +def get_policies(policy_config_dir): + """Return all system policies in policy_config_dir. + + :param policy_config_dir: Name of file to read. + :type policy_config_dir: str + :returns: Policy + :rtype: List[SystemPolicy] + """ + _policy = [] + for f in glob.glob('{}/*.policy'.format(policy_config_dir)): + _policy.extend(read_policy_file(f)) + return _policy + + +def record_blocked_action(service, action, blocking_policies, policy_log_dir): + """Record that an action was requested but deniedl + + :param service: Service that was blocked + :type service: str + :param action: Action that was blocked. + :type action: str + :param blocking_policies: Policies that blocked the action on the service. + :type blocking_policies: List[SystemPolicy] + :param policy_log_dir: Directory to place the blocking action record. + :type policy_log_dir: str + """ + if not os.path.exists(policy_log_dir): + os.mkdir(policy_log_dir) + seconds = round(time.time()) + for policy in blocking_policies: + if not os.path.exists(policy_log_dir): + os.mkdir(policy_log_dir) + file_name = '{}/{}-{}-{}.deferred'.format( + policy_log_dir, + policy.policy_requestor_type, + policy.policy_requestor_name, + uuid.uuid1()) + with open(file_name, 'w') as f: + data = { + 'timestamp': seconds, + 'service': service, + 'action': action, + 'reason': 'Package update', + 'policy_requestor_type': policy.policy_requestor_type, + 'policy_requestor_name': policy.policy_requestor_name} + yaml.dump(data, f) + + +def get_blocking_policies(service, action, policy_config_dir): + """Record that an action was requested but deniedl + + :param service: Service that action is requested against. + :type service: str + :param action: Action that is requested. + :type action: str + :param policy_config_dir: Directory that stores policy files. + :type policy_config_dir: str + :returns: Policies + :rtype: List[SystemPolicy] + """ + service = service.replace('.service', '') + blocking_policies = [ + policy + for policy in get_policies(policy_config_dir) + if policy.service == service and action in policy.blocked_actions] + return blocking_policies + + +def process_action_request(service, action, policy_config_dir, policy_log_dir): + """Take the requested action against service and check if it is permitted. + + :param service: Service that action is requested against. + :type service: str + :param action: Action that is requested. + :type action: str + :param policy_config_dir: Directory that stores policy files. + :type policy_config_dir: str + :param policy_log_dir: Directory that stores policy files. + :type policy_log_dir: str + :returns: Tuple of whether the action is permitted and explanation. + :rtype: (boolean, str) + """ + blocking_policies = get_blocking_policies( + service, + action, + policy_config_dir) + if blocking_policies: + policy_msg = [ + '{} {}'.format(p.policy_requestor_type, p.policy_requestor_name) + for p in sorted(blocking_policies)] + message = '{} of {} blocked by {}'.format( + action, + service, + ', '.join(policy_msg)) + record_blocked_action( + service, + action, + blocking_policies, + policy_log_dir) + action_permitted = False + else: + message = "Permitting {} {}".format(service, action) + action_permitted = True + return action_permitted, message + + +def main(): + logging.basicConfig( + filename='/var/log/policy-rc.d.log', + level=logging.DEBUG, + format='%(asctime)s %(message)s') + + service = sys.argv[1] + action = sys.argv[2] + + permitted, message = process_action_request( + service, + action, + DEFAULT_POLICY_CONFIG_DIR, + DEFAULT_POLICY_LOG_DIR) + logging.info(message) + + # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt + # Exit status codes: + # 0 - action allowed + # 1 - unknown action (therefore, undefined policy) + # 100 - unknown initscript id + # 101 - action forbidden by policy + # 102 - subsystem error + # 103 - syntax error + # 104 - [reserved] + # 105 - behaviour uncertain, policy undefined. + # 106 - action not allowed. Use the returned fallback actions + # (which are implied to be "allowed") instead. + + if permitted: + return 0 + else: + return 101 + + +if __name__ == "__main__": + rc = main() + sys.exit(rc) diff --git a/hooks/charmhelpers/contrib/openstack/policy_rcd.py b/hooks/charmhelpers/contrib/openstack/policy_rcd.py new file mode 100644 index 00000000..ecffbc68 --- /dev/null +++ b/hooks/charmhelpers/contrib/openstack/policy_rcd.py @@ -0,0 +1,173 @@ +# Copyright 2021 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module for managing policy-rc.d script and associated files. + +This module manages the installation of /usr/sbin/policy-rc.d, the +policy files and the event files. When a package update occurs the +packaging system calls: + +policy-rc.d [options] + +The return code of the script determines if the packaging system +will perform that action on the given service. The policy-rc.d +implementation installed by this module checks if an action is +permitted by checking policy files placed in /etc/policy-rc.d. +If a policy file exists which denies the requested action then +this is recorded in an event file which is placed in +/var/lib/policy-rc.d. +""" + +import os +import shutil +import tempfile +import yaml + +import charmhelpers.contrib.openstack.files as os_files +import charmhelpers.contrib.openstack.alternatives as alternatives +import charmhelpers.core.hookenv as hookenv +import charmhelpers.core.host as host + +POLICY_HEADER = """# Managed by juju\n""" +POLICY_DEFERRED_EVENTS_DIR = '/var/lib/policy-rc.d' +POLICY_CONFIG_DIR = '/etc/policy-rc.d' + + +def get_policy_file_name(): + """Get the name of the policy file for this application. + + :returns: Policy file name + :rtype: str + """ + application_name = hookenv.service_name() + return '{}/charm-{}.policy'.format(POLICY_CONFIG_DIR, application_name) + + +def read_default_policy_file(): + """Return the policy file. + + A policy is in the form: + blocked_actions: + neutron-dhcp-agent: [restart, stop, try-restart] + neutron-l3-agent: [restart, stop, try-restart] + neutron-metadata-agent: [restart, stop, try-restart] + neutron-openvswitch-agent: [restart, stop, try-restart] + openvswitch-switch: [restart, stop, try-restart] + ovs-vswitchd: [restart, stop, try-restart] + ovs-vswitchd-dpdk: [restart, stop, try-restart] + ovsdb-server: [restart, stop, try-restart] + policy_requestor_name: neutron-openvswitch + policy_requestor_type: charm + + :returns: Policy + :rtype: Dict[str, Union[str, Dict[str, List[str]]] + """ + policy = {} + policy_file = get_policy_file_name() + if os.path.exists(policy_file): + with open(policy_file, 'r') as f: + policy = yaml.safe_load(f) + return policy + + +def write_policy_file(policy_file, policy): + """Write policy to disk. + + :param policy_file: Name of policy file + :type policy_file: str + :param policy: Policy + :type policy: Dict[str, Union[str, Dict[str, List[str]]]] + """ + with tempfile.NamedTemporaryFile('w', delete=False) as f: + f.write(POLICY_HEADER) + yaml.dump(policy, f) + tmp_file_name = f.name + shutil.move(tmp_file_name, policy_file) + + +def remove_policy_file(): + """Remove policy file.""" + try: + os.remove(get_policy_file_name()) + except FileNotFoundError: + pass + + +def install_policy_rcd(): + """Install policy-rc.d components.""" + source_file_dir = os.path.dirname(os.path.abspath(os_files.__file__)) + policy_rcd_exec = "/var/lib/charm/{}/policy-rc.d".format( + hookenv.service_name()) + host.mkdir(os.path.dirname(policy_rcd_exec)) + shutil.copy2( + '{}/policy_rc_d_script.py'.format(source_file_dir), + policy_rcd_exec) + # policy-rc.d must be installed via the alternatives system: + # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt + if not os.path.exists('/usr/sbin/policy-rc.d'): + alternatives.install_alternative( + 'policy-rc.d', + '/usr/sbin/policy-rc.d', + policy_rcd_exec) + host.mkdir(POLICY_CONFIG_DIR) + + +def get_default_policy(): + """Return the default policy structure. + + :returns: Policy + :rtype: Dict[str, Union[str, Dict[str, List[str]]] + """ + policy = { + 'policy_requestor_name': hookenv.service_name(), + 'policy_requestor_type': 'charm', + 'blocked_actions': {}} + return policy + + +def add_policy_block(service, blocked_actions): + """Update a policy file with new list of actions. + + :param service: Service name + :type service: str + :param blocked_actions: Action to block + :type blocked_actions: List[str] + """ + policy = read_default_policy_file() or get_default_policy() + policy_file = get_policy_file_name() + if policy['blocked_actions'].get(service): + policy['blocked_actions'][service].extend(blocked_actions) + else: + policy['blocked_actions'][service] = blocked_actions + policy['blocked_actions'][service] = sorted( + list(set(policy['blocked_actions'][service]))) + write_policy_file(policy_file, policy) + + +def remove_policy_block(service, unblocked_actions): + """Remove list of actions from policy file. + + :param service: Service name + :type service: str + :param unblocked_actions: Action to unblock + :type unblocked_actions: List[str] + """ + policy_file = get_policy_file_name() + policy = read_default_policy_file() + for action in unblocked_actions: + try: + policy['blocked_actions'][service].remove(action) + except (KeyError, ValueError): + continue + write_policy_file(policy_file, policy) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index d0bef935..2ad8ab94 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -14,7 +14,7 @@ # Common python helper functions used for OpenStack charms. from collections import OrderedDict, namedtuple -from functools import wraps +from functools import partial, wraps import subprocess import json @@ -36,9 +36,12 @@ from charmhelpers.contrib.network import ip from charmhelpers.core import decorators, unitdata +import charmhelpers.contrib.openstack.deferred_events as deferred_events + from charmhelpers.core.hookenv import ( WORKLOAD_STATES, action_fail, + action_get, action_set, config, expected_peer_units, @@ -112,7 +115,7 @@ from charmhelpers.fetch.snap import ( from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device -from charmhelpers.contrib.openstack.exceptions import OSContextError +from charmhelpers.contrib.openstack.exceptions import OSContextError, ServiceActionError from charmhelpers.contrib.openstack.policyd import ( policyd_status_message_prefix, POLICYD_CONFIG_NAME, @@ -148,6 +151,7 @@ OPENSTACK_RELEASES = ( 'train', 'ussuri', 'victoria', + 'wallaby', ) UBUNTU_OPENSTACK_RELEASE = OrderedDict([ @@ -170,6 +174,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([ ('eoan', 'train'), ('focal', 'ussuri'), ('groovy', 'victoria'), + ('hirsute', 'wallaby'), ]) @@ -193,6 +198,7 @@ OPENSTACK_CODENAMES = OrderedDict([ ('2019.2', 'train'), ('2020.1', 'ussuri'), ('2020.2', 'victoria'), + ('2021.1', 'wallaby'), ]) # The ugly duckling - must list releases oldest to newest @@ -301,8 +307,8 @@ PACKAGE_CODENAMES = { ('14', 'rocky'), ('15', 'stein'), ('16', 'train'), - ('18', 'ussuri'), - ('19', 'victoria'), + ('18', 'ussuri'), # Note this was actually 17.0 - 18.3 + ('19', 'victoria'), # Note this is really 18.6 ]), 'ceilometer-common': OrderedDict([ ('5', 'liberty'), @@ -1083,6 +1089,18 @@ def _determine_os_workload_status( try: if config(POLICYD_CONFIG_NAME): message = "{} {}".format(policyd_status_message_prefix(), message) + deferred_restarts = list(set( + [e.service for e in deferred_events.get_deferred_restarts()])) + if deferred_restarts: + svc_msg = "Services queued for restart: {}".format( + ', '.join(sorted(deferred_restarts))) + message = "{}. {}".format(message, svc_msg) + deferred_hooks = deferred_events.get_deferred_hooks() + if deferred_hooks: + svc_msg = "Hooks skipped due to disabled auto restarts: {}".format( + ', '.join(sorted(deferred_hooks))) + message = "{}. {}".format(message, svc_msg) + except Exception: pass @@ -1567,6 +1585,33 @@ def is_unit_paused_set(): return False +def is_hook_allowed(hookname, check_deferred_restarts=True): + """Check if hook can run. + + :param hookname: Name of hook to check.. + :type hookname: str + :param check_deferred_restarts: Whether to check deferred restarts. + :type check_deferred_restarts: bool + """ + permitted = True + reasons = [] + if is_unit_paused_set(): + reasons.append( + "Unit is pause or upgrading. Skipping {}".format(hookname)) + permitted = False + + if check_deferred_restarts: + if deferred_events.is_restart_permitted(): + permitted = True + deferred_events.clear_deferred_hook(hookname) + else: + if not config().changed('enable-auto-restarts'): + deferred_events.set_deferred_hook(hookname) + reasons.append("auto restarts are disabled") + permitted = False + return permitted, " and ".join(reasons) + + def manage_payload_services(action, services=None, charm_func=None): """Run an action against all services. @@ -1727,6 +1772,43 @@ def resume_unit(assess_status_func, services=None, ports=None, raise Exception("Couldn't resume: {}".format("; ".join(messages))) +def restart_services_action(services=None, when_all_stopped_func=None, + deferred_only=None): + """Manage a service restart request via charm action. + + :param services: Services to be restarted + :type model_name: List[str] + :param when_all_stopped_func: Function to call when all services are + stopped. + :type when_all_stopped_func: Callable[] + :param model_name: Only restart services which have a deferred restart + event. + :type model_name: bool + """ + if services and deferred_only: + raise ValueError( + "services and deferred_only are mutually exclusive") + if deferred_only: + services = list(set( + [a.service for a in deferred_events.get_deferred_restarts()])) + _, messages = manage_payload_services( + 'stop', + services=services, + charm_func=when_all_stopped_func) + if messages: + raise ServiceActionError( + "Error processing service stop request: {}".format( + "; ".join(messages))) + _, messages = manage_payload_services( + 'start', + services=services) + if messages: + raise ServiceActionError( + "Error processing service start request: {}".format( + "; ".join(messages))) + deferred_events.clear_deferred_restarts(services) + + def make_assess_status_func(*args, **kwargs): """Creates an assess_status_func() suitable for handing to pause_unit() and resume_unit(). @@ -2201,6 +2283,23 @@ def container_scoped_relations(): return relations +def container_scoped_relation_get(attribute=None): + """Get relation data from all container scoped relations. + + :param attribute: Name of attribute to get + :type attribute: Optional[str] + :returns: Iterator with relation data + :rtype: Iterator[Optional[any]] + """ + for endpoint_name in container_scoped_relations(): + for rid in relation_ids(endpoint_name): + for unit in related_units(rid): + yield relation_get( + attribute=attribute, + unit=unit, + rid=rid) + + def is_db_ready(use_current_context=False, rel_name=None): """Check remote database is ready to be used. @@ -2497,3 +2596,84 @@ def sequence_status_check_functions(*functions): return state, message return _inner_sequenced_functions + + +SubordinatePackages = namedtuple('SubordinatePackages', ['install', 'purge']) + + +def get_subordinate_release_packages(os_release, package_type='deb'): + """Iterate over subordinate relations and get package information. + + :param os_release: OpenStack release to look for + :type os_release: str + :param package_type: Package type (one of 'deb' or 'snap') + :type package_type: str + :returns: Packages to install and packages to purge or None + :rtype: SubordinatePackages[set,set] + """ + install = set() + purge = set() + + for rdata in container_scoped_relation_get('releases-packages-map'): + rp_map = json.loads(rdata or '{}') + # The map provided by subordinate has OpenStack release name as key. + # Find package information from subordinate matching requested release + # or the most recent release prior to requested release by sorting the + # keys in reverse order. This follows established patterns in our + # charms for templates and reactive charm implementations, i.e. as long + # as nothing has changed the definitions for the prior OpenStack + # release is still valid. + for release in sorted(rp_map.keys(), reverse=True): + if (CompareOpenStackReleases(release) <= os_release and + package_type in rp_map[release]): + for name, container in ( + ('install', install), + ('purge', purge)): + for pkg in rp_map[release][package_type].get(name, []): + container.add(pkg) + break + return SubordinatePackages(install, purge) + + +os_restart_on_change = partial( + pausable_restart_on_change, + can_restart_now_f=deferred_events.check_and_record_restart_request, + post_svc_restart_f=deferred_events.process_svc_restart) + + +def restart_services_action_helper(all_services): + """Helper to run the restart-services action. + + NOTE: all_services is all services that could be restarted but + depending on the action arguments it may be a subset of + these that are actually restarted. + + :param all_services: All services that could be restarted + :type all_services: List[str] + """ + deferred_only = action_get("deferred-only") + services = action_get("services") + if services: + services = services.split() + else: + services = all_services + if deferred_only: + restart_services_action(deferred_only=True) + else: + restart_services_action(services=services) + + +def show_deferred_events_action_helper(): + """Helper to run the show-deferred-restarts action.""" + restarts = [] + for event in deferred_events.get_deferred_events(): + restarts.append('{} {} {}'.format( + str(event.timestamp), + event.service.ljust(40), + event.reason)) + restarts.sort() + output = { + 'restarts': restarts, + 'hooks': deferred_events.get_deferred_hooks()} + action_set({'output': "{}".format( + yaml.dump(output, default_flow_style=False))}) diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index f6fa7fce..778aa4b6 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -1622,3 +1622,12 @@ def _contains_range(addresses): addresses.startswith(".") or ",." in addresses or " ." in addresses) + + +def is_subordinate(): + """Check whether charm is subordinate in unit metadata. + + :returns: True if unit is subordniate, False otherwise. + :rtype: bool + """ + return metadata().get('subordinate') is True diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 22e2e951..d25e6c59 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -694,39 +694,93 @@ class ChecksumError(ValueError): pass -def restart_on_change(restart_map, stopstart=False, restart_functions=None): - """Restart services based on configuration files changing +class restart_on_change(object): + """Decorator and context manager to handle restarts. - This function is used a decorator, for example:: + Usage: - @restart_on_change({ - '/etc/ceph/ceph.conf': [ 'cinder-api', 'cinder-volume' ] - '/etc/apache/sites-enabled/*': [ 'apache2' ] - }) - def config_changed(): - pass # your code here + @restart_on_change(restart_map, ...) + def function_that_might_trigger_a_restart(...) + ... - In this example, the cinder-api and cinder-volume services - would be restarted if /etc/ceph/ceph.conf is changed by the - ceph_client_changed function. The apache2 service would be - restarted if any file matching the pattern got changed, created - or removed. Standard wildcards are supported, see documentation - for the 'glob' module for more information. + Or: - @param restart_map: {path_file_name: [service_name, ...] - @param stopstart: DEFAULT false; whether to stop, start OR restart - @param restart_functions: nonstandard functions to use to restart services - {svc: func, ...} - @returns result from decorated function + with restart_on_change(restart_map, ...): + do_stuff_that_might_trigger_a_restart() + ... """ - def wrap(f): + + def __init__(self, restart_map, stopstart=False, restart_functions=None, + can_restart_now_f=None, post_svc_restart_f=None, + pre_restarts_wait_f=None): + """ + :param restart_map: {file: [service, ...]} + :type restart_map: Dict[str, List[str,]] + :param stopstart: whether to stop, start or restart a service + :type stopstart: booleean + :param restart_functions: nonstandard functions to use to restart + services {svc: func, ...} + :type restart_functions: Dict[str, Callable[[str], None]] + :param can_restart_now_f: A function used to check if the restart is + permitted. + :type can_restart_now_f: Callable[[str, List[str]], boolean] + :param post_svc_restart_f: A function run after a service has + restarted. + :type post_svc_restart_f: Callable[[str], None] + :param pre_restarts_wait_f: A function callled before any restarts. + :type pre_restarts_wait_f: Callable[None, None] + """ + self.restart_map = restart_map + self.stopstart = stopstart + self.restart_functions = restart_functions + self.can_restart_now_f = can_restart_now_f + self.post_svc_restart_f = post_svc_restart_f + self.pre_restarts_wait_f = pre_restarts_wait_f + + def __call__(self, f): + """Work like a decorator. + + Returns a wrapped function that performs the restart if triggered. + + :param f: The function that is being wrapped. + :type f: Callable[[Any], Any] + :returns: the wrapped function + :rtype: Callable[[Any], Any] + """ @functools.wraps(f) def wrapped_f(*args, **kwargs): return restart_on_change_helper( - (lambda: f(*args, **kwargs)), restart_map, stopstart, - restart_functions) + (lambda: f(*args, **kwargs)), + self.restart_map, + stopstart=self.stopstart, + restart_functions=self.restart_functions, + can_restart_now_f=self.can_restart_now_f, + post_svc_restart_f=self.post_svc_restart_f, + pre_restarts_wait_f=self.pre_restarts_wait_f) return wrapped_f - return wrap + + def __enter__(self): + """Enter the runtime context related to this object. """ + self.checksums = _pre_restart_on_change_helper(self.restart_map) + + def __exit__(self, exc_type, exc_val, exc_tb): + """Exit the runtime context related to this object. + + The parameters describe the exception that caused the context to be + exited. If the context was exited without an exception, all three + arguments will be None. + """ + if exc_type is None: + _post_restart_on_change_helper( + self.checksums, + self.restart_map, + stopstart=self.stopstart, + restart_functions=self.restart_functions, + can_restart_now_f=self.can_restart_now_f, + post_svc_restart_f=self.post_svc_restart_f, + pre_restarts_wait_f=self.pre_restarts_wait_f) + # All is good, so return False; any exceptions will propagate. + return False def restart_on_change_helper(lambda_f, restart_map, stopstart=False, @@ -747,8 +801,8 @@ def restart_on_change_helper(lambda_f, restart_map, stopstart=False, `my_restart_func` should a function which takes one argument which is the service name to be retstarted. - `can_restart_now_f` is a function which checks that a restart is permitted. It - should returna bool which indicates if a restart is allowed and should + `can_restart_now_f` is a function which checks that a restart is permitted. + It should return a bool which indicates if a restart is allowed and should take a service name (str) and a list of changed files (List[str]) as arguments. @@ -779,10 +833,58 @@ def restart_on_change_helper(lambda_f, restart_map, stopstart=False, :returns: result of lambda_f() :rtype: ANY """ + checksums = _pre_restart_on_change_helper(restart_map) + r = lambda_f() + _post_restart_on_change_helper(checksums, + restart_map, + stopstart, + restart_functions, + can_restart_now_f, + post_svc_restart_f, + pre_restarts_wait_f) + return r + + +def _pre_restart_on_change_helper(restart_map): + """Take a snapshot of file hashes. + + :param restart_map: {file: [service, ...]} + :type restart_map: Dict[str, List[str,]] + :returns: Dictionary of file paths and the files checksum. + :rtype: Dict[str, str] + """ + return {path: path_hash(path) for path in restart_map} + + +def _post_restart_on_change_helper(checksums, + restart_map, + stopstart=False, + restart_functions=None, + can_restart_now_f=None, + post_svc_restart_f=None, + pre_restarts_wait_f=None): + """Check whether files have changed. + + :param checksums: Dictionary of file paths and the files checksum. + :type checksums: Dict[str, str] + :param restart_map: {file: [service, ...]} + :type restart_map: Dict[str, List[str,]] + :param stopstart: whether to stop, start or restart a service + :type stopstart: booleean + :param restart_functions: nonstandard functions to use to restart services + {svc: func, ...} + :type restart_functions: Dict[str, Callable[[str], None]] + :param can_restart_now_f: A function used to check if the restart is + permitted. + :type can_restart_now_f: Callable[[str, List[str]], boolean] + :param post_svc_restart_f: A function run after a service has + restarted. + :type post_svc_restart_f: Callable[[str], None] + :param pre_restarts_wait_f: A function callled before any restarts. + :type pre_restarts_wait_f: Callable[None, None] + """ if restart_functions is None: restart_functions = {} - checksums = {path: path_hash(path) for path in restart_map} - r = lambda_f() changed_files = defaultdict(list) restarts = [] # create a list of lists of the services to restart @@ -799,7 +901,8 @@ def restart_on_change_helper(lambda_f, restart_map, stopstart=False, actions = ('stop', 'start') if stopstart else ('restart',) for service_name in services_list: if can_restart_now_f: - if not can_restart_now_f(service_name, changed_files[service_name]): + if not can_restart_now_f(service_name, + changed_files[service_name]): continue if service_name in restart_functions: restart_functions[service_name](service_name) @@ -808,7 +911,6 @@ def restart_on_change_helper(lambda_f, restart_map, stopstart=False, service(action, service_name) if post_svc_restart_f: post_svc_restart_f(service_name) - return r def pwgen(length=None): diff --git a/hooks/charmhelpers/core/host_factory/ubuntu.py b/hooks/charmhelpers/core/host_factory/ubuntu.py index a3ec6947..7ee8a6ed 100644 --- a/hooks/charmhelpers/core/host_factory/ubuntu.py +++ b/hooks/charmhelpers/core/host_factory/ubuntu.py @@ -96,12 +96,14 @@ def cmp_pkgrevno(package, revno, pkgcache=None): the pkgcache argument is None. Be sure to add charmhelpers.fetch if you call this function, or pass an apt_pkg.Cache() instance. """ - from charmhelpers.fetch import apt_pkg + from charmhelpers.fetch import apt_pkg, get_installed_version if not pkgcache: - from charmhelpers.fetch import apt_cache - pkgcache = apt_cache() - pkg = pkgcache[package] - return apt_pkg.version_compare(pkg.current_ver.ver_str, revno) + current_ver = get_installed_version(package) + else: + pkg = pkgcache[package] + current_ver = pkg.current_ver + + return apt_pkg.version_compare(current_ver.ver_str, revno) @cached diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 0cc7fc85..5b689f5b 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -105,6 +105,7 @@ if __platform__ == "ubuntu": get_upstream_version = fetch.get_upstream_version apt_pkg = fetch.ubuntu_apt_pkg get_apt_dpkg_env = fetch.get_apt_dpkg_env + get_installed_version = fetch.get_installed_version elif __platform__ == "centos": yum_search = fetch.yum_search diff --git a/hooks/charmhelpers/fetch/ubuntu.py b/hooks/charmhelpers/fetch/ubuntu.py index 2bf5ce55..b38edcc1 100644 --- a/hooks/charmhelpers/fetch/ubuntu.py +++ b/hooks/charmhelpers/fetch/ubuntu.py @@ -200,6 +200,14 @@ CLOUD_ARCHIVE_POCKETS = { 'victoria/proposed': 'focal-proposed/victoria', 'focal-victoria/proposed': 'focal-proposed/victoria', 'focal-proposed/victoria': 'focal-proposed/victoria', + # Wallaby + 'wallaby': 'focal-updates/wallaby', + 'focal-wallaby': 'focal-updates/wallaby', + 'focal-wallaby/updates': 'focal-updates/wallaby', + 'focal-updates/wallaby': 'focal-updates/wallaby', + 'wallaby/proposed': 'focal-proposed/wallaby', + 'focal-wallaby/proposed': 'focal-proposed/wallaby', + 'focal-proposed/wallaby': 'focal-proposed/wallaby', } @@ -650,14 +658,17 @@ def _add_apt_repository(spec): :param spec: the parameter to pass to add_apt_repository :type spec: str """ + series = get_distrib_codename() if '{series}' in spec: - series = get_distrib_codename() spec = spec.replace('{series}', series) # software-properties package for bionic properly reacts to proxy settings - # passed as environment variables (See lp:1433761). This is not the case - # LTS and non-LTS releases below bionic. - _run_with_retries(['add-apt-repository', '--yes', spec], - cmd_env=env_proxy_settings(['https', 'http'])) + # set via apt.conf (see lp:1433761), however this is not the case for LTS + # and non-LTS releases before bionic. + if series in ('trusty', 'xenial'): + _run_with_retries(['add-apt-repository', '--yes', spec], + cmd_env=env_proxy_settings(['https', 'http'])) + else: + _run_with_retries(['add-apt-repository', '--yes', spec]) def _add_cloud_pocket(pocket): @@ -828,6 +839,22 @@ def get_upstream_version(package): return ubuntu_apt_pkg.upstream_version(pkg.current_ver.ver_str) +def get_installed_version(package): + """Determine installed version of a package + + @returns None (if not installed) or the installed version as + Version object + """ + cache = apt_cache() + dpkg_result = cache._dpkg_list([package]).get(package, {}) + current_ver = None + installed_version = dpkg_result.get('version') + + if installed_version: + current_ver = ubuntu_apt_pkg.Version({'ver_str': installed_version}) + return current_ver + + def get_apt_dpkg_env(): """Get environment suitable for execution of APT and DPKG tools. diff --git a/test-requirements.txt b/test-requirements.txt index 9aea716b..394e4d37 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -42,8 +42,8 @@ oslo.utils<=3.41.0;python_version<'3.6' coverage>=4.5.2 pyudev # for ceph-* charm unit tests (need to fix the ceph-* charm unit tests/mocking) -git+https://github.com/openstack-charmers/zaza.git#egg=zaza;python_version>='3.0' -git+https://github.com/openstack-charmers/zaza-openstack-tests.git#egg=zaza.openstack +git+https://github.com/openstack-charmers/zaza.git@stable/21.04#egg=zaza;python_version>='3.0' +git+https://github.com/openstack-charmers/zaza-openstack-tests.git@stable/21.04#egg=zaza.openstack # Needed for charm-glance: git+https://opendev.org/openstack/tempest.git#egg=tempest;python_version>='3.6'