From ec488a74fd41c641e6c1ec591e430e86ccb06ebd Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Sun, 17 Jan 2016 21:18:57 +0000 Subject: [PATCH] [trivial] charm-helpers sync --- .../charmhelpers/contrib/openstack/context.py | 14 +- hooks/charmhelpers/contrib/openstack/utils.py | 158 +++++++++++------- hooks/charmhelpers/core/host.py | 115 ++++++++----- hooks/charmhelpers/fetch/giturl.py | 7 +- 4 files changed, 182 insertions(+), 112 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 25125de..e930bf3 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -57,6 +57,7 @@ from charmhelpers.core.host import ( get_nic_hwaddr, mkdir, write_file, + pwgen, ) from charmhelpers.contrib.hahelpers.cluster import ( determine_apache_port, @@ -87,6 +88,8 @@ from charmhelpers.contrib.network.ip import ( is_bridge_member, ) from charmhelpers.contrib.openstack.utils import get_host_ip +from charmhelpers.core.unitdata import kv + CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' ADDRESS_TYPES = ['admin', 'internal', 'public'] @@ -636,11 +639,18 @@ class HAProxyContext(OSContextGenerator): ctxt['ipv6'] = True ctxt['local_host'] = 'ip6-localhost' ctxt['haproxy_host'] = '::' - ctxt['stat_port'] = ':::8888' else: ctxt['local_host'] = '127.0.0.1' ctxt['haproxy_host'] = '0.0.0.0' - ctxt['stat_port'] = ':8888' + + ctxt['stat_port'] = '8888' + + db = kv() + ctxt['stat_password'] = db.get('stat-password') + if not ctxt['stat_password']: + ctxt['stat_password'] = db.set('stat-password', + pwgen(32)) + db.flush() for frontend in cluster_hosts: if (len(cluster_hosts[frontend]['backends']) > 1 or diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index f4af176..2ed7955 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -103,68 +103,67 @@ OPENSTACK_CODENAMES = OrderedDict([ ('2016.1', 'mitaka'), ]) -# The ugly duckling +# The ugly duckling - must list releases oldest to newest SWIFT_CODENAMES = OrderedDict([ - ('1.4.3', 'diablo'), - ('1.4.8', 'essex'), - ('1.7.4', 'folsom'), - ('1.8.0', 'grizzly'), - ('1.7.7', 'grizzly'), - ('1.7.6', 'grizzly'), - ('1.10.0', 'havana'), - ('1.9.1', 'havana'), - ('1.9.0', 'havana'), - ('1.13.1', 'icehouse'), - ('1.13.0', 'icehouse'), - ('1.12.0', 'icehouse'), - ('1.11.0', 'icehouse'), - ('2.0.0', 'juno'), - ('2.1.0', 'juno'), - ('2.2.0', 'juno'), - ('2.2.1', 'kilo'), - ('2.2.2', 'kilo'), - ('2.3.0', 'liberty'), - ('2.4.0', 'liberty'), - ('2.5.0', 'liberty'), + ('diablo', + ['1.4.3']), + ('essex', + ['1.4.8']), + ('folsom', + ['1.7.4']), + ('grizzly', + ['1.7.6', '1.7.7', '1.8.0']), + ('havana', + ['1.9.0', '1.9.1', '1.10.0']), + ('icehouse', + ['1.11.0', '1.12.0', '1.13.0', '1.13.1']), + ('juno', + ['2.0.0', '2.1.0', '2.2.0']), + ('kilo', + ['2.2.1', '2.2.2']), + ('liberty', + ['2.3.0', '2.4.0', '2.5.0']), + ('mitaka', + ['2.5.0']), ]) # >= Liberty version->codename mapping PACKAGE_CODENAMES = { 'nova-common': OrderedDict([ - ('12.0.0', 'liberty'), - ('13.0.0', 'mitaka'), + ('12.0', 'liberty'), + ('13.0', 'mitaka'), ]), 'neutron-common': OrderedDict([ - ('7.0.0', 'liberty'), - ('8.0.0', 'mitaka'), + ('7.0', 'liberty'), + ('8.0', 'mitaka'), ]), 'cinder-common': OrderedDict([ - ('7.0.0', 'liberty'), - ('8.0.0', 'mitaka'), + ('7.0', 'liberty'), + ('8.0', 'mitaka'), ]), 'keystone': OrderedDict([ - ('8.0.0', 'liberty'), - ('9.0.0', 'mitaka'), + ('8.0', 'liberty'), + ('9.0', 'mitaka'), ]), 'horizon-common': OrderedDict([ - ('8.0.0', 'liberty'), - ('9.0.0', 'mitaka'), + ('8.0', 'liberty'), + ('9.0', 'mitaka'), ]), 'ceilometer-common': OrderedDict([ - ('5.0.0', 'liberty'), - ('6.0.0', 'mitaka'), + ('5.0', 'liberty'), + ('6.0', 'mitaka'), ]), 'heat-common': OrderedDict([ - ('5.0.0', 'liberty'), - ('6.0.0', 'mitaka'), + ('5.0', 'liberty'), + ('6.0', 'mitaka'), ]), 'glance-common': OrderedDict([ - ('11.0.0', 'liberty'), - ('12.0.0', 'mitaka'), + ('11.0', 'liberty'), + ('12.0', 'mitaka'), ]), 'openstack-dashboard': OrderedDict([ - ('8.0.0', 'liberty'), - ('9.0.0', 'mitaka'), + ('8.0', 'liberty'), + ('9.0', 'mitaka'), ]), } @@ -227,6 +226,33 @@ def get_os_version_codename(codename, version_map=OPENSTACK_CODENAMES): error_out(e) +def get_os_version_codename_swift(codename): + '''Determine OpenStack version number of swift from codename.''' + for k, v in six.iteritems(SWIFT_CODENAMES): + if k == codename: + return v[-1] + e = 'Could not derive swift version for '\ + 'codename: %s' % codename + error_out(e) + + +def get_swift_codename(version): + '''Determine OpenStack codename that corresponds to swift version.''' + codenames = [k for k, v in six.iteritems(SWIFT_CODENAMES) if version in v] + if len(codenames) > 1: + # If more than one release codename contains this version we determine + # the actual codename based on the highest available install source. + for codename in reversed(codenames): + releases = UBUNTU_OPENSTACK_RELEASE + release = [k for k, v in six.iteritems(releases) if codename in v] + ret = subprocess.check_output(['apt-cache', 'policy', 'swift']) + if codename in ret or release[0] in ret: + return codename + elif len(codenames) == 1: + return codenames[0] + return None + + def get_os_codename_package(package, fatal=True): '''Derive OpenStack release codename from an installed package.''' import apt_pkg as apt @@ -251,7 +277,14 @@ def get_os_codename_package(package, fatal=True): error_out(e) vers = apt.upstream_version(pkg.current_ver.ver_str) - match = re.match('^(\d+)\.(\d+)\.(\d+)', vers) + if 'swift' in pkg.name: + # Fully x.y.z match for swift versions + match = re.match('^(\d+)\.(\d+)\.(\d+)', vers) + else: + # x.y match only for 20XX.X + # and ignore patch level for other packages + match = re.match('^(\d+)\.(\d+)', vers) + if match: vers = match.group(0) @@ -263,13 +296,8 @@ def get_os_codename_package(package, fatal=True): # < Liberty co-ordinated project versions try: if 'swift' in pkg.name: - swift_vers = vers[:5] - if swift_vers not in SWIFT_CODENAMES: - # Deal with 1.10.0 upward - swift_vers = vers[:6] - return SWIFT_CODENAMES[swift_vers] + return get_swift_codename(vers) else: - vers = vers[:6] return OPENSTACK_CODENAMES[vers] except KeyError: if not fatal: @@ -287,12 +315,14 @@ def get_os_version_package(pkg, fatal=True): if 'swift' in pkg: vers_map = SWIFT_CODENAMES + for cname, version in six.iteritems(vers_map): + if cname == codename: + return version[-1] else: vers_map = OPENSTACK_CODENAMES - - for version, cname in six.iteritems(vers_map): - if cname == codename: - return version + for version, cname in six.iteritems(vers_map): + if cname == codename: + return version # e = "Could not determine OpenStack version for package: %s" % pkg # error_out(e) @@ -458,11 +488,16 @@ def openstack_upgrade_available(package): cur_vers = get_os_version_package(package) if "swift" in package: codename = get_os_codename_install_source(src) - available_vers = get_os_version_codename(codename, SWIFT_CODENAMES) + avail_vers = get_os_version_codename_swift(codename) else: - available_vers = get_os_version_install_source(src) + avail_vers = get_os_version_install_source(src) apt.init() - return apt.version_compare(available_vers, cur_vers) == 1 + if "swift" in package: + major_cur_vers = cur_vers.split('.', 1)[0] + major_avail_vers = avail_vers.split('.', 1)[0] + major_diff = apt.version_compare(major_avail_vers, major_cur_vers) + return avail_vers > cur_vers and (major_diff == 1 or major_diff == 0) + return apt.version_compare(avail_vers, cur_vers) == 1 def ensure_block_device(block_device): @@ -591,7 +626,7 @@ def _git_yaml_load(projects_yaml): return yaml.load(projects_yaml) -def git_clone_and_install(projects_yaml, core_project, depth=1): +def git_clone_and_install(projects_yaml, core_project): """ Clone/install all specified OpenStack repositories. @@ -641,6 +676,9 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): for p in projects['repositories']: repo = p['repository'] branch = p['branch'] + depth = '1' + if 'depth' in p.keys(): + depth = p['depth'] if p['name'] == 'requirements': repo_dir = _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy, @@ -685,19 +723,13 @@ def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy, """ Clone and install a single git repository. """ - dest_dir = os.path.join(parent_dir, os.path.basename(repo)) - if not os.path.exists(parent_dir): juju_log('Directory already exists at {}. ' 'No need to create directory.'.format(parent_dir)) os.mkdir(parent_dir) - if not os.path.exists(dest_dir): - juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) - repo_dir = install_remote(repo, dest=parent_dir, branch=branch, - depth=depth) - else: - repo_dir = dest_dir + juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) + repo_dir = install_remote(repo, dest=parent_dir, branch=branch, depth=depth) venv = os.path.join(parent_dir, 'venv') diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index c5fd81c..aeada95 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -72,7 +72,9 @@ def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d"): stopped = service_stop(service_name) upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) sysv_file = os.path.join(initd_dir, service_name) - if os.path.exists(upstart_file): + if init_is_systemd(): + service('disable', service_name) + elif os.path.exists(upstart_file): override_path = os.path.join( init_dir, '{}.override'.format(service_name)) with open(override_path, 'w') as fh: @@ -80,9 +82,9 @@ def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d"): elif os.path.exists(sysv_file): subprocess.check_call(["update-rc.d", service_name, "disable"]) else: - # XXX: Support SystemD too raise ValueError( - "Unable to detect {0} as either Upstart {1} or SysV {2}".format( + "Unable to detect {0} as SystemD, Upstart {1} or" + " SysV {2}".format( service_name, upstart_file, sysv_file)) return stopped @@ -94,7 +96,9 @@ def service_resume(service_name, init_dir="/etc/init", Reenable starting again at boot. Start the service""" upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) sysv_file = os.path.join(initd_dir, service_name) - if os.path.exists(upstart_file): + if init_is_systemd(): + service('enable', service_name) + elif os.path.exists(upstart_file): override_path = os.path.join( init_dir, '{}.override'.format(service_name)) if os.path.exists(override_path): @@ -102,9 +106,9 @@ def service_resume(service_name, init_dir="/etc/init", elif os.path.exists(sysv_file): subprocess.check_call(["update-rc.d", service_name, "enable"]) else: - # XXX: Support SystemD too raise ValueError( - "Unable to detect {0} as either Upstart {1} or SysV {2}".format( + "Unable to detect {0} as SystemD, Upstart {1} or" + " SysV {2}".format( service_name, upstart_file, sysv_file)) started = service_running(service_name) @@ -115,23 +119,29 @@ def service_resume(service_name, init_dir="/etc/init", def service(action, service_name): """Control a system service""" - cmd = ['service', service_name, action] + if init_is_systemd(): + cmd = ['systemctl', action, service_name] + else: + cmd = ['service', service_name, action] return subprocess.call(cmd) == 0 -def service_running(service): +def service_running(service_name): """Determine whether a system service is running""" - try: - output = subprocess.check_output( - ['service', service, 'status'], - stderr=subprocess.STDOUT).decode('UTF-8') - except subprocess.CalledProcessError: - return False + if init_is_systemd(): + return service('is-active', service_name) else: - if ("start/running" in output or "is running" in output): - return True - else: + try: + output = subprocess.check_output( + ['service', service_name, 'status'], + stderr=subprocess.STDOUT).decode('UTF-8') + except subprocess.CalledProcessError: return False + else: + if ("start/running" in output or "is running" in output): + return True + else: + return False def service_available(service_name): @@ -146,10 +156,17 @@ def service_available(service_name): return True +SYSTEMD_SYSTEM = '/run/systemd/system' + + +def init_is_systemd(): + """Return True if the host system uses systemd, False otherwise.""" + return os.path.isdir(SYSTEMD_SYSTEM) + + def adduser(username, password=None, shell='/bin/bash', system_user=False, primary_group=None, secondary_groups=None): - """ - Add a user to the system. + """Add a user to the system. Will log but otherwise succeed if the user already exists. @@ -157,7 +174,7 @@ def adduser(username, password=None, shell='/bin/bash', system_user=False, :param str password: Password for user; if ``None``, create a system user :param str shell: The default shell for the user :param bool system_user: Whether to create a login or system user - :param str primary_group: Primary group for user; defaults to their username + :param str primary_group: Primary group for user; defaults to username :param list secondary_groups: Optional list of additional groups :returns: The password database entry struct, as returned by `pwd.getpwnam` @@ -283,14 +300,12 @@ def write_file(path, content, owner='root', group='root', perms=0o444): def fstab_remove(mp): - """Remove the given mountpoint entry from /etc/fstab - """ + """Remove the given mountpoint entry from /etc/fstab""" return Fstab.remove_by_mountpoint(mp) def fstab_add(dev, mp, fs, options=None): - """Adds the given device entry to the /etc/fstab file - """ + """Adds the given device entry to the /etc/fstab file""" return Fstab.add(dev, mp, fs, options=options) @@ -346,8 +361,7 @@ def fstab_mount(mountpoint): def file_hash(path, hash_type='md5'): - """ - Generate a hash checksum of the contents of 'path' or None if not found. + """Generate a hash checksum of the contents of 'path' or None if not found. :param str hash_type: Any hash alrgorithm supported by :mod:`hashlib`, such as md5, sha1, sha256, sha512, etc. @@ -362,10 +376,9 @@ def file_hash(path, hash_type='md5'): def path_hash(path): - """ - Generate a hash checksum of all files matching 'path'. Standard wildcards - like '*' and '?' are supported, see documentation for the 'glob' module for - more information. + """Generate a hash checksum of all files matching 'path'. Standard + wildcards like '*' and '?' are supported, see documentation for the 'glob' + module for more information. :return: dict: A { filename: hash } dictionary for all matched files. Empty if none found. @@ -377,8 +390,7 @@ def path_hash(path): def check_hash(path, checksum, hash_type='md5'): - """ - Validate a file using a cryptographic checksum. + """Validate a file using a cryptographic checksum. :param str checksum: Value of the checksum used to validate the file. :param str hash_type: Hash algorithm used to generate `checksum`. @@ -393,6 +405,7 @@ def check_hash(path, checksum, hash_type='md5'): class ChecksumError(ValueError): + """A class derived from Value error to indicate the checksum failed.""" pass @@ -498,7 +511,7 @@ def get_bond_master(interface): def list_nics(nic_type=None): - '''Return a list of nics of given type(s)''' + """Return a list of nics of given type(s)""" if isinstance(nic_type, six.string_types): int_types = [nic_type] else: @@ -540,12 +553,13 @@ def list_nics(nic_type=None): def set_nic_mtu(nic, mtu): - '''Set MTU on a network interface''' + """Set the Maximum Transmission Unit (MTU) on a network interface.""" cmd = ['ip', 'link', 'set', nic, 'mtu', mtu] subprocess.check_call(cmd) def get_nic_mtu(nic): + """Return the Maximum Transmission Unit (MTU) for a network interface.""" cmd = ['ip', 'addr', 'show', nic] ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n') mtu = "" @@ -557,6 +571,7 @@ def get_nic_mtu(nic): def get_nic_hwaddr(nic): + """Return the Media Access Control (MAC) for a network interface.""" cmd = ['ip', '-o', '-0', 'addr', 'show', nic] ip_output = subprocess.check_output(cmd).decode('UTF-8') hwaddr = "" @@ -567,7 +582,7 @@ def get_nic_hwaddr(nic): def cmp_pkgrevno(package, revno, pkgcache=None): - '''Compare supplied revno with the revno of the installed package + """Compare supplied revno with the revno of the installed package * 1 => Installed revno is greater than supplied arg * 0 => Installed revno is the same as supplied arg @@ -576,7 +591,7 @@ def cmp_pkgrevno(package, revno, pkgcache=None): This function imports apt_cache function from charmhelpers.fetch if the pkgcache argument is None. Be sure to add charmhelpers.fetch if you call this function, or pass an apt_pkg.Cache() instance. - ''' + """ import apt_pkg if not pkgcache: from charmhelpers.fetch import apt_cache @@ -586,19 +601,27 @@ def cmp_pkgrevno(package, revno, pkgcache=None): @contextmanager -def chdir(d): +def chdir(directory): + """Change the current working directory to a different directory for a code + block and return the previous directory after the block exits. Useful to + run commands from a specificed directory. + + :param str directory: The directory path to change to for this context. + """ cur = os.getcwd() try: - yield os.chdir(d) + yield os.chdir(directory) finally: os.chdir(cur) def chownr(path, owner, group, follow_links=True, chowntopdir=False): - """ - Recursively change user and group ownership of files and directories + """Recursively change user and group ownership of files and directories in given path. Doesn't chown path itself by default, only its children. + :param str path: The string path to start changing ownership. + :param str owner: The owner string to use when looking up the uid. + :param str group: The group string to use when looking up the gid. :param bool follow_links: Also Chown links if True :param bool chowntopdir: Also chown path itself if True """ @@ -622,15 +645,23 @@ def chownr(path, owner, group, follow_links=True, chowntopdir=False): def lchownr(path, owner, group): + """Recursively change user and group ownership of files and directories + in a given path, not following symbolic links. See the documentation for + 'os.lchown' for more information. + + :param str path: The string path to start changing ownership. + :param str owner: The owner string to use when looking up the uid. + :param str group: The group string to use when looking up the gid. + """ chownr(path, owner, group, follow_links=False) def get_total_ram(): - '''The total amount of system RAM in bytes. + """The total amount of system RAM in bytes. This is what is reported by the OS, and may be overcommitted when there are multiple containers hosted on the same machine. - ''' + """ with open('/proc/meminfo', 'r') as f: for line in f.readlines(): if line: diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index bbf89d5..9ad8dc6 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -22,7 +22,6 @@ from charmhelpers.fetch import ( filter_installed_packages, apt_install, ) -from charmhelpers.core.host import mkdir if filter_installed_packages(['git']) != []: apt_install(['git']) @@ -50,8 +49,8 @@ class GitUrlFetchHandler(BaseFetchHandler): cmd = ['git', '-C', dest, 'pull', source, branch] else: cmd = ['git', 'clone', source, dest, '--branch', branch] - if depth: - cmd.extend(['--depth', depth]) + if depth: + cmd.extend(['--depth', depth]) check_call(cmd) def install(self, source, branch="master", dest=None, depth=None): @@ -62,8 +61,6 @@ class GitUrlFetchHandler(BaseFetchHandler): else: dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", branch_name) - if not os.path.exists(dest_dir): - mkdir(dest_dir, perms=0o755) try: self.clone(source, dest_dir, branch, depth) except OSError as e: