summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--hooks/charmhelpers/__init__.py8
-rw-r--r--hooks/charmhelpers/contrib/hahelpers/apache.py14
-rw-r--r--hooks/charmhelpers/contrib/hardening/apache/checks/config.py3
-rw-r--r--hooks/charmhelpers/contrib/hardening/audits/apache.py6
-rw-r--r--hooks/charmhelpers/contrib/hardening/harden.py18
-rw-r--r--hooks/charmhelpers/contrib/openstack/amulet/utils.py116
-rw-r--r--hooks/charmhelpers/contrib/openstack/cert_utils.py48
-rw-r--r--hooks/charmhelpers/contrib/openstack/context.py39
-rw-r--r--hooks/charmhelpers/contrib/openstack/ha/utils.py12
-rw-r--r--hooks/charmhelpers/contrib/openstack/utils.py57
-rw-r--r--hooks/charmhelpers/contrib/storage/linux/loopback.py2
-rw-r--r--hooks/charmhelpers/core/hookenv.py65
-rw-r--r--hooks/charmhelpers/core/host.py55
-rw-r--r--hooks/charmhelpers/core/kernel.py4
-rw-r--r--hooks/charmhelpers/fetch/__init__.py2
-rw-r--r--hooks/charmhelpers/fetch/bzrurl.py4
-rw-r--r--hooks/charmhelpers/fetch/giturl.py4
-rw-r--r--hooks/charmhelpers/fetch/ubuntu.py25
18 files changed, 395 insertions, 87 deletions
diff --git a/hooks/charmhelpers/__init__.py b/hooks/charmhelpers/__init__.py
index e7aa471..61ef907 100644
--- a/hooks/charmhelpers/__init__.py
+++ b/hooks/charmhelpers/__init__.py
@@ -23,22 +23,22 @@ import subprocess
23import sys 23import sys
24 24
25try: 25try:
26 import six # flake8: noqa 26 import six # NOQA:F401
27except ImportError: 27except ImportError:
28 if sys.version_info.major == 2: 28 if sys.version_info.major == 2:
29 subprocess.check_call(['apt-get', 'install', '-y', 'python-six']) 29 subprocess.check_call(['apt-get', 'install', '-y', 'python-six'])
30 else: 30 else:
31 subprocess.check_call(['apt-get', 'install', '-y', 'python3-six']) 31 subprocess.check_call(['apt-get', 'install', '-y', 'python3-six'])
32 import six # flake8: noqa 32 import six # NOQA:F401
33 33
34try: 34try:
35 import yaml # flake8: noqa 35 import yaml # NOQA:F401
36except ImportError: 36except ImportError:
37 if sys.version_info.major == 2: 37 if sys.version_info.major == 2:
38 subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml']) 38 subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml'])
39 else: 39 else:
40 subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml']) 40 subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml'])
41 import yaml # flake8: noqa 41 import yaml # NOQA:F401
42 42
43 43
44# Holds a list of mapping of mangled function names that have been deprecated 44# Holds a list of mapping of mangled function names that have been deprecated
diff --git a/hooks/charmhelpers/contrib/hahelpers/apache.py b/hooks/charmhelpers/contrib/hahelpers/apache.py
index 605a1be..2c1e371 100644
--- a/hooks/charmhelpers/contrib/hahelpers/apache.py
+++ b/hooks/charmhelpers/contrib/hahelpers/apache.py
@@ -23,8 +23,8 @@
23# 23#
24 24
25import os 25import os
26import subprocess
27 26
27from charmhelpers.core import host
28from charmhelpers.core.hookenv import ( 28from charmhelpers.core.hookenv import (
29 config as config_get, 29 config as config_get,
30 relation_get, 30 relation_get,
@@ -83,14 +83,4 @@ def retrieve_ca_cert(cert_file):
83 83
84 84
85def install_ca_cert(ca_cert): 85def install_ca_cert(ca_cert):
86 if ca_cert: 86 host.install_ca_cert(ca_cert, 'keystone_juju_ca_cert')
87 cert_file = ('/usr/local/share/ca-certificates/'
88 'keystone_juju_ca_cert.crt')
89 old_cert = retrieve_ca_cert(cert_file)
90 if old_cert and old_cert == ca_cert:
91 log("CA cert is the same as installed version", level=INFO)
92 else:
93 log("Installing new CA cert", level=INFO)
94 with open(cert_file, 'wb') as crt:
95 crt.write(ca_cert)
96 subprocess.check_call(['update-ca-certificates', '--fresh'])
diff --git a/hooks/charmhelpers/contrib/hardening/apache/checks/config.py b/hooks/charmhelpers/contrib/hardening/apache/checks/config.py
index 06482aa..341da9e 100644
--- a/hooks/charmhelpers/contrib/hardening/apache/checks/config.py
+++ b/hooks/charmhelpers/contrib/hardening/apache/checks/config.py
@@ -14,6 +14,7 @@
14 14
15import os 15import os
16import re 16import re
17import six
17import subprocess 18import subprocess
18 19
19 20
@@ -95,6 +96,8 @@ class ApacheConfContext(object):
95 ctxt = settings['hardening'] 96 ctxt = settings['hardening']
96 97
97 out = subprocess.check_output(['apache2', '-v']) 98 out = subprocess.check_output(['apache2', '-v'])
99 if six.PY3:
100 out = out.decode('utf-8')
98 ctxt['apache_version'] = re.search(r'.+version: Apache/(.+?)\s.+', 101 ctxt['apache_version'] = re.search(r'.+version: Apache/(.+?)\s.+',
99 out).group(1) 102 out).group(1)
100 ctxt['apache_icondir'] = '/usr/share/apache2/icons/' 103 ctxt['apache_icondir'] = '/usr/share/apache2/icons/'
diff --git a/hooks/charmhelpers/contrib/hardening/audits/apache.py b/hooks/charmhelpers/contrib/hardening/audits/apache.py
index d32bf44..04825f5 100644
--- a/hooks/charmhelpers/contrib/hardening/audits/apache.py
+++ b/hooks/charmhelpers/contrib/hardening/audits/apache.py
@@ -15,7 +15,7 @@
15import re 15import re
16import subprocess 16import subprocess
17 17
18from six import string_types 18import six
19 19
20from charmhelpers.core.hookenv import ( 20from charmhelpers.core.hookenv import (
21 log, 21 log,
@@ -35,7 +35,7 @@ class DisabledModuleAudit(BaseAudit):
35 def __init__(self, modules): 35 def __init__(self, modules):
36 if modules is None: 36 if modules is None:
37 self.modules = [] 37 self.modules = []
38 elif isinstance(modules, string_types): 38 elif isinstance(modules, six.string_types):
39 self.modules = [modules] 39 self.modules = [modules]
40 else: 40 else:
41 self.modules = modules 41 self.modules = modules
@@ -69,6 +69,8 @@ class DisabledModuleAudit(BaseAudit):
69 def _get_loaded_modules(): 69 def _get_loaded_modules():
70 """Returns the modules which are enabled in Apache.""" 70 """Returns the modules which are enabled in Apache."""
71 output = subprocess.check_output(['apache2ctl', '-M']) 71 output = subprocess.check_output(['apache2ctl', '-M'])
72 if six.PY3:
73 output = output.decode('utf-8')
72 modules = [] 74 modules = []
73 for line in output.splitlines(): 75 for line in output.splitlines():
74 # Each line of the enabled module output looks like: 76 # Each line of the enabled module output looks like:
diff --git a/hooks/charmhelpers/contrib/hardening/harden.py b/hooks/charmhelpers/contrib/hardening/harden.py
index b55764c..63f21b9 100644
--- a/hooks/charmhelpers/contrib/hardening/harden.py
+++ b/hooks/charmhelpers/contrib/hardening/harden.py
@@ -27,6 +27,8 @@ from charmhelpers.contrib.hardening.ssh.checks import run_ssh_checks
27from charmhelpers.contrib.hardening.mysql.checks import run_mysql_checks 27from charmhelpers.contrib.hardening.mysql.checks import run_mysql_checks
28from charmhelpers.contrib.hardening.apache.checks import run_apache_checks 28from charmhelpers.contrib.hardening.apache.checks import run_apache_checks
29 29
30_DISABLE_HARDENING_FOR_UNIT_TEST = False
31
30 32
31def harden(overrides=None): 33def harden(overrides=None):
32 """Hardening decorator. 34 """Hardening decorator.
@@ -47,16 +49,28 @@ def harden(overrides=None):
47 provided with 'harden' config. 49 provided with 'harden' config.
48 :returns: Returns value returned by decorated function once executed. 50 :returns: Returns value returned by decorated function once executed.
49 """ 51 """
52 if overrides is None:
53 overrides = []
54
50 def _harden_inner1(f): 55 def _harden_inner1(f):
51 log("Hardening function '%s'" % (f.__name__), level=DEBUG) 56 # As this has to be py2.7 compat, we can't use nonlocal. Use a trick
57 # to capture the dictionary that can then be updated.
58 _logged = {'done': False}
52 59
53 def _harden_inner2(*args, **kwargs): 60 def _harden_inner2(*args, **kwargs):
61 # knock out hardening via a config var; normally it won't get
62 # disabled.
63 if _DISABLE_HARDENING_FOR_UNIT_TEST:
64 return f(*args, **kwargs)
65 if not _logged['done']:
66 log("Hardening function '%s'" % (f.__name__), level=DEBUG)
67 _logged['done'] = True
54 RUN_CATALOG = OrderedDict([('os', run_os_checks), 68 RUN_CATALOG = OrderedDict([('os', run_os_checks),
55 ('ssh', run_ssh_checks), 69 ('ssh', run_ssh_checks),
56 ('mysql', run_mysql_checks), 70 ('mysql', run_mysql_checks),
57 ('apache', run_apache_checks)]) 71 ('apache', run_apache_checks)])
58 72
59 enabled = overrides or (config("harden") or "").split() 73 enabled = overrides[:] or (config("harden") or "").split()
60 if enabled: 74 if enabled:
61 modules_to_run = [] 75 modules_to_run = []
62 # modules will always be performed in the following order 76 # modules will always be performed in the following order
diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py
index 936b403..9133e9b 100644
--- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py
@@ -618,12 +618,12 @@ class OpenStackAmuletUtils(AmuletUtils):
618 return self.authenticate_keystone(keystone_ip, user, password, 618 return self.authenticate_keystone(keystone_ip, user, password,
619 project_name=tenant) 619 project_name=tenant)
620 620
621 def authenticate_glance_admin(self, keystone): 621 def authenticate_glance_admin(self, keystone, force_v1_client=False):
622 """Authenticates admin user with glance.""" 622 """Authenticates admin user with glance."""
623 self.log.debug('Authenticating glance admin...') 623 self.log.debug('Authenticating glance admin...')
624 ep = keystone.service_catalog.url_for(service_type='image', 624 ep = keystone.service_catalog.url_for(service_type='image',
625 interface='adminURL') 625 interface='adminURL')
626 if keystone.session: 626 if not force_v1_client and keystone.session:
627 return glance_clientv2.Client("2", session=keystone.session) 627 return glance_clientv2.Client("2", session=keystone.session)
628 else: 628 else:
629 return glance_client.Client(ep, token=keystone.auth_token) 629 return glance_client.Client(ep, token=keystone.auth_token)
@@ -680,18 +680,30 @@ class OpenStackAmuletUtils(AmuletUtils):
680 nova.flavors.create(name, ram, vcpus, disk, flavorid, 680 nova.flavors.create(name, ram, vcpus, disk, flavorid,
681 ephemeral, swap, rxtx_factor, is_public) 681 ephemeral, swap, rxtx_factor, is_public)
682 682
683 def create_cirros_image(self, glance, image_name): 683 def glance_create_image(self, glance, image_name, image_url,
684 """Download the latest cirros image and upload it to glance, 684 download_dir='tests',
685 validate and return a resource pointer. 685 hypervisor_type=None,
686 686 disk_format='qcow2',
687 :param glance: pointer to authenticated glance connection 687 architecture='x86_64',
688 container_format='bare'):
689 """Download an image and upload it to glance, validate its status
690 and return an image object pointer. KVM defaults, can override for
691 LXD.
692
693 :param glance: pointer to authenticated glance api connection
688 :param image_name: display name for new image 694 :param image_name: display name for new image
695 :param image_url: url to retrieve
696 :param download_dir: directory to store downloaded image file
697 :param hypervisor_type: glance image hypervisor property
698 :param disk_format: glance image disk format
699 :param architecture: glance image architecture property
700 :param container_format: glance image container format
689 :returns: glance image pointer 701 :returns: glance image pointer
690 """ 702 """
691 self.log.debug('Creating glance cirros image ' 703 self.log.debug('Creating glance image ({}) from '
692 '({})...'.format(image_name)) 704 '{}...'.format(image_name, image_url))
693 705
694 # Download cirros image 706 # Download image
695 http_proxy = os.getenv('AMULET_HTTP_PROXY') 707 http_proxy = os.getenv('AMULET_HTTP_PROXY')
696 self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy)) 708 self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
697 if http_proxy: 709 if http_proxy:
@@ -700,31 +712,34 @@ class OpenStackAmuletUtils(AmuletUtils):
700 else: 712 else:
701 opener = urllib.FancyURLopener() 713 opener = urllib.FancyURLopener()
702 714
703 f = opener.open('http://download.cirros-cloud.net/version/released') 715 abs_file_name = os.path.join(download_dir, image_name)
704 version = f.read().strip() 716 if not os.path.exists(abs_file_name):
705 cirros_img = 'cirros-{}-x86_64-disk.img'.format(version) 717 opener.retrieve(image_url, abs_file_name)
706 local_path = os.path.join('tests', cirros_img)
707
708 if not os.path.exists(local_path):
709 cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
710 version, cirros_img)
711 opener.retrieve(cirros_url, local_path)
712 f.close()
713 718
714 # Create glance image 719 # Create glance image
720 glance_properties = {
721 'architecture': architecture,
722 }
723 if hypervisor_type:
724 glance_properties['hypervisor_type'] = hypervisor_type
725 # Create glance image
715 if float(glance.version) < 2.0: 726 if float(glance.version) < 2.0:
716 with open(local_path) as fimage: 727 with open(abs_file_name) as f:
717 image = glance.images.create(name=image_name, is_public=True, 728 image = glance.images.create(
718 disk_format='qcow2', 729 name=image_name,
719 container_format='bare', 730 is_public=True,
720 data=fimage) 731 disk_format=disk_format,
732 container_format=container_format,
733 properties=glance_properties,
734 data=f)
721 else: 735 else:
722 image = glance.images.create( 736 image = glance.images.create(
723 name=image_name, 737 name=image_name,
724 disk_format="qcow2",
725 visibility="public", 738 visibility="public",
726 container_format="bare") 739 disk_format=disk_format,
727 glance.images.upload(image.id, open(local_path, 'rb')) 740 container_format=container_format)
741 glance.images.upload(image.id, open(abs_file_name, 'rb'))
742 glance.images.update(image.id, **glance_properties)
728 743
729 # Wait for image to reach active status 744 # Wait for image to reach active status
730 img_id = image.id 745 img_id = image.id
@@ -753,15 +768,54 @@ class OpenStackAmuletUtils(AmuletUtils):
753 val_img_stat, val_img_cfmt, val_img_dfmt)) 768 val_img_stat, val_img_cfmt, val_img_dfmt))
754 769
755 if val_img_name == image_name and val_img_stat == 'active' \ 770 if val_img_name == image_name and val_img_stat == 'active' \
756 and val_img_pub is True and val_img_cfmt == 'bare' \ 771 and val_img_pub is True and val_img_cfmt == container_format \
757 and val_img_dfmt == 'qcow2': 772 and val_img_dfmt == disk_format:
758 self.log.debug(msg_attr) 773 self.log.debug(msg_attr)
759 else: 774 else:
760 msg = ('Volume validation failed, {}'.format(msg_attr)) 775 msg = ('Image validation failed, {}'.format(msg_attr))
761 amulet.raise_status(amulet.FAIL, msg=msg) 776 amulet.raise_status(amulet.FAIL, msg=msg)
762 777
763 return image 778 return image
764 779
780 def create_cirros_image(self, glance, image_name, hypervisor_type=None):
781 """Download the latest cirros image and upload it to glance,
782 validate and return a resource pointer.
783
784 :param glance: pointer to authenticated glance connection
785 :param image_name: display name for new image
786 :param hypervisor_type: glance image hypervisor property
787 :returns: glance image pointer
788 """
789 # /!\ DEPRECATION WARNING
790 self.log.warn('/!\\ DEPRECATION WARNING: use '
791 'glance_create_image instead of '
792 'create_cirros_image.')
793
794 self.log.debug('Creating glance cirros image '
795 '({})...'.format(image_name))
796
797 # Get cirros image URL
798 http_proxy = os.getenv('AMULET_HTTP_PROXY')
799 self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
800 if http_proxy:
801 proxies = {'http': http_proxy}
802 opener = urllib.FancyURLopener(proxies)
803 else:
804 opener = urllib.FancyURLopener()
805
806 f = opener.open('http://download.cirros-cloud.net/version/released')
807 version = f.read().strip()
808 cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
809 cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
810 version, cirros_img)
811 f.close()
812
813 return self.glance_create_image(
814 glance,
815 image_name,
816 cirros_url,
817 hypervisor_type=hypervisor_type)
818
765 def delete_image(self, glance, image): 819 def delete_image(self, glance, image):
766 """Delete the specified image.""" 820 """Delete the specified image."""
767 821
diff --git a/hooks/charmhelpers/contrib/openstack/cert_utils.py b/hooks/charmhelpers/contrib/openstack/cert_utils.py
index de853b5..3e07870 100644
--- a/hooks/charmhelpers/contrib/openstack/cert_utils.py
+++ b/hooks/charmhelpers/contrib/openstack/cert_utils.py
@@ -25,7 +25,9 @@ from charmhelpers.core.hookenv import (
25 local_unit, 25 local_unit,
26 network_get_primary_address, 26 network_get_primary_address,
27 config, 27 config,
28 related_units,
28 relation_get, 29 relation_get,
30 relation_ids,
29 unit_get, 31 unit_get,
30 NoNetworkBinding, 32 NoNetworkBinding,
31 log, 33 log,
@@ -225,3 +227,49 @@ def process_certificates(service_name, relation_id, unit,
225 create_ip_cert_links( 227 create_ip_cert_links(
226 ssl_dir, 228 ssl_dir,
227 custom_hostname_link=custom_hostname_link) 229 custom_hostname_link=custom_hostname_link)
230
231
232def get_requests_for_local_unit(relation_name=None):
233 """Extract any certificates data targeted at this unit down relation_name.
234
235 :param relation_name: str Name of relation to check for data.
236 :returns: List of bundles of certificates.
237 :rtype: List of dicts
238 """
239 local_name = local_unit().replace('/', '_')
240 raw_certs_key = '{}.processed_requests'.format(local_name)
241 relation_name = relation_name or 'certificates'
242 bundles = []
243 for rid in relation_ids(relation_name):
244 for unit in related_units(rid):
245 data = relation_get(rid=rid, unit=unit)
246 if data.get(raw_certs_key):
247 bundles.append({
248 'ca': data['ca'],
249 'chain': data.get('chain'),
250 'certs': json.loads(data[raw_certs_key])})
251 return bundles
252
253
254def get_bundle_for_cn(cn, relation_name=None):
255 """Extract certificates for the given cn.
256
257 :param cn: str Canonical Name on certificate.
258 :param relation_name: str Relation to check for certificates down.
259 :returns: Dictionary of certificate data,
260 :rtype: dict.
261 """
262 entries = get_requests_for_local_unit(relation_name)
263 cert_bundle = {}
264 for entry in entries:
265 for _cn, bundle in entry['certs'].items():
266 if _cn == cn:
267 cert_bundle = {
268 'cert': bundle['cert'],
269 'key': bundle['key'],
270 'chain': entry['chain'],
271 'ca': entry['ca']}
272 break
273 if cert_bundle:
274 break
275 return cert_bundle
diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py
index ca91396..72084cb 100644
--- a/hooks/charmhelpers/contrib/openstack/context.py
+++ b/hooks/charmhelpers/contrib/openstack/context.py
@@ -642,7 +642,7 @@ class HAProxyContext(OSContextGenerator):
642 return {} 642 return {}
643 643
644 l_unit = local_unit().replace('/', '-') 644 l_unit = local_unit().replace('/', '-')
645 cluster_hosts = {} 645 cluster_hosts = collections.OrderedDict()
646 646
647 # NOTE(jamespage): build out map of configured network endpoints 647 # NOTE(jamespage): build out map of configured network endpoints
648 # and associated backends 648 # and associated backends
@@ -1519,6 +1519,10 @@ class NeutronAPIContext(OSContextGenerator):
1519 'rel_key': 'enable-qos', 1519 'rel_key': 'enable-qos',
1520 'default': False, 1520 'default': False,
1521 }, 1521 },
1522 'enable_nsg_logging': {
1523 'rel_key': 'enable-nsg-logging',
1524 'default': False,
1525 },
1522 } 1526 }
1523 ctxt = self.get_neutron_options({}) 1527 ctxt = self.get_neutron_options({})
1524 for rid in relation_ids('neutron-plugin-api'): 1528 for rid in relation_ids('neutron-plugin-api'):
@@ -1530,10 +1534,15 @@ class NeutronAPIContext(OSContextGenerator):
1530 if 'l2-population' in rdata: 1534 if 'l2-population' in rdata:
1531 ctxt.update(self.get_neutron_options(rdata)) 1535 ctxt.update(self.get_neutron_options(rdata))
1532 1536
1537 extension_drivers = []
1538
1533 if ctxt['enable_qos']: 1539 if ctxt['enable_qos']:
1534 ctxt['extension_drivers'] = 'qos' 1540 extension_drivers.append('qos')
1535 else: 1541
1536 ctxt['extension_drivers'] = '' 1542 if ctxt['enable_nsg_logging']:
1543 extension_drivers.append('log')
1544
1545 ctxt['extension_drivers'] = ','.join(extension_drivers)
1537 1546
1538 return ctxt 1547 return ctxt
1539 1548
@@ -1893,7 +1902,7 @@ class EnsureDirContext(OSContextGenerator):
1893 Some software requires a user to create a target directory to be 1902 Some software requires a user to create a target directory to be
1894 scanned for drop-in files with a specific format. This is why this 1903 scanned for drop-in files with a specific format. This is why this
1895 context is needed to do that before rendering a template. 1904 context is needed to do that before rendering a template.
1896 ''' 1905 '''
1897 1906
1898 def __init__(self, dirname, **kwargs): 1907 def __init__(self, dirname, **kwargs):
1899 '''Used merely to ensure that a given directory exists.''' 1908 '''Used merely to ensure that a given directory exists.'''
@@ -1903,3 +1912,23 @@ class EnsureDirContext(OSContextGenerator):
1903 def __call__(self): 1912 def __call__(self):
1904 mkdir(self.dirname, **self.kwargs) 1913 mkdir(self.dirname, **self.kwargs)
1905 return {} 1914 return {}
1915
1916
1917class VersionsContext(OSContextGenerator):
1918 """Context to return the openstack and operating system versions.
1919
1920 """
1921 def __init__(self, pkg='python-keystone'):
1922 """Initialise context.
1923
1924 :param pkg: Package to extrapolate openstack version from.
1925 :type pkg: str
1926 """
1927 self.pkg = pkg
1928
1929 def __call__(self):
1930 ostack = os_release(self.pkg, base='icehouse')
1931 osystem = lsb_release()['DISTRIB_CODENAME'].lower()
1932 return {
1933 'openstack_release': ostack,
1934 'operating_system_release': osystem}
diff --git a/hooks/charmhelpers/contrib/openstack/ha/utils.py b/hooks/charmhelpers/contrib/openstack/ha/utils.py
index 6060ae5..add8eb9 100644
--- a/hooks/charmhelpers/contrib/openstack/ha/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/ha/utils.py
@@ -28,6 +28,7 @@ import json
28import re 28import re
29 29
30from charmhelpers.core.hookenv import ( 30from charmhelpers.core.hookenv import (
31 expected_related_units,
31 log, 32 log,
32 relation_set, 33 relation_set,
33 charm_name, 34 charm_name,
@@ -110,12 +111,17 @@ def assert_charm_supports_dns_ha():
110def expect_ha(): 111def expect_ha():
111 """ Determine if the unit expects to be in HA 112 """ Determine if the unit expects to be in HA
112 113
113 Check for VIP or dns-ha settings which indicate the unit should expect to 114 Check juju goal-state if ha relation is expected, check for VIP or dns-ha
114 be related to hacluster. 115 settings which indicate the unit should expect to be related to hacluster.
115 116
116 @returns boolean 117 @returns boolean
117 """ 118 """
118 return config('vip') or config('dns-ha') 119 ha_related_units = []
120 try:
121 ha_related_units = list(expected_related_units(reltype='ha'))
122 except (NotImplementedError, KeyError):
123 pass
124 return len(ha_related_units) > 0 or config('vip') or config('dns-ha')
119 125
120 126
121def generate_ha_relation_data(service): 127def generate_ha_relation_data(service):
diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py
index 24f5b80..29cad08 100644
--- a/hooks/charmhelpers/contrib/openstack/utils.py
+++ b/hooks/charmhelpers/contrib/openstack/utils.py
@@ -186,7 +186,7 @@ SWIFT_CODENAMES = OrderedDict([
186 ('queens', 186 ('queens',
187 ['2.16.0', '2.17.0']), 187 ['2.16.0', '2.17.0']),
188 ('rocky', 188 ('rocky',
189 ['2.18.0']), 189 ['2.18.0', '2.19.0']),
190]) 190])
191 191
192# >= Liberty version->codename mapping 192# >= Liberty version->codename mapping
@@ -375,7 +375,7 @@ def get_swift_codename(version):
375 return codenames[0] 375 return codenames[0]
376 376
377 # NOTE: fallback - attempt to match with just major.minor version 377 # NOTE: fallback - attempt to match with just major.minor version
378 match = re.match('^(\d+)\.(\d+)', version) 378 match = re.match(r'^(\d+)\.(\d+)', version)
379 if match: 379 if match:
380 major_minor_version = match.group(0) 380 major_minor_version = match.group(0)
381 for codename, versions in six.iteritems(SWIFT_CODENAMES): 381 for codename, versions in six.iteritems(SWIFT_CODENAMES):
@@ -395,7 +395,7 @@ def get_os_codename_package(package, fatal=True):
395 out = subprocess.check_output(cmd) 395 out = subprocess.check_output(cmd)
396 if six.PY3: 396 if six.PY3:
397 out = out.decode('UTF-8') 397 out = out.decode('UTF-8')
398 except subprocess.CalledProcessError as e: 398 except subprocess.CalledProcessError:
399 return None 399 return None
400 lines = out.split('\n') 400 lines = out.split('\n')
401 for line in lines: 401 for line in lines:
@@ -427,11 +427,11 @@ def get_os_codename_package(package, fatal=True):
427 vers = apt.upstream_version(pkg.current_ver.ver_str) 427 vers = apt.upstream_version(pkg.current_ver.ver_str)
428 if 'swift' in pkg.name: 428 if 'swift' in pkg.name:
429 # Fully x.y.z match for swift versions 429 # Fully x.y.z match for swift versions
430 match = re.match('^(\d+)\.(\d+)\.(\d+)', vers) 430 match = re.match(r'^(\d+)\.(\d+)\.(\d+)', vers)
431 else: 431 else:
432 # x.y match only for 20XX.X 432 # x.y match only for 20XX.X
433 # and ignore patch level for other packages 433 # and ignore patch level for other packages
434 match = re.match('^(\d+)\.(\d+)', vers) 434 match = re.match(r'^(\d+)\.(\d+)', vers)
435 435
436 if match: 436 if match:
437 vers = match.group(0) 437 vers = match.group(0)
@@ -1450,20 +1450,33 @@ def pausable_restart_on_change(restart_map, stopstart=False,
1450 1450
1451 see core.utils.restart_on_change() for more details. 1451 see core.utils.restart_on_change() for more details.
1452 1452
1453 Note restart_map can be a callable, in which case, restart_map is only
1454 evaluated at runtime. This means that it is lazy and the underlying
1455 function won't be called if the decorated function is never called. Note,
1456 retains backwards compatibility for passing a non-callable dictionary.
1457
1453 @param f: the function to decorate 1458 @param f: the function to decorate
1454 @param restart_map: the restart map {conf_file: [services]} 1459 @param restart_map: (optionally callable, which then returns the
1460 restart_map) the restart map {conf_file: [services]}
1455 @param stopstart: DEFAULT false; whether to stop, start or just restart 1461 @param stopstart: DEFAULT false; whether to stop, start or just restart
1456 @returns decorator to use a restart_on_change with pausability 1462 @returns decorator to use a restart_on_change with pausability
1457 """ 1463 """
1458 def wrap(f): 1464 def wrap(f):
1465 # py27 compatible nonlocal variable. When py3 only, replace with
1466 # nonlocal keyword
1467 __restart_map_cache = {'cache': None}
1468
1459 @functools.wraps(f) 1469 @functools.wraps(f)
1460 def wrapped_f(*args, **kwargs): 1470 def wrapped_f(*args, **kwargs):
1461 if is_unit_paused_set(): 1471 if is_unit_paused_set():
1462 return f(*args, **kwargs) 1472 return f(*args, **kwargs)
1473 if __restart_map_cache['cache'] is None:
1474 __restart_map_cache['cache'] = restart_map() \
1475 if callable(restart_map) else restart_map
1463 # otherwise, normal restart_on_change functionality 1476 # otherwise, normal restart_on_change functionality
1464 return restart_on_change_helper( 1477 return restart_on_change_helper(
1465 (lambda: f(*args, **kwargs)), restart_map, stopstart, 1478 (lambda: f(*args, **kwargs)), __restart_map_cache['cache'],
1466 restart_functions) 1479 stopstart, restart_functions)
1467 return wrapped_f 1480 return wrapped_f
1468 return wrap 1481 return wrap
1469 1482
@@ -1733,3 +1746,31 @@ def is_unit_upgrading_set():
1733 return not(not(kv.get('unit-upgrading'))) 1746 return not(not(kv.get('unit-upgrading')))
1734 except Exception: 1747 except Exception:
1735 return False 1748 return False
1749
1750
1751def series_upgrade_prepare(pause_unit_helper=None, configs=None):
1752 """ Run common series upgrade prepare tasks.
1753
1754 :param pause_unit_helper: function: Function to pause unit
1755 :param configs: OSConfigRenderer object: Configurations
1756 :returns None:
1757 """
1758 set_unit_upgrading()
1759 if pause_unit_helper and configs:
1760 if not is_unit_paused_set():
1761 pause_unit_helper(configs)
1762
1763
1764def series_upgrade_complete(resume_unit_helper=None, configs=None):
1765 """ Run common series upgrade complete tasks.
1766
1767 :param resume_unit_helper: function: Function to resume unit
1768 :param configs: OSConfigRenderer object: Configurations
1769 :returns None:
1770 """
1771 clear_unit_paused()
1772 clear_unit_upgrading()
1773 if configs:
1774 configs.write_all()
1775 if resume_unit_helper:
1776 resume_unit_helper(configs)
diff --git a/hooks/charmhelpers/contrib/storage/linux/loopback.py b/hooks/charmhelpers/contrib/storage/linux/loopback.py
index 1d6ae6f..0dfdae5 100644
--- a/hooks/charmhelpers/contrib/storage/linux/loopback.py
+++ b/hooks/charmhelpers/contrib/storage/linux/loopback.py
@@ -39,7 +39,7 @@ def loopback_devices():
39 devs = [d.strip().split(' ') for d in 39 devs = [d.strip().split(' ') for d in
40 check_output(cmd).splitlines() if d != ''] 40 check_output(cmd).splitlines() if d != '']
41 for dev, _, f in devs: 41 for dev, _, f in devs:
42 loopbacks[dev.replace(':', '')] = re.search('\((\S+)\)', f).groups()[0] 42 loopbacks[dev.replace(':', '')] = re.search(r'\((\S+)\)', f).groups()[0]
43 return loopbacks 43 return loopbacks
44 44
45 45
diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py
index 6880007..2e28765 100644
--- a/hooks/charmhelpers/core/hookenv.py
+++ b/hooks/charmhelpers/core/hookenv.py
@@ -48,6 +48,7 @@ INFO = "INFO"
48DEBUG = "DEBUG" 48DEBUG = "DEBUG"
49TRACE = "TRACE" 49TRACE = "TRACE"
50MARKER = object() 50MARKER = object()
51SH_MAX_ARG = 131071
51 52
52cache = {} 53cache = {}
53 54
@@ -98,7 +99,7 @@ def log(message, level=None):
98 command += ['-l', level] 99 command += ['-l', level]
99 if not isinstance(message, six.string_types): 100 if not isinstance(message, six.string_types):
100 message = repr(message) 101 message = repr(message)
101 command += [message] 102 command += [message[:SH_MAX_ARG]]
102 # Missing juju-log should not cause failures in unit tests 103 # Missing juju-log should not cause failures in unit tests
103 # Send log output to stderr 104 # Send log output to stderr
104 try: 105 try:
@@ -509,6 +510,67 @@ def related_units(relid=None):
509 subprocess.check_output(units_cmd_line).decode('UTF-8')) or [] 510 subprocess.check_output(units_cmd_line).decode('UTF-8')) or []
510 511
511 512
513def expected_peer_units():
514 """Get a generator for units we expect to join peer relation based on
515 goal-state.
516
517 The local unit is excluded from the result to make it easy to gauge
518 completion of all peers joining the relation with existing hook tools.
519
520 Example usage:
521 log('peer {} of {} joined peer relation'
522 .format(len(related_units()),
523 len(list(expected_peer_units()))))
524
525 This function will raise NotImplementedError if used with juju versions
526 without goal-state support.
527
528 :returns: iterator
529 :rtype: types.GeneratorType
530 :raises: NotImplementedError
531 """
532 if not has_juju_version("2.4.0"):
533 # goal-state first appeared in 2.4.0.
534 raise NotImplementedError("goal-state")
535 _goal_state = goal_state()
536 return (key for key in _goal_state['units']
537 if '/' in key and key != local_unit())
538
539
540def expected_related_units(reltype=None):
541 """Get a generator for units we expect to join relation based on
542 goal-state.
543
544 Note that you can not use this function for the peer relation, take a look
545 at expected_peer_units() for that.
546
547 This function will raise KeyError if you request information for a
548 relation type for which juju goal-state does not have information. It will
549 raise NotImplementedError if used with juju versions without goal-state
550 support.
551
552 Example usage:
553 log('participant {} of {} joined relation {}'
554 .format(len(related_units()),
555 len(list(expected_related_units())),
556 relation_type()))
557
558 :param reltype: Relation type to list data for, default is to list data for
559 the realtion type we are currently executing a hook for.
560 :type reltype: str
561 :returns: iterator
562 :rtype: types.GeneratorType
563 :raises: KeyError, NotImplementedError
564 """
565 if not has_juju_version("2.4.4"):
566 # goal-state existed in 2.4.0, but did not list individual units to
567 # join a relation in 2.4.1 through 2.4.3. (LP: #1794739)
568 raise NotImplementedError("goal-state relation unit count")
569 reltype = reltype or relation_type()
570 _goal_state = goal_state()
571 return (key for key in _goal_state['relations'][reltype] if '/' in key)
572
573
512@cached 574@cached
513def relation_for_unit(unit=None, rid=None): 575def relation_for_unit(unit=None, rid=None):
514 """Get the json represenation of a unit's relation""" 576 """Get the json represenation of a unit's relation"""
@@ -997,6 +1059,7 @@ def application_version_set(version):
997 1059
998 1060
999@translate_exc(from_exc=OSError, to_exc=NotImplementedError) 1061@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
1062@cached
1000def goal_state(): 1063def goal_state():
1001 """Juju goal state values""" 1064 """Juju goal state values"""
1002 cmd = ['goal-state', '--format=json'] 1065 cmd = ['goal-state', '--format=json']
diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py
index e9fd38a..79953a4 100644
--- a/hooks/charmhelpers/core/host.py
+++ b/hooks/charmhelpers/core/host.py
@@ -34,13 +34,13 @@ import six
34 34
35from contextlib import contextmanager 35from contextlib import contextmanager
36from collections import OrderedDict 36from collections import OrderedDict
37from .hookenv import log, DEBUG, local_unit 37from .hookenv import log, INFO, DEBUG, local_unit, charm_name
38from .fstab import Fstab 38from .fstab import Fstab
39from charmhelpers.osplatform import get_platform 39from charmhelpers.osplatform import get_platform
40 40
41__platform__ = get_platform() 41__platform__ = get_platform()
42if __platform__ == "ubuntu": 42if __platform__ == "ubuntu":
43 from charmhelpers.core.host_factory.ubuntu import ( 43 from charmhelpers.core.host_factory.ubuntu import ( # NOQA:F401
44 service_available, 44 service_available,
45 add_new_group, 45 add_new_group,
46 lsb_release, 46 lsb_release,
@@ -48,7 +48,7 @@ if __platform__ == "ubuntu":
48 CompareHostReleases, 48 CompareHostReleases,
49 ) # flake8: noqa -- ignore F401 for this import 49 ) # flake8: noqa -- ignore F401 for this import
50elif __platform__ == "centos": 50elif __platform__ == "centos":
51 from charmhelpers.core.host_factory.centos import ( 51 from charmhelpers.core.host_factory.centos import ( # NOQA:F401
52 service_available, 52 service_available,
53 add_new_group, 53 add_new_group,
54 lsb_release, 54 lsb_release,
@@ -58,6 +58,7 @@ elif __platform__ == "centos":
58 58
59UPDATEDB_PATH = '/etc/updatedb.conf' 59UPDATEDB_PATH = '/etc/updatedb.conf'
60 60
61
61def service_start(service_name, **kwargs): 62def service_start(service_name, **kwargs):
62 """Start a system service. 63 """Start a system service.
63 64
@@ -287,8 +288,8 @@ def service_running(service_name, **kwargs):
287 for key, value in six.iteritems(kwargs): 288 for key, value in six.iteritems(kwargs):
288 parameter = '%s=%s' % (key, value) 289 parameter = '%s=%s' % (key, value)
289 cmd.append(parameter) 290 cmd.append(parameter)
290 output = subprocess.check_output(cmd, 291 output = subprocess.check_output(
291 stderr=subprocess.STDOUT).decode('UTF-8') 292 cmd, stderr=subprocess.STDOUT).decode('UTF-8')
292 except subprocess.CalledProcessError: 293 except subprocess.CalledProcessError:
293 return False 294 return False
294 else: 295 else:
@@ -442,7 +443,7 @@ def add_user_to_group(username, group):
442 443
443 444
444def chage(username, lastday=None, expiredate=None, inactive=None, 445def chage(username, lastday=None, expiredate=None, inactive=None,
445 mindays=None, maxdays=None, root=None, warndays=None): 446 mindays=None, maxdays=None, root=None, warndays=None):
446 """Change user password expiry information 447 """Change user password expiry information
447 448
448 :param str username: User to update 449 :param str username: User to update
@@ -482,8 +483,10 @@ def chage(username, lastday=None, expiredate=None, inactive=None,
482 cmd.append(username) 483 cmd.append(username)
483 subprocess.check_call(cmd) 484 subprocess.check_call(cmd)
484 485
486
485remove_password_expiry = functools.partial(chage, expiredate='-1', inactive='-1', mindays='0', maxdays='-1') 487remove_password_expiry = functools.partial(chage, expiredate='-1', inactive='-1', mindays='0', maxdays='-1')
486 488
489
487def rsync(from_path, to_path, flags='-r', options=None, timeout=None): 490def rsync(from_path, to_path, flags='-r', options=None, timeout=None):
488 """Replicate the contents of a path""" 491 """Replicate the contents of a path"""
489 options = options or ['--delete', '--executability'] 492 options = options or ['--delete', '--executability']
@@ -535,13 +538,15 @@ def write_file(path, content, owner='root', group='root', perms=0o444):
535 # lets see if we can grab the file and compare the context, to avoid doing 538 # lets see if we can grab the file and compare the context, to avoid doing
536 # a write. 539 # a write.
537 existing_content = None 540 existing_content = None
538 existing_uid, existing_gid = None, None 541 existing_uid, existing_gid, existing_perms = None, None, None
539 try: 542 try:
540 with open(path, 'rb') as target: 543 with open(path, 'rb') as target:
541 existing_content = target.read() 544 existing_content = target.read()
542 stat = os.stat(path) 545 stat = os.stat(path)
543 existing_uid, existing_gid = stat.st_uid, stat.st_gid 546 existing_uid, existing_gid, existing_perms = (
544 except: 547 stat.st_uid, stat.st_gid, stat.st_mode
548 )
549 except Exception:
545 pass 550 pass
546 if content != existing_content: 551 if content != existing_content:
547 log("Writing file {} {}:{} {:o}".format(path, owner, group, perms), 552 log("Writing file {} {}:{} {:o}".format(path, owner, group, perms),
@@ -554,7 +559,7 @@ def write_file(path, content, owner='root', group='root', perms=0o444):
554 target.write(content) 559 target.write(content)
555 return 560 return
556 # the contents were the same, but we might still need to change the 561 # the contents were the same, but we might still need to change the
557 # ownership. 562 # ownership or permissions.
558 if existing_uid != uid: 563 if existing_uid != uid:
559 log("Changing uid on already existing content: {} -> {}" 564 log("Changing uid on already existing content: {} -> {}"
560 .format(existing_uid, uid), level=DEBUG) 565 .format(existing_uid, uid), level=DEBUG)
@@ -563,6 +568,10 @@ def write_file(path, content, owner='root', group='root', perms=0o444):
563 log("Changing gid on already existing content: {} -> {}" 568 log("Changing gid on already existing content: {} -> {}"
564 .format(existing_gid, gid), level=DEBUG) 569 .format(existing_gid, gid), level=DEBUG)
565 os.chown(path, -1, gid) 570 os.chown(path, -1, gid)
571 if existing_perms != perms:
572 log("Changing permissions on existing content: {} -> {}"
573 .format(existing_perms, perms), level=DEBUG)
574 os.chmod(path, perms)
566 575
567 576
568def fstab_remove(mp): 577def fstab_remove(mp):
@@ -827,7 +836,7 @@ def list_nics(nic_type=None):
827 ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n') 836 ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
828 ip_output = (line.strip() for line in ip_output if line) 837 ip_output = (line.strip() for line in ip_output if line)
829 838
830 key = re.compile('^[0-9]+:\s+(.+):') 839 key = re.compile(r'^[0-9]+:\s+(.+):')
831 for line in ip_output: 840 for line in ip_output:
832 matched = re.search(key, line) 841 matched = re.search(key, line)
833 if matched: 842 if matched:
@@ -1040,3 +1049,27 @@ def modulo_distribution(modulo=3, wait=30, non_zero_wait=False):
1040 return modulo * wait 1049 return modulo * wait
1041 else: 1050 else:
1042 return calculated_wait_time 1051 return calculated_wait_time
1052
1053
1054def install_ca_cert(ca_cert, name=None):
1055 """
1056 Install the given cert as a trusted CA.
1057
1058 The ``name`` is the stem of the filename where the cert is written, and if
1059 not provided, it will default to ``juju-{charm_name}``.
1060
1061 If the cert is empty or None, or is unchanged, nothing is done.
1062 """
1063 if not ca_cert:
1064 return
1065 if not isinstance(ca_cert, bytes):
1066 ca_cert = ca_cert.encode('utf8')
1067 if not name:
1068 name = 'juju-{}'.format(charm_name())
1069 cert_file = '/usr/local/share/ca-certificates/{}.crt'.format(name)
1070 new_hash = hashlib.md5(ca_cert).hexdigest()
1071 if file_hash(cert_file) == new_hash:
1072 return
1073 log("Installing new CA cert at: {}".format(cert_file), level=INFO)
1074 write_file(cert_file, ca_cert)
1075 subprocess.check_call(['update-ca-certificates', '--fresh'])
diff --git a/hooks/charmhelpers/core/kernel.py b/hooks/charmhelpers/core/kernel.py
index 2d40452..e01f4f8 100644
--- a/hooks/charmhelpers/core/kernel.py
+++ b/hooks/charmhelpers/core/kernel.py
@@ -26,12 +26,12 @@ from charmhelpers.core.hookenv import (
26 26
27__platform__ = get_platform() 27__platform__ = get_platform()
28if __platform__ == "ubuntu": 28if __platform__ == "ubuntu":
29 from charmhelpers.core.kernel_factory.ubuntu import ( 29 from charmhelpers.core.kernel_factory.ubuntu import ( # NOQA:F401
30 persistent_modprobe, 30 persistent_modprobe,
31 update_initramfs, 31 update_initramfs,
32 ) # flake8: noqa -- ignore F401 for this import 32 ) # flake8: noqa -- ignore F401 for this import
33elif __platform__ == "centos": 33elif __platform__ == "centos":
34 from charmhelpers.core.kernel_factory.centos import ( 34 from charmhelpers.core.kernel_factory.centos import ( # NOQA:F401
35 persistent_modprobe, 35 persistent_modprobe,
36 update_initramfs, 36 update_initramfs,
37 ) # flake8: noqa -- ignore F401 for this import 37 ) # flake8: noqa -- ignore F401 for this import
diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py
index 480a627..8572d34 100644
--- a/hooks/charmhelpers/fetch/__init__.py
+++ b/hooks/charmhelpers/fetch/__init__.py
@@ -84,6 +84,7 @@ module = "charmhelpers.fetch.%s" % __platform__
84fetch = importlib.import_module(module) 84fetch = importlib.import_module(module)
85 85
86filter_installed_packages = fetch.filter_installed_packages 86filter_installed_packages = fetch.filter_installed_packages
87filter_missing_packages = fetch.filter_missing_packages
87install = fetch.apt_install 88install = fetch.apt_install
88upgrade = fetch.apt_upgrade 89upgrade = fetch.apt_upgrade
89update = _fetch_update = fetch.apt_update 90update = _fetch_update = fetch.apt_update
@@ -96,6 +97,7 @@ if __platform__ == "ubuntu":
96 apt_update = fetch.apt_update 97 apt_update = fetch.apt_update
97 apt_upgrade = fetch.apt_upgrade 98 apt_upgrade = fetch.apt_upgrade
98 apt_purge = fetch.apt_purge 99 apt_purge = fetch.apt_purge
100 apt_autoremove = fetch.apt_autoremove
99 apt_mark = fetch.apt_mark 101 apt_mark = fetch.apt_mark
100 apt_hold = fetch.apt_hold 102 apt_hold = fetch.apt_hold
101 apt_unhold = fetch.apt_unhold 103 apt_unhold = fetch.apt_unhold
diff --git a/hooks/charmhelpers/fetch/bzrurl.py b/hooks/charmhelpers/fetch/bzrurl.py
index 07cd029..c4ab3ff 100644
--- a/hooks/charmhelpers/fetch/bzrurl.py
+++ b/hooks/charmhelpers/fetch/bzrurl.py
@@ -13,7 +13,7 @@
13# limitations under the License. 13# limitations under the License.
14 14
15import os 15import os
16from subprocess import check_call 16from subprocess import STDOUT, check_output
17from charmhelpers.fetch import ( 17from charmhelpers.fetch import (
18 BaseFetchHandler, 18 BaseFetchHandler,
19 UnhandledSource, 19 UnhandledSource,
@@ -55,7 +55,7 @@ class BzrUrlFetchHandler(BaseFetchHandler):
55 cmd = ['bzr', 'branch'] 55 cmd = ['bzr', 'branch']
56 cmd += cmd_opts 56 cmd += cmd_opts
57 cmd += [source, dest] 57 cmd += [source, dest]
58 check_call(cmd) 58 check_output(cmd, stderr=STDOUT)
59 59
60 def install(self, source, dest=None, revno=None): 60 def install(self, source, dest=None, revno=None):
61 url_parts = self.parse_url(source) 61 url_parts = self.parse_url(source)
diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py
index 4cf21bc..070ca9b 100644
--- a/hooks/charmhelpers/fetch/giturl.py
+++ b/hooks/charmhelpers/fetch/giturl.py
@@ -13,7 +13,7 @@
13# limitations under the License. 13# limitations under the License.
14 14
15import os 15import os
16from subprocess import check_call, CalledProcessError 16from subprocess import check_output, CalledProcessError, STDOUT
17from charmhelpers.fetch import ( 17from charmhelpers.fetch import (
18 BaseFetchHandler, 18 BaseFetchHandler,
19 UnhandledSource, 19 UnhandledSource,
@@ -50,7 +50,7 @@ class GitUrlFetchHandler(BaseFetchHandler):
50 cmd = ['git', 'clone', source, dest, '--branch', branch] 50 cmd = ['git', 'clone', source, dest, '--branch', branch]
51 if depth: 51 if depth:
52 cmd.extend(['--depth', depth]) 52 cmd.extend(['--depth', depth])
53 check_call(cmd) 53 check_output(cmd, stderr=STDOUT)
54 54
55 def install(self, source, branch="master", dest=None, depth=None): 55 def install(self, source, branch="master", dest=None, depth=None):
56 url_parts = self.parse_url(source) 56 url_parts = self.parse_url(source)
diff --git a/hooks/charmhelpers/fetch/ubuntu.py b/hooks/charmhelpers/fetch/ubuntu.py
index 19aa6ba..c7ad128 100644
--- a/hooks/charmhelpers/fetch/ubuntu.py
+++ b/hooks/charmhelpers/fetch/ubuntu.py
@@ -189,6 +189,18 @@ def filter_installed_packages(packages):
189 return _pkgs 189 return _pkgs
190 190
191 191
192def filter_missing_packages(packages):
193 """Return a list of packages that are installed.
194
195 :param packages: list of packages to evaluate.
196 :returns list: Packages that are installed.
197 """
198 return list(
199 set(packages) -
200 set(filter_installed_packages(packages))
201 )
202
203
192def apt_cache(in_memory=True, progress=None): 204def apt_cache(in_memory=True, progress=None):
193 """Build and return an apt cache.""" 205 """Build and return an apt cache."""
194 from apt import apt_pkg 206 from apt import apt_pkg
@@ -248,6 +260,14 @@ def apt_purge(packages, fatal=False):
248 _run_apt_command(cmd, fatal) 260 _run_apt_command(cmd, fatal)
249 261
250 262
263def apt_autoremove(purge=True, fatal=False):
264 """Purge one or more packages."""
265 cmd = ['apt-get', '--assume-yes', 'autoremove']
266 if purge:
267 cmd.append('--purge')
268 _run_apt_command(cmd, fatal)
269
270
251def apt_mark(packages, mark, fatal=False): 271def apt_mark(packages, mark, fatal=False):
252 """Flag one or more packages using apt-mark.""" 272 """Flag one or more packages using apt-mark."""
253 log("Marking {} as {}".format(packages, mark)) 273 log("Marking {} as {}".format(packages, mark))
@@ -274,7 +294,7 @@ def apt_unhold(packages, fatal=False):
274def import_key(key): 294def import_key(key):
275 """Import an ASCII Armor key. 295 """Import an ASCII Armor key.
276 296
277 /!\ A Radix64 format keyid is also supported for backwards 297 A Radix64 format keyid is also supported for backwards
278 compatibility, but should never be used; the key retrieval 298 compatibility, but should never be used; the key retrieval
279 mechanism is insecure and subject to man-in-the-middle attacks 299 mechanism is insecure and subject to man-in-the-middle attacks
280 voiding all signature checks using that key. 300 voiding all signature checks using that key.
@@ -434,6 +454,9 @@ def _add_apt_repository(spec):
434 454
435 :param spec: the parameter to pass to add_apt_repository 455 :param spec: the parameter to pass to add_apt_repository
436 """ 456 """
457 if '{series}' in spec:
458 series = lsb_release()['DISTRIB_CODENAME']
459 spec = spec.replace('{series}', series)
437 _run_with_retries(['add-apt-repository', '--yes', spec]) 460 _run_with_retries(['add-apt-repository', '--yes', spec])
438 461
439 462